Merge pull request #854 from james-6-23/main

feat(admin): 支持定时测试自动恢复并统一账号恢复入口
This commit is contained in:
Wesley Liddick
2026-03-09 08:48:36 +08:00
committed by GitHub
23 changed files with 535 additions and 92 deletions

View File

@@ -229,7 +229,7 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) {
tokenRefreshService := service.ProvideTokenRefreshService(accountRepository, soraAccountRepository, oAuthService, openAIOAuthService, geminiOAuthService, antigravityOAuthService, compositeTokenCacheInvalidator, schedulerCache, configConfig, tempUnschedCache) tokenRefreshService := service.ProvideTokenRefreshService(accountRepository, soraAccountRepository, oAuthService, openAIOAuthService, geminiOAuthService, antigravityOAuthService, compositeTokenCacheInvalidator, schedulerCache, configConfig, tempUnschedCache)
accountExpiryService := service.ProvideAccountExpiryService(accountRepository) accountExpiryService := service.ProvideAccountExpiryService(accountRepository)
subscriptionExpiryService := service.ProvideSubscriptionExpiryService(userSubscriptionRepository) subscriptionExpiryService := service.ProvideSubscriptionExpiryService(userSubscriptionRepository)
scheduledTestRunnerService := service.ProvideScheduledTestRunnerService(scheduledTestPlanRepository, scheduledTestService, accountTestService, configConfig) scheduledTestRunnerService := service.ProvideScheduledTestRunnerService(scheduledTestPlanRepository, scheduledTestService, accountTestService, rateLimitService, configConfig)
v := provideCleanup(client, redisClient, opsMetricsCollector, opsAggregationService, opsAlertEvaluatorService, opsCleanupService, opsScheduledReportService, opsSystemLogSink, soraMediaCleanupService, schedulerSnapshotService, tokenRefreshService, accountExpiryService, subscriptionExpiryService, usageCleanupService, idempotencyCleanupService, pricingService, emailQueueService, billingCacheService, usageRecordWorkerPool, subscriptionService, oAuthService, openAIOAuthService, geminiOAuthService, antigravityOAuthService, openAIGatewayService, scheduledTestRunnerService) v := provideCleanup(client, redisClient, opsMetricsCollector, opsAggregationService, opsAlertEvaluatorService, opsCleanupService, opsScheduledReportService, opsSystemLogSink, soraMediaCleanupService, schedulerSnapshotService, tokenRefreshService, accountExpiryService, subscriptionExpiryService, usageCleanupService, idempotencyCleanupService, pricingService, emailQueueService, billingCacheService, usageRecordWorkerPool, subscriptionService, oAuthService, openAIOAuthService, geminiOAuthService, antigravityOAuthService, openAIGatewayService, scheduledTestRunnerService)
application := &Application{ application := &Application{
Server: httpServer, Server: httpServer,

View File

@@ -660,6 +660,42 @@ func (h *AccountHandler) Test(c *gin.Context) {
// Error already sent via SSE, just log // Error already sent via SSE, just log
return return
} }
if h.rateLimitService != nil {
if _, err := h.rateLimitService.RecoverAccountAfterSuccessfulTest(c.Request.Context(), accountID); err != nil {
_ = c.Error(err)
}
}
}
// RecoverState handles unified recovery of recoverable account runtime state.
// POST /api/v1/admin/accounts/:id/recover-state
func (h *AccountHandler) RecoverState(c *gin.Context) {
accountID, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil {
response.BadRequest(c, "Invalid account ID")
return
}
if h.rateLimitService == nil {
response.Error(c, http.StatusServiceUnavailable, "Rate limit service unavailable")
return
}
if _, err := h.rateLimitService.RecoverAccountState(c.Request.Context(), accountID, service.AccountRecoveryOptions{
InvalidateToken: true,
}); err != nil {
response.ErrorFrom(c, err)
return
}
account, err := h.adminService.GetAccount(c.Request.Context(), accountID)
if err != nil {
response.ErrorFrom(c, err)
return
}
response.Success(c, h.buildAccountResponseWithRuntime(c.Request.Context(), account))
} }
// SyncFromCRS handles syncing accounts from claude-relay-service (CRS) // SyncFromCRS handles syncing accounts from claude-relay-service (CRS)

View File

@@ -25,6 +25,7 @@ type createScheduledTestPlanRequest struct {
CronExpression string `json:"cron_expression" binding:"required"` CronExpression string `json:"cron_expression" binding:"required"`
Enabled *bool `json:"enabled"` Enabled *bool `json:"enabled"`
MaxResults int `json:"max_results"` MaxResults int `json:"max_results"`
AutoRecover *bool `json:"auto_recover"`
} }
type updateScheduledTestPlanRequest struct { type updateScheduledTestPlanRequest struct {
@@ -32,6 +33,7 @@ type updateScheduledTestPlanRequest struct {
CronExpression string `json:"cron_expression"` CronExpression string `json:"cron_expression"`
Enabled *bool `json:"enabled"` Enabled *bool `json:"enabled"`
MaxResults int `json:"max_results"` MaxResults int `json:"max_results"`
AutoRecover *bool `json:"auto_recover"`
} }
// ListByAccount GET /admin/accounts/:id/scheduled-test-plans // ListByAccount GET /admin/accounts/:id/scheduled-test-plans
@@ -68,6 +70,9 @@ func (h *ScheduledTestHandler) Create(c *gin.Context) {
if req.Enabled != nil { if req.Enabled != nil {
plan.Enabled = *req.Enabled plan.Enabled = *req.Enabled
} }
if req.AutoRecover != nil {
plan.AutoRecover = *req.AutoRecover
}
created, err := h.scheduledTestSvc.CreatePlan(c.Request.Context(), plan) created, err := h.scheduledTestSvc.CreatePlan(c.Request.Context(), plan)
if err != nil { if err != nil {
@@ -109,6 +114,9 @@ func (h *ScheduledTestHandler) Update(c *gin.Context) {
if req.MaxResults > 0 { if req.MaxResults > 0 {
existing.MaxResults = req.MaxResults existing.MaxResults = req.MaxResults
} }
if req.AutoRecover != nil {
existing.AutoRecover = *req.AutoRecover
}
updated, err := h.scheduledTestSvc.UpdatePlan(c.Request.Context(), existing) updated, err := h.scheduledTestSvc.UpdatePlan(c.Request.Context(), existing)
if err != nil { if err != nil {

View File

@@ -659,13 +659,10 @@ func (r *accountRepository) ClearError(ctx context.Context, id int64) error {
if err != nil { if err != nil {
return err return err
} }
// 清除临时不可调度状态,重置 401 升级链 if err := enqueueSchedulerOutbox(ctx, r.sql, service.SchedulerOutboxEventAccountChanged, &id, nil, nil); err != nil {
_, _ = r.sql.ExecContext(ctx, ` logger.LegacyPrintf("repository.account", "[SchedulerOutbox] enqueue clear error failed: account=%d err=%v", id, err)
UPDATE accounts }
SET temp_unschedulable_until = NULL, r.syncSchedulerAccountSnapshot(ctx, id)
temp_unschedulable_reason = NULL
WHERE id = $1 AND deleted_at IS NULL
`, id)
return nil return nil
} }

View File

@@ -558,6 +558,26 @@ func (s *AccountRepoSuite) TestSetError() {
s.Require().Equal("something went wrong", got.ErrorMessage) s.Require().Equal("something went wrong", got.ErrorMessage)
} }
func (s *AccountRepoSuite) TestClearError_SyncSchedulerSnapshotOnRecovery() {
account := mustCreateAccount(s.T(), s.client, &service.Account{
Name: "acc-clear-err",
Status: service.StatusError,
ErrorMessage: "temporary error",
})
cacheRecorder := &schedulerCacheRecorder{}
s.repo.schedulerCache = cacheRecorder
s.Require().NoError(s.repo.ClearError(s.ctx, account.ID))
got, err := s.repo.GetByID(s.ctx, account.ID)
s.Require().NoError(err)
s.Require().Equal(service.StatusActive, got.Status)
s.Require().Empty(got.ErrorMessage)
s.Require().Len(cacheRecorder.setAccounts, 1)
s.Require().Equal(account.ID, cacheRecorder.setAccounts[0].ID)
s.Require().Equal(service.StatusActive, cacheRecorder.setAccounts[0].Status)
}
// --- UpdateSessionWindow --- // --- UpdateSessionWindow ---
func (s *AccountRepoSuite) TestUpdateSessionWindow() { func (s *AccountRepoSuite) TestUpdateSessionWindow() {

View File

@@ -20,16 +20,16 @@ func NewScheduledTestPlanRepository(db *sql.DB) service.ScheduledTestPlanReposit
func (r *scheduledTestPlanRepository) Create(ctx context.Context, plan *service.ScheduledTestPlan) (*service.ScheduledTestPlan, error) { func (r *scheduledTestPlanRepository) Create(ctx context.Context, plan *service.ScheduledTestPlan) (*service.ScheduledTestPlan, error) {
row := r.db.QueryRowContext(ctx, ` row := r.db.QueryRowContext(ctx, `
INSERT INTO scheduled_test_plans (account_id, model_id, cron_expression, enabled, max_results, next_run_at, created_at, updated_at) INSERT INTO scheduled_test_plans (account_id, model_id, cron_expression, enabled, max_results, auto_recover, next_run_at, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, NOW(), NOW()) VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW())
RETURNING id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at RETURNING id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
`, plan.AccountID, plan.ModelID, plan.CronExpression, plan.Enabled, plan.MaxResults, plan.NextRunAt) `, plan.AccountID, plan.ModelID, plan.CronExpression, plan.Enabled, plan.MaxResults, plan.AutoRecover, plan.NextRunAt)
return scanPlan(row) return scanPlan(row)
} }
func (r *scheduledTestPlanRepository) GetByID(ctx context.Context, id int64) (*service.ScheduledTestPlan, error) { func (r *scheduledTestPlanRepository) GetByID(ctx context.Context, id int64) (*service.ScheduledTestPlan, error) {
row := r.db.QueryRowContext(ctx, ` row := r.db.QueryRowContext(ctx, `
SELECT id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at SELECT id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
FROM scheduled_test_plans WHERE id = $1 FROM scheduled_test_plans WHERE id = $1
`, id) `, id)
return scanPlan(row) return scanPlan(row)
@@ -37,7 +37,7 @@ func (r *scheduledTestPlanRepository) GetByID(ctx context.Context, id int64) (*s
func (r *scheduledTestPlanRepository) ListByAccountID(ctx context.Context, accountID int64) ([]*service.ScheduledTestPlan, error) { func (r *scheduledTestPlanRepository) ListByAccountID(ctx context.Context, accountID int64) ([]*service.ScheduledTestPlan, error) {
rows, err := r.db.QueryContext(ctx, ` rows, err := r.db.QueryContext(ctx, `
SELECT id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at SELECT id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
FROM scheduled_test_plans WHERE account_id = $1 FROM scheduled_test_plans WHERE account_id = $1
ORDER BY created_at DESC ORDER BY created_at DESC
`, accountID) `, accountID)
@@ -50,7 +50,7 @@ func (r *scheduledTestPlanRepository) ListByAccountID(ctx context.Context, accou
func (r *scheduledTestPlanRepository) ListDue(ctx context.Context, now time.Time) ([]*service.ScheduledTestPlan, error) { func (r *scheduledTestPlanRepository) ListDue(ctx context.Context, now time.Time) ([]*service.ScheduledTestPlan, error) {
rows, err := r.db.QueryContext(ctx, ` rows, err := r.db.QueryContext(ctx, `
SELECT id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at SELECT id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
FROM scheduled_test_plans FROM scheduled_test_plans
WHERE enabled = true AND next_run_at <= $1 WHERE enabled = true AND next_run_at <= $1
ORDER BY next_run_at ASC ORDER BY next_run_at ASC
@@ -65,10 +65,10 @@ func (r *scheduledTestPlanRepository) ListDue(ctx context.Context, now time.Time
func (r *scheduledTestPlanRepository) Update(ctx context.Context, plan *service.ScheduledTestPlan) (*service.ScheduledTestPlan, error) { func (r *scheduledTestPlanRepository) Update(ctx context.Context, plan *service.ScheduledTestPlan) (*service.ScheduledTestPlan, error) {
row := r.db.QueryRowContext(ctx, ` row := r.db.QueryRowContext(ctx, `
UPDATE scheduled_test_plans UPDATE scheduled_test_plans
SET model_id = $2, cron_expression = $3, enabled = $4, max_results = $5, next_run_at = $6, updated_at = NOW() SET model_id = $2, cron_expression = $3, enabled = $4, max_results = $5, auto_recover = $6, next_run_at = $7, updated_at = NOW()
WHERE id = $1 WHERE id = $1
RETURNING id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at RETURNING id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
`, plan.ID, plan.ModelID, plan.CronExpression, plan.Enabled, plan.MaxResults, plan.NextRunAt) `, plan.ID, plan.ModelID, plan.CronExpression, plan.Enabled, plan.MaxResults, plan.AutoRecover, plan.NextRunAt)
return scanPlan(row) return scanPlan(row)
} }
@@ -162,7 +162,7 @@ type scannable interface {
func scanPlan(row scannable) (*service.ScheduledTestPlan, error) { func scanPlan(row scannable) (*service.ScheduledTestPlan, error) {
p := &service.ScheduledTestPlan{} p := &service.ScheduledTestPlan{}
if err := row.Scan( if err := row.Scan(
&p.ID, &p.AccountID, &p.ModelID, &p.CronExpression, &p.Enabled, &p.MaxResults, &p.ID, &p.AccountID, &p.ModelID, &p.CronExpression, &p.Enabled, &p.MaxResults, &p.AutoRecover,
&p.LastRunAt, &p.NextRunAt, &p.CreatedAt, &p.UpdatedAt, &p.LastRunAt, &p.NextRunAt, &p.CreatedAt, &p.UpdatedAt,
); err != nil { ); err != nil {
return nil, err return nil, err

View File

@@ -244,6 +244,7 @@ func registerAccountRoutes(admin *gin.RouterGroup, h *handler.Handlers) {
accounts.PUT("/:id", h.Admin.Account.Update) accounts.PUT("/:id", h.Admin.Account.Update)
accounts.DELETE("/:id", h.Admin.Account.Delete) accounts.DELETE("/:id", h.Admin.Account.Delete)
accounts.POST("/:id/test", h.Admin.Account.Test) accounts.POST("/:id/test", h.Admin.Account.Test)
accounts.POST("/:id/recover-state", h.Admin.Account.RecoverState)
accounts.POST("/:id/refresh", h.Admin.Account.Refresh) accounts.POST("/:id/refresh", h.Admin.Account.Refresh)
accounts.POST("/:id/refresh-tier", h.Admin.Account.RefreshTier) accounts.POST("/:id/refresh-tier", h.Admin.Account.RefreshTier)
accounts.GET("/:id/stats", h.Admin.Account.GetStats) accounts.GET("/:id/stats", h.Admin.Account.GetStats)

View File

@@ -1723,16 +1723,10 @@ func (s *adminServiceImpl) RefreshAccountCredentials(ctx context.Context, id int
} }
func (s *adminServiceImpl) ClearAccountError(ctx context.Context, id int64) (*Account, error) { func (s *adminServiceImpl) ClearAccountError(ctx context.Context, id int64) (*Account, error) {
account, err := s.accountRepo.GetByID(ctx, id) if err := s.accountRepo.ClearError(ctx, id); err != nil {
if err != nil {
return nil, err return nil, err
} }
account.Status = StatusActive return s.accountRepo.GetByID(ctx, id)
account.ErrorMessage = ""
if err := s.accountRepo.Update(ctx, account); err != nil {
return nil, err
}
return account, nil
} }
func (s *adminServiceImpl) SetAccountError(ctx context.Context, id int64, errorMsg string) error { func (s *adminServiceImpl) SetAccountError(ctx context.Context, id int64, errorMsg string) error {

View File

@@ -28,6 +28,17 @@ type RateLimitService struct {
usageCache map[int64]*geminiUsageCacheEntry usageCache map[int64]*geminiUsageCacheEntry
} }
// SuccessfulTestRecoveryResult 表示测试成功后恢复了哪些运行时状态。
type SuccessfulTestRecoveryResult struct {
ClearedError bool
ClearedRateLimit bool
}
// AccountRecoveryOptions 控制账号恢复时的附加行为。
type AccountRecoveryOptions struct {
InvalidateToken bool
}
type geminiUsageCacheEntry struct { type geminiUsageCacheEntry struct {
windowStart time.Time windowStart time.Time
cachedAt time.Time cachedAt time.Time
@@ -1040,6 +1051,42 @@ func (s *RateLimitService) ClearRateLimit(ctx context.Context, accountID int64)
return nil return nil
} }
// RecoverAccountState 按需恢复账号的可恢复运行时状态。
func (s *RateLimitService) RecoverAccountState(ctx context.Context, accountID int64, options AccountRecoveryOptions) (*SuccessfulTestRecoveryResult, error) {
account, err := s.accountRepo.GetByID(ctx, accountID)
if err != nil {
return nil, err
}
result := &SuccessfulTestRecoveryResult{}
if account.Status == StatusError {
if err := s.accountRepo.ClearError(ctx, accountID); err != nil {
return nil, err
}
result.ClearedError = true
if options.InvalidateToken && s.tokenCacheInvalidator != nil && account.IsOAuth() {
if invalidateErr := s.tokenCacheInvalidator.InvalidateToken(ctx, account); invalidateErr != nil {
slog.Warn("recover_account_state_invalidate_token_failed", "account_id", accountID, "error", invalidateErr)
}
}
}
if hasRecoverableRuntimeState(account) {
if err := s.ClearRateLimit(ctx, accountID); err != nil {
return nil, err
}
result.ClearedRateLimit = true
}
return result, nil
}
// RecoverAccountAfterSuccessfulTest 将一次成功测试视为正常请求,
// 按需恢复 error / rate-limit / overload / temp-unsched / model-rate-limit 等运行时状态。
func (s *RateLimitService) RecoverAccountAfterSuccessfulTest(ctx context.Context, accountID int64) (*SuccessfulTestRecoveryResult, error) {
return s.RecoverAccountState(ctx, accountID, AccountRecoveryOptions{})
}
func (s *RateLimitService) ClearTempUnschedulable(ctx context.Context, accountID int64) error { func (s *RateLimitService) ClearTempUnschedulable(ctx context.Context, accountID int64) error {
if err := s.accountRepo.ClearTempUnschedulable(ctx, accountID); err != nil { if err := s.accountRepo.ClearTempUnschedulable(ctx, accountID); err != nil {
return err return err
@@ -1056,6 +1103,36 @@ func (s *RateLimitService) ClearTempUnschedulable(ctx context.Context, accountID
return nil return nil
} }
func hasRecoverableRuntimeState(account *Account) bool {
if account == nil {
return false
}
if account.RateLimitedAt != nil || account.RateLimitResetAt != nil || account.OverloadUntil != nil || account.TempUnschedulableUntil != nil {
return true
}
if len(account.Extra) == 0 {
return false
}
return hasNonEmptyMapValue(account.Extra, "model_rate_limits") || hasNonEmptyMapValue(account.Extra, "antigravity_quota_scopes")
}
func hasNonEmptyMapValue(extra map[string]any, key string) bool {
raw, ok := extra[key]
if !ok || raw == nil {
return false
}
switch typed := raw.(type) {
case map[string]any:
return len(typed) > 0
case map[string]string:
return len(typed) > 0
case []any:
return len(typed) > 0
default:
return true
}
}
func (s *RateLimitService) GetTempUnschedStatus(ctx context.Context, accountID int64) (*TempUnschedState, error) { func (s *RateLimitService) GetTempUnschedStatus(ctx context.Context, accountID int64) (*TempUnschedState, error) {
now := time.Now().Unix() now := time.Now().Unix()
if s.tempUnschedCache != nil { if s.tempUnschedCache != nil {

View File

@@ -6,6 +6,7 @@ import (
"context" "context"
"errors" "errors"
"testing" "testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/config"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -13,16 +14,34 @@ import (
type rateLimitClearRepoStub struct { type rateLimitClearRepoStub struct {
mockAccountRepoForGemini mockAccountRepoForGemini
getByIDAccount *Account
getByIDErr error
getByIDCalls int
clearErrorCalls int
clearRateLimitCalls int clearRateLimitCalls int
clearAntigravityCalls int clearAntigravityCalls int
clearModelRateLimitCalls int clearModelRateLimitCalls int
clearTempUnschedCalls int clearTempUnschedCalls int
clearErrorErr error
clearRateLimitErr error clearRateLimitErr error
clearAntigravityErr error clearAntigravityErr error
clearModelRateLimitErr error clearModelRateLimitErr error
clearTempUnschedulableErr error clearTempUnschedulableErr error
} }
func (r *rateLimitClearRepoStub) GetByID(ctx context.Context, id int64) (*Account, error) {
r.getByIDCalls++
if r.getByIDErr != nil {
return nil, r.getByIDErr
}
return r.getByIDAccount, nil
}
func (r *rateLimitClearRepoStub) ClearError(ctx context.Context, id int64) error {
r.clearErrorCalls++
return r.clearErrorErr
}
func (r *rateLimitClearRepoStub) ClearRateLimit(ctx context.Context, id int64) error { func (r *rateLimitClearRepoStub) ClearRateLimit(ctx context.Context, id int64) error {
r.clearRateLimitCalls++ r.clearRateLimitCalls++
return r.clearRateLimitErr return r.clearRateLimitErr
@@ -48,6 +67,11 @@ type tempUnschedCacheRecorder struct {
deleteErr error deleteErr error
} }
type recoverTokenInvalidatorStub struct {
accounts []*Account
err error
}
func (c *tempUnschedCacheRecorder) SetTempUnsched(ctx context.Context, accountID int64, state *TempUnschedState) error { func (c *tempUnschedCacheRecorder) SetTempUnsched(ctx context.Context, accountID int64, state *TempUnschedState) error {
return nil return nil
} }
@@ -61,6 +85,11 @@ func (c *tempUnschedCacheRecorder) DeleteTempUnsched(ctx context.Context, accoun
return c.deleteErr return c.deleteErr
} }
func (s *recoverTokenInvalidatorStub) InvalidateToken(ctx context.Context, account *Account) error {
s.accounts = append(s.accounts, account)
return s.err
}
func TestRateLimitService_ClearRateLimit_AlsoClearsTempUnschedulable(t *testing.T) { func TestRateLimitService_ClearRateLimit_AlsoClearsTempUnschedulable(t *testing.T) {
repo := &rateLimitClearRepoStub{} repo := &rateLimitClearRepoStub{}
cache := &tempUnschedCacheRecorder{} cache := &tempUnschedCacheRecorder{}
@@ -170,3 +199,108 @@ func TestRateLimitService_ClearRateLimit_WithoutTempUnschedCache(t *testing.T) {
require.Equal(t, 1, repo.clearModelRateLimitCalls) require.Equal(t, 1, repo.clearModelRateLimitCalls)
require.Equal(t, 1, repo.clearTempUnschedCalls) require.Equal(t, 1, repo.clearTempUnschedCalls)
} }
func TestRateLimitService_RecoverAccountAfterSuccessfulTest_ClearsErrorAndRateLimitRelatedState(t *testing.T) {
now := time.Now()
repo := &rateLimitClearRepoStub{
getByIDAccount: &Account{
ID: 42,
Status: StatusError,
RateLimitedAt: &now,
TempUnschedulableUntil: &now,
Extra: map[string]any{
"model_rate_limits": map[string]any{
"claude-sonnet-4-5": map[string]any{
"rate_limit_reset_at": now.Format(time.RFC3339),
},
},
"antigravity_quota_scopes": map[string]any{"gemini": true},
},
},
}
cache := &tempUnschedCacheRecorder{}
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, cache)
result, err := svc.RecoverAccountAfterSuccessfulTest(context.Background(), 42)
require.NoError(t, err)
require.NotNil(t, result)
require.True(t, result.ClearedError)
require.True(t, result.ClearedRateLimit)
require.Equal(t, 1, repo.getByIDCalls)
require.Equal(t, 1, repo.clearErrorCalls)
require.Equal(t, 1, repo.clearRateLimitCalls)
require.Equal(t, 1, repo.clearAntigravityCalls)
require.Equal(t, 1, repo.clearModelRateLimitCalls)
require.Equal(t, 1, repo.clearTempUnschedCalls)
require.Equal(t, []int64{42}, cache.deletedIDs)
}
func TestRateLimitService_RecoverAccountAfterSuccessfulTest_NoRecoverableStateIsNoop(t *testing.T) {
repo := &rateLimitClearRepoStub{
getByIDAccount: &Account{
ID: 7,
Status: StatusActive,
Schedulable: true,
Extra: map[string]any{},
},
}
cache := &tempUnschedCacheRecorder{}
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, cache)
result, err := svc.RecoverAccountAfterSuccessfulTest(context.Background(), 7)
require.NoError(t, err)
require.NotNil(t, result)
require.False(t, result.ClearedError)
require.False(t, result.ClearedRateLimit)
require.Equal(t, 1, repo.getByIDCalls)
require.Equal(t, 0, repo.clearErrorCalls)
require.Equal(t, 0, repo.clearRateLimitCalls)
require.Equal(t, 0, repo.clearAntigravityCalls)
require.Equal(t, 0, repo.clearModelRateLimitCalls)
require.Equal(t, 0, repo.clearTempUnschedCalls)
require.Empty(t, cache.deletedIDs)
}
func TestRateLimitService_RecoverAccountAfterSuccessfulTest_ClearErrorFailed(t *testing.T) {
repo := &rateLimitClearRepoStub{
getByIDAccount: &Account{
ID: 9,
Status: StatusError,
},
clearErrorErr: errors.New("clear error failed"),
}
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
result, err := svc.RecoverAccountAfterSuccessfulTest(context.Background(), 9)
require.Error(t, err)
require.Nil(t, result)
require.Equal(t, 1, repo.getByIDCalls)
require.Equal(t, 1, repo.clearErrorCalls)
require.Equal(t, 0, repo.clearRateLimitCalls)
}
func TestRateLimitService_RecoverAccountState_InvalidatesOAuthTokenOnErrorRecovery(t *testing.T) {
repo := &rateLimitClearRepoStub{
getByIDAccount: &Account{
ID: 21,
Type: AccountTypeOAuth,
Status: StatusError,
},
}
invalidator := &recoverTokenInvalidatorStub{}
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc.SetTokenCacheInvalidator(invalidator)
result, err := svc.RecoverAccountState(context.Background(), 21, AccountRecoveryOptions{
InvalidateToken: true,
})
require.NoError(t, err)
require.NotNil(t, result)
require.True(t, result.ClearedError)
require.False(t, result.ClearedRateLimit)
require.Equal(t, 1, repo.clearErrorCalls)
require.Len(t, invalidator.accounts, 1)
require.Equal(t, int64(21), invalidator.accounts[0].ID)
}

View File

@@ -13,6 +13,7 @@ type ScheduledTestPlan struct {
CronExpression string `json:"cron_expression"` CronExpression string `json:"cron_expression"`
Enabled bool `json:"enabled"` Enabled bool `json:"enabled"`
MaxResults int `json:"max_results"` MaxResults int `json:"max_results"`
AutoRecover bool `json:"auto_recover"`
LastRunAt *time.Time `json:"last_run_at"` LastRunAt *time.Time `json:"last_run_at"`
NextRunAt *time.Time `json:"next_run_at"` NextRunAt *time.Time `json:"next_run_at"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`

View File

@@ -17,6 +17,7 @@ type ScheduledTestRunnerService struct {
planRepo ScheduledTestPlanRepository planRepo ScheduledTestPlanRepository
scheduledSvc *ScheduledTestService scheduledSvc *ScheduledTestService
accountTestSvc *AccountTestService accountTestSvc *AccountTestService
rateLimitSvc *RateLimitService
cfg *config.Config cfg *config.Config
cron *cron.Cron cron *cron.Cron
@@ -29,12 +30,14 @@ func NewScheduledTestRunnerService(
planRepo ScheduledTestPlanRepository, planRepo ScheduledTestPlanRepository,
scheduledSvc *ScheduledTestService, scheduledSvc *ScheduledTestService,
accountTestSvc *AccountTestService, accountTestSvc *AccountTestService,
rateLimitSvc *RateLimitService,
cfg *config.Config, cfg *config.Config,
) *ScheduledTestRunnerService { ) *ScheduledTestRunnerService {
return &ScheduledTestRunnerService{ return &ScheduledTestRunnerService{
planRepo: planRepo, planRepo: planRepo,
scheduledSvc: scheduledSvc, scheduledSvc: scheduledSvc,
accountTestSvc: accountTestSvc, accountTestSvc: accountTestSvc,
rateLimitSvc: rateLimitSvc,
cfg: cfg, cfg: cfg,
} }
} }
@@ -127,6 +130,11 @@ func (s *ScheduledTestRunnerService) runOnePlan(ctx context.Context, plan *Sched
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d SaveResult error: %v", plan.ID, err) logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d SaveResult error: %v", plan.ID, err)
} }
// Auto-recover account if test succeeded and auto_recover is enabled.
if result.Status == "success" && plan.AutoRecover {
s.tryRecoverAccount(ctx, plan.AccountID, plan.ID)
}
nextRun, err := computeNextRun(plan.CronExpression, time.Now()) nextRun, err := computeNextRun(plan.CronExpression, time.Now())
if err != nil { if err != nil {
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d computeNextRun error: %v", plan.ID, err) logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d computeNextRun error: %v", plan.ID, err)
@@ -137,3 +145,26 @@ func (s *ScheduledTestRunnerService) runOnePlan(ctx context.Context, plan *Sched
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d UpdateAfterRun error: %v", plan.ID, err) logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d UpdateAfterRun error: %v", plan.ID, err)
} }
} }
// tryRecoverAccount attempts to recover an account from recoverable runtime state.
func (s *ScheduledTestRunnerService) tryRecoverAccount(ctx context.Context, accountID int64, planID int64) {
if s.rateLimitSvc == nil {
return
}
recovery, err := s.rateLimitSvc.RecoverAccountAfterSuccessfulTest(ctx, accountID)
if err != nil {
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d auto-recover failed: %v", planID, err)
return
}
if recovery == nil {
return
}
if recovery.ClearedError {
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d auto-recover: account=%d recovered from error status", planID, accountID)
}
if recovery.ClearedRateLimit {
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d auto-recover: account=%d cleared rate-limit/runtime state", planID, accountID)
}
}

View File

@@ -287,9 +287,10 @@ func ProvideScheduledTestRunnerService(
planRepo ScheduledTestPlanRepository, planRepo ScheduledTestPlanRepository,
scheduledSvc *ScheduledTestService, scheduledSvc *ScheduledTestService,
accountTestSvc *AccountTestService, accountTestSvc *AccountTestService,
rateLimitSvc *RateLimitService,
cfg *config.Config, cfg *config.Config,
) *ScheduledTestRunnerService { ) *ScheduledTestRunnerService {
svc := NewScheduledTestRunnerService(planRepo, scheduledSvc, accountTestSvc, cfg) svc := NewScheduledTestRunnerService(planRepo, scheduledSvc, accountTestSvc, rateLimitSvc, cfg)
svc.Start() svc.Start()
return svc return svc
} }

View File

@@ -0,0 +1,4 @@
-- 070: Add auto_recover column to scheduled_test_plans
-- When enabled, automatically recovers account from error/rate-limited state on successful test
ALTER TABLE scheduled_test_plans ADD COLUMN IF NOT EXISTS auto_recover BOOLEAN NOT NULL DEFAULT false;

View File

@@ -240,6 +240,16 @@ export async function clearRateLimit(id: number): Promise<Account> {
return data return data
} }
/**
* Recover account runtime state in one call
* @param id - Account ID
* @returns Updated account
*/
export async function recoverState(id: number): Promise<Account> {
const { data } = await apiClient.post<Account>(`/admin/accounts/${id}/recover-state`)
return data
}
/** /**
* Reset account quota usage * Reset account quota usage
* @param id - Account ID * @param id - Account ID
@@ -588,6 +598,7 @@ export const accountsAPI = {
getTodayStats, getTodayStats,
getBatchTodayStats, getBatchTodayStats,
clearRateLimit, clearRateLimit,
recoverState,
resetAccountQuota, resetAccountQuota,
getTempUnschedulableStatus, getTempUnschedulableStatus,
resetTempUnschedulable, resetTempUnschedulable,

View File

@@ -29,6 +29,10 @@
</div> </div>
<div v-else class="space-y-4"> <div v-else class="space-y-4">
<div class="rounded-lg border border-emerald-200 bg-emerald-50 p-3 text-sm text-emerald-800 dark:border-emerald-500/30 dark:bg-emerald-500/10 dark:text-emerald-300">
{{ t('admin.accounts.recoverStateHint') }}
</div>
<div class="rounded-lg border border-gray-200 p-4 dark:border-dark-600"> <div class="rounded-lg border border-gray-200 p-4 dark:border-dark-600">
<p class="text-xs text-gray-500 dark:text-gray-400"> <p class="text-xs text-gray-500 dark:text-gray-400">
{{ t('admin.accounts.tempUnschedulable.accountName') }} {{ t('admin.accounts.tempUnschedulable.accountName') }}
@@ -131,7 +135,7 @@
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
></path> ></path>
</svg> </svg>
{{ t('admin.accounts.tempUnschedulable.reset') }} {{ t('admin.accounts.recoverState') }}
</button> </button>
</div> </div>
</template> </template>
@@ -154,7 +158,7 @@ const props = defineProps<{
const emit = defineEmits<{ const emit = defineEmits<{
close: [] close: []
reset: [] reset: [account: Account]
}>() }>()
const { t } = useI18n() const { t } = useI18n()
@@ -225,12 +229,12 @@ const handleReset = async () => {
if (!props.account) return if (!props.account) return
resetting.value = true resetting.value = true
try { try {
await adminAPI.accounts.resetTempUnschedulable(props.account.id) const updated = await adminAPI.accounts.recoverState(props.account.id)
appStore.showSuccess(t('admin.accounts.tempUnschedulable.resetSuccess')) appStore.showSuccess(t('admin.accounts.recoverStateSuccess'))
emit('reset') emit('reset', updated)
handleClose() handleClose()
} catch (error: any) { } catch (error: any) {
appStore.showError(error?.message || t('admin.accounts.tempUnschedulable.resetFailed')) appStore.showError(error?.message || t('admin.accounts.recoverStateFailed'))
} finally { } finally {
resetting.value = false resetting.value = false
} }

View File

@@ -32,14 +32,10 @@
{{ t('admin.accounts.refreshToken') }} {{ t('admin.accounts.refreshToken') }}
</button> </button>
</template> </template>
<div v-if="account.status === 'error' || isRateLimited || isOverloaded" class="my-1 border-t border-gray-100 dark:border-dark-700"></div> <div v-if="hasRecoverableState" class="my-1 border-t border-gray-100 dark:border-dark-700"></div>
<button v-if="account.status === 'error'" @click="$emit('reset-status', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-yellow-600 hover:bg-gray-100 dark:hover:bg-dark-700"> <button v-if="hasRecoverableState" @click="$emit('recover-state', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-emerald-600 hover:bg-gray-100 dark:hover:bg-dark-700">
<Icon name="sync" size="sm" /> <Icon name="sync" size="sm" />
{{ t('admin.accounts.resetStatus') }} {{ t('admin.accounts.recoverState') }}
</button>
<button v-if="isRateLimited || isOverloaded" @click="$emit('clear-rate-limit', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-amber-600 hover:bg-gray-100 dark:hover:bg-dark-700">
<Icon name="clock" size="sm" />
{{ t('admin.accounts.clearRateLimit') }}
</button> </button>
<button v-if="hasQuotaLimit" @click="$emit('reset-quota', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-teal-600 hover:bg-gray-100 dark:hover:bg-dark-700"> <button v-if="hasQuotaLimit" @click="$emit('reset-quota', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-teal-600 hover:bg-gray-100 dark:hover:bg-dark-700">
<Icon name="refresh" size="sm" /> <Icon name="refresh" size="sm" />
@@ -59,7 +55,7 @@ import { Icon } from '@/components/icons'
import type { Account } from '@/types' import type { Account } from '@/types'
const props = defineProps<{ show: boolean; account: Account | null; position: { top: number; left: number } | null }>() const props = defineProps<{ show: boolean; account: Account | null; position: { top: number; left: number } | null }>()
const emit = defineEmits(['close', 'test', 'stats', 'schedule', 'reauth', 'refresh-token', 'reset-status', 'clear-rate-limit', 'reset-quota']) const emit = defineEmits(['close', 'test', 'stats', 'schedule', 'reauth', 'refresh-token', 'recover-state', 'reset-quota'])
const { t } = useI18n() const { t } = useI18n()
const isRateLimited = computed(() => { const isRateLimited = computed(() => {
if (props.account?.rate_limit_reset_at && new Date(props.account.rate_limit_reset_at) > new Date()) { if (props.account?.rate_limit_reset_at && new Date(props.account.rate_limit_reset_at) > new Date()) {
@@ -75,6 +71,10 @@ const isRateLimited = computed(() => {
return false return false
}) })
const isOverloaded = computed(() => props.account?.overload_until && new Date(props.account.overload_until) > new Date()) const isOverloaded = computed(() => props.account?.overload_until && new Date(props.account.overload_until) > new Date())
const isTempUnschedulable = computed(() => props.account?.temp_unschedulable_until && new Date(props.account.temp_unschedulable_until) > new Date())
const hasRecoverableState = computed(() => {
return props.account?.status === 'error' || Boolean(isRateLimited.value) || Boolean(isOverloaded.value) || Boolean(isTempUnschedulable.value)
})
const hasQuotaLimit = computed(() => { const hasQuotaLimit = computed(() => {
return props.account?.type === 'apikey' && ( return props.account?.type === 'apikey' && (
(props.account?.quota_limit ?? 0) > 0 || (props.account?.quota_limit ?? 0) > 0 ||

View File

@@ -41,8 +41,24 @@
/> />
</div> </div>
<div> <div>
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400"> <label class="mb-1 flex items-center gap-1 text-xs font-medium text-gray-600 dark:text-gray-400">
{{ t('admin.scheduledTests.cronExpression') }} {{ t('admin.scheduledTests.cronExpression') }}
<HelpTooltip>
<template #trigger>
<span class="inline-flex h-4 w-4 cursor-help items-center justify-center rounded-full border border-gray-400/70 text-[10px] font-semibold text-gray-400 transition-colors hover:border-primary-500 hover:text-primary-600 dark:border-gray-500 dark:text-gray-500 dark:hover:border-primary-400 dark:hover:text-primary-400">
?
</span>
</template>
<div class="space-y-1.5">
<p class="font-medium">{{ t('admin.scheduledTests.cronTooltipTitle') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipMeaning') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipExampleEvery30Min') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipExampleHourly') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipExampleDaily') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipExampleWeekly') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipRange') }}</p>
</div>
</HelpTooltip>
</label> </label>
<Input <Input
v-model="newPlan.cron_expression" v-model="newPlan.cron_expression"
@@ -51,8 +67,22 @@
/> />
</div> </div>
<div> <div>
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400"> <label class="mb-1 flex items-center gap-1 text-xs font-medium text-gray-600 dark:text-gray-400">
{{ t('admin.scheduledTests.maxResults') }} {{ t('admin.scheduledTests.maxResults') }}
<HelpTooltip>
<template #trigger>
<span class="inline-flex h-4 w-4 cursor-help items-center justify-center rounded-full border border-gray-400/70 text-[10px] font-semibold text-gray-400 transition-colors hover:border-primary-500 hover:text-primary-600 dark:border-gray-500 dark:text-gray-500 dark:hover:border-primary-400 dark:hover:text-primary-400">
?
</span>
</template>
<div class="space-y-1.5">
<p class="font-medium">{{ t('admin.scheduledTests.maxResultsTooltipTitle') }}</p>
<p>{{ t('admin.scheduledTests.maxResultsTooltipMeaning') }}</p>
<p>{{ t('admin.scheduledTests.maxResultsTooltipBody') }}</p>
<p>{{ t('admin.scheduledTests.maxResultsTooltipExample') }}</p>
<p>{{ t('admin.scheduledTests.maxResultsTooltipRange') }}</p>
</div>
</HelpTooltip>
</label> </label>
<Input <Input
v-model="newPlan.max_results" v-model="newPlan.max_results"
@@ -66,6 +96,17 @@
{{ t('admin.scheduledTests.enabled') }} {{ t('admin.scheduledTests.enabled') }}
</label> </label>
</div> </div>
<div class="flex items-end">
<div>
<label class="flex items-center gap-2 text-sm text-gray-700 dark:text-gray-300">
<Toggle v-model="newPlan.auto_recover" />
{{ t('admin.scheduledTests.autoRecover') }}
</label>
<p class="mt-0.5 text-xs text-gray-400 dark:text-gray-500">
{{ t('admin.scheduledTests.autoRecoverHelp') }}
</p>
</div>
</div>
</div> </div>
<div class="mt-3 flex justify-end gap-2"> <div class="mt-3 flex justify-end gap-2">
<button <button
@@ -135,6 +176,14 @@
{{ plan.enabled ? t('admin.scheduledTests.enabled') : '' }} {{ plan.enabled ? t('admin.scheduledTests.enabled') : '' }}
</span> </span>
</div> </div>
<!-- Auto Recover Badge -->
<span
v-if="plan.auto_recover"
class="inline-flex items-center rounded-full bg-emerald-100 px-2 py-0.5 text-xs font-medium text-emerald-700 dark:bg-emerald-500/20 dark:text-emerald-400"
>
{{ t('admin.scheduledTests.autoRecover') }}
</span>
</div> </div>
<div class="flex items-center gap-3"> <div class="flex items-center gap-3">
@@ -202,8 +251,24 @@
/> />
</div> </div>
<div> <div>
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400"> <label class="mb-1 flex items-center gap-1 text-xs font-medium text-gray-600 dark:text-gray-400">
{{ t('admin.scheduledTests.cronExpression') }} {{ t('admin.scheduledTests.cronExpression') }}
<HelpTooltip>
<template #trigger>
<span class="inline-flex h-4 w-4 cursor-help items-center justify-center rounded-full border border-gray-400/70 text-[10px] font-semibold text-gray-400 transition-colors hover:border-primary-500 hover:text-primary-600 dark:border-gray-500 dark:text-gray-500 dark:hover:border-primary-400 dark:hover:text-primary-400">
?
</span>
</template>
<div class="space-y-1.5">
<p class="font-medium">{{ t('admin.scheduledTests.cronTooltipTitle') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipMeaning') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipExampleEvery30Min') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipExampleHourly') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipExampleDaily') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipExampleWeekly') }}</p>
<p>{{ t('admin.scheduledTests.cronTooltipRange') }}</p>
</div>
</HelpTooltip>
</label> </label>
<Input <Input
v-model="editForm.cron_expression" v-model="editForm.cron_expression"
@@ -212,8 +277,22 @@
/> />
</div> </div>
<div> <div>
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400"> <label class="mb-1 flex items-center gap-1 text-xs font-medium text-gray-600 dark:text-gray-400">
{{ t('admin.scheduledTests.maxResults') }} {{ t('admin.scheduledTests.maxResults') }}
<HelpTooltip>
<template #trigger>
<span class="inline-flex h-4 w-4 cursor-help items-center justify-center rounded-full border border-gray-400/70 text-[10px] font-semibold text-gray-400 transition-colors hover:border-primary-500 hover:text-primary-600 dark:border-gray-500 dark:text-gray-500 dark:hover:border-primary-400 dark:hover:text-primary-400">
?
</span>
</template>
<div class="space-y-1.5">
<p class="font-medium">{{ t('admin.scheduledTests.maxResultsTooltipTitle') }}</p>
<p>{{ t('admin.scheduledTests.maxResultsTooltipMeaning') }}</p>
<p>{{ t('admin.scheduledTests.maxResultsTooltipBody') }}</p>
<p>{{ t('admin.scheduledTests.maxResultsTooltipExample') }}</p>
<p>{{ t('admin.scheduledTests.maxResultsTooltipRange') }}</p>
</div>
</HelpTooltip>
</label> </label>
<Input <Input
v-model="editForm.max_results" v-model="editForm.max_results"
@@ -227,6 +306,17 @@
{{ t('admin.scheduledTests.enabled') }} {{ t('admin.scheduledTests.enabled') }}
</label> </label>
</div> </div>
<div class="flex items-end">
<div>
<label class="flex items-center gap-2 text-sm text-gray-700 dark:text-gray-300">
<Toggle v-model="editForm.auto_recover" />
{{ t('admin.scheduledTests.autoRecover') }}
</label>
<p class="mt-0.5 text-xs text-gray-400 dark:text-gray-500">
{{ t('admin.scheduledTests.autoRecoverHelp') }}
</p>
</div>
</div>
</div> </div>
<div class="mt-3 flex justify-end gap-2"> <div class="mt-3 flex justify-end gap-2">
<button <button
@@ -377,6 +467,7 @@ import { ref, reactive, watch } from 'vue'
import { useI18n } from 'vue-i18n' import { useI18n } from 'vue-i18n'
import BaseDialog from '@/components/common/BaseDialog.vue' import BaseDialog from '@/components/common/BaseDialog.vue'
import ConfirmDialog from '@/components/common/ConfirmDialog.vue' import ConfirmDialog from '@/components/common/ConfirmDialog.vue'
import HelpTooltip from '@/components/common/HelpTooltip.vue'
import Select, { type SelectOption } from '@/components/common/Select.vue' import Select, { type SelectOption } from '@/components/common/Select.vue'
import Input from '@/components/common/Input.vue' import Input from '@/components/common/Input.vue'
import Toggle from '@/components/common/Toggle.vue' import Toggle from '@/components/common/Toggle.vue'
@@ -416,14 +507,16 @@ const editForm = reactive({
model_id: '' as string, model_id: '' as string,
cron_expression: '' as string, cron_expression: '' as string,
max_results: '100' as string, max_results: '100' as string,
enabled: true enabled: true,
auto_recover: false
}) })
const newPlan = reactive({ const newPlan = reactive({
model_id: '' as string, model_id: '' as string,
cron_expression: '' as string, cron_expression: '' as string,
max_results: '100' as string, max_results: '100' as string,
enabled: true enabled: true,
auto_recover: false
}) })
const resetNewPlan = () => { const resetNewPlan = () => {
@@ -431,6 +524,7 @@ const resetNewPlan = () => {
newPlan.cron_expression = '' newPlan.cron_expression = ''
newPlan.max_results = '100' newPlan.max_results = '100'
newPlan.enabled = true newPlan.enabled = true
newPlan.auto_recover = false
} }
// Load plans when dialog opens // Load plans when dialog opens
@@ -472,7 +566,8 @@ const handleCreate = async () => {
model_id: newPlan.model_id, model_id: newPlan.model_id,
cron_expression: newPlan.cron_expression, cron_expression: newPlan.cron_expression,
enabled: newPlan.enabled, enabled: newPlan.enabled,
max_results: maxResults max_results: maxResults,
auto_recover: newPlan.auto_recover
}) })
appStore.showSuccess(t('admin.scheduledTests.createSuccess')) appStore.showSuccess(t('admin.scheduledTests.createSuccess'))
showAddForm.value = false showAddForm.value = false
@@ -504,6 +599,7 @@ const startEdit = (plan: ScheduledTestPlan) => {
editForm.cron_expression = plan.cron_expression editForm.cron_expression = plan.cron_expression
editForm.max_results = String(plan.max_results) editForm.max_results = String(plan.max_results)
editForm.enabled = plan.enabled editForm.enabled = plan.enabled
editForm.auto_recover = plan.auto_recover
} }
const cancelEdit = () => { const cancelEdit = () => {
@@ -518,7 +614,8 @@ const handleEdit = async () => {
model_id: editForm.model_id, model_id: editForm.model_id,
cron_expression: editForm.cron_expression, cron_expression: editForm.cron_expression,
max_results: Number(editForm.max_results) || 100, max_results: Number(editForm.max_results) || 100,
enabled: editForm.enabled enabled: editForm.enabled,
auto_recover: editForm.auto_recover
}) })
const index = plans.value.findIndex((p) => p.id === editingPlanId.value) const index = plans.value.findIndex((p) => p.id === editingPlanId.value)
if (index !== -1) { if (index !== -1) {

View File

@@ -1775,9 +1775,9 @@ export default {
remaining: 'Remaining', remaining: 'Remaining',
matchedKeyword: 'Matched Keyword', matchedKeyword: 'Matched Keyword',
errorMessage: 'Error Details', errorMessage: 'Error Details',
reset: 'Reset Status', reset: 'Recover State',
resetSuccess: 'Temp unschedulable status reset', resetSuccess: 'Account state recovered successfully',
resetFailed: 'Failed to reset temp unschedulable status', resetFailed: 'Failed to recover account state',
failedToLoad: 'Failed to load temp unschedulable status', failedToLoad: 'Failed to load temp unschedulable status',
notActive: 'This account is not temporarily unschedulable.', notActive: 'This account is not temporarily unschedulable.',
expired: 'Expired', expired: 'Expired',
@@ -1849,6 +1849,10 @@ export default {
bulkDeleteSuccess: 'Deleted {count} account(s)', bulkDeleteSuccess: 'Deleted {count} account(s)',
bulkDeletePartial: 'Partially deleted: {success} succeeded, {failed} failed', bulkDeletePartial: 'Partially deleted: {success} succeeded, {failed} failed',
bulkDeleteFailed: 'Bulk delete failed', bulkDeleteFailed: 'Bulk delete failed',
recoverState: 'Recover State',
recoverStateHint: 'Used to recover error, rate-limit, and temporary unschedulable runtime state.',
recoverStateSuccess: 'Account state recovered successfully',
recoverStateFailed: 'Failed to recover account state',
resetStatus: 'Reset Status', resetStatus: 'Reset Status',
statusReset: 'Account status reset successfully', statusReset: 'Account status reset successfully',
failedToResetStatus: 'Failed to reset account status', failedToResetStatus: 'Failed to reset account status',
@@ -2480,7 +2484,21 @@ export default {
failed: 'Failed', failed: 'Failed',
running: 'Running', running: 'Running',
schedule: 'Schedule', schedule: 'Schedule',
cronHelp: 'Standard 5-field cron expression (e.g., */30 * * * *)' cronHelp: 'Standard 5-field cron expression (e.g., */30 * * * *)',
cronTooltipTitle: 'Cron expression examples:',
cronTooltipMeaning: 'Defines when the test runs automatically. The 5 fields are: minute, hour, day, month, and weekday.',
cronTooltipExampleEvery30Min: '*/30 * * * *: run every 30 minutes',
cronTooltipExampleHourly: '0 * * * *: run at the start of every hour',
cronTooltipExampleDaily: '0 9 * * *: run every day at 09:00',
cronTooltipExampleWeekly: '0 9 * * 1: run every Monday at 09:00',
cronTooltipRange: 'Recommended range: use standard 5-field cron. For health checks, start with a moderate frequency such as every 30 minutes, every hour, or once a day instead of running too often.',
maxResultsTooltipTitle: 'What Max Results means:',
maxResultsTooltipMeaning: 'Sets how many historical test results are kept for a single plan so the result list does not grow without limit.',
maxResultsTooltipBody: 'Only the newest test results are kept. Once the number of saved results exceeds this value, older records are pruned automatically so the history list and storage stay under control.',
maxResultsTooltipExample: 'For example, 100 means keeping at most the latest 100 test results. When the 101st result is saved, the oldest one is removed.',
maxResultsTooltipRange: 'Recommended range: usually 20 to 200. Use 20-50 when you only care about recent health status, or 100-200 if you want a longer trend history.',
autoRecover: 'Auto Recover',
autoRecoverHelp: 'Automatically recover account from error/rate-limited state on successful test'
}, },
// Proxies // Proxies

View File

@@ -1892,9 +1892,9 @@ export default {
remaining: '剩余时间', remaining: '剩余时间',
matchedKeyword: '匹配关键词', matchedKeyword: '匹配关键词',
errorMessage: '错误详情', errorMessage: '错误详情',
reset: '重置状态', reset: '恢复状态',
resetSuccess: '临时不可调度已重置', resetSuccess: '账号状态已恢复',
resetFailed: '重置临时不可调度失败', resetFailed: '恢复账号状态失败',
failedToLoad: '加载临时不可调度状态失败', failedToLoad: '加载临时不可调度状态失败',
notActive: '当前账号未处于临时不可调度状态。', notActive: '当前账号未处于临时不可调度状态。',
expired: '已到期', expired: '已到期',
@@ -1995,6 +1995,10 @@ export default {
bulkDeleteSuccess: '成功删除 {count} 个账号', bulkDeleteSuccess: '成功删除 {count} 个账号',
bulkDeletePartial: '部分删除成功:成功 {success} 个,失败 {failed} 个', bulkDeletePartial: '部分删除成功:成功 {success} 个,失败 {failed} 个',
bulkDeleteFailed: '批量删除失败', bulkDeleteFailed: '批量删除失败',
recoverState: '恢复状态',
recoverStateHint: '用于恢复错误、限流和临时不可调度等可恢复状态。',
recoverStateSuccess: '账号状态已恢复',
recoverStateFailed: '恢复账号状态失败',
resetStatus: '重置状态', resetStatus: '重置状态',
statusReset: '账号状态已重置', statusReset: '账号状态已重置',
failedToResetStatus: '重置账号状态失败', failedToResetStatus: '重置账号状态失败',
@@ -2587,7 +2591,21 @@ export default {
failed: '失败', failed: '失败',
running: '运行中', running: '运行中',
schedule: '定时测试', schedule: '定时测试',
cronHelp: '标准 5 字段 cron 表达式(例如 */30 * * * *' cronHelp: '标准 5 字段 cron 表达式(例如 */30 * * * *',
cronTooltipTitle: 'Cron 表达式示例:',
cronTooltipMeaning: '用于定义自动执行测试的时间规则,格式依次为:分钟 小时 日 月 星期。',
cronTooltipExampleEvery30Min: '*/30 * * * *:每 30 分钟运行一次',
cronTooltipExampleHourly: '0 * * * *:每小时整点运行一次',
cronTooltipExampleDaily: '0 9 * * *:每天 09:00 运行一次',
cronTooltipExampleWeekly: '0 9 * * 1每周一 09:00 运行一次',
cronTooltipRange: '推荐填写范围:使用标准 5 字段 cron如果只是健康检查建议从每 30 分钟、每 1 小时或每天固定时间开始,不建议一开始就设置过高频率。',
maxResultsTooltipTitle: '最大结果数说明:',
maxResultsTooltipMeaning: '用于限制单个计划最多保留多少条历史测试结果,避免结果列表无限增长。',
maxResultsTooltipBody: '系统只会保留最近的测试结果;当保存数量超过这个值时,更早的历史记录会自动清理,避免列表过长和存储持续增长。',
maxResultsTooltipExample: '例如填写 100表示最多保存最近 100 次测试结果;第 101 次结果写入后,最早的一条会被清理。',
maxResultsTooltipRange: '推荐填写范围:一般可填 20 到 200。只关注近期可用性时可填 20-50需要回看较长时间的波动趋势时可填 100-200。',
autoRecover: '自动恢复',
autoRecoverHelp: '测试成功后自动恢复异常状态的账号'
}, },
// Proxies Management // Proxies Management

View File

@@ -1491,6 +1491,7 @@ export interface ScheduledTestPlan {
cron_expression: string cron_expression: string
enabled: boolean enabled: boolean
max_results: number max_results: number
auto_recover: boolean
last_run_at: string | null last_run_at: string | null
next_run_at: string | null next_run_at: string | null
created_at: string created_at: string
@@ -1515,6 +1516,7 @@ export interface CreateScheduledTestPlanRequest {
cron_expression: string cron_expression: string
enabled?: boolean enabled?: boolean
max_results?: number max_results?: number
auto_recover?: boolean
} }
export interface UpdateScheduledTestPlanRequest { export interface UpdateScheduledTestPlanRequest {
@@ -1522,4 +1524,5 @@ export interface UpdateScheduledTestPlanRequest {
cron_expression?: string cron_expression?: string
enabled?: boolean enabled?: boolean
max_results?: number max_results?: number
auto_recover?: boolean
} }

View File

@@ -263,7 +263,7 @@
<AccountTestModal :show="showTest" :account="testingAcc" @close="closeTestModal" /> <AccountTestModal :show="showTest" :account="testingAcc" @close="closeTestModal" />
<AccountStatsModal :show="showStats" :account="statsAcc" @close="closeStatsModal" /> <AccountStatsModal :show="showStats" :account="statsAcc" @close="closeStatsModal" />
<ScheduledTestsPanel :show="showSchedulePanel" :account-id="scheduleAcc?.id ?? null" :model-options="scheduleModelOptions" @close="closeSchedulePanel" /> <ScheduledTestsPanel :show="showSchedulePanel" :account-id="scheduleAcc?.id ?? null" :model-options="scheduleModelOptions" @close="closeSchedulePanel" />
<AccountActionMenu :show="menu.show" :account="menu.acc" :position="menu.pos" @close="menu.show = false" @test="handleTest" @stats="handleViewStats" @schedule="handleSchedule" @reauth="handleReAuth" @refresh-token="handleRefresh" @reset-status="handleResetStatus" @clear-rate-limit="handleClearRateLimit" @reset-quota="handleResetQuota" /> <AccountActionMenu :show="menu.show" :account="menu.acc" :position="menu.pos" @close="menu.show = false" @test="handleTest" @stats="handleViewStats" @schedule="handleSchedule" @reauth="handleReAuth" @refresh-token="handleRefresh" @recover-state="handleRecoverState" @reset-quota="handleResetQuota" />
<SyncFromCrsModal :show="showSync" @close="showSync = false" @synced="reload" /> <SyncFromCrsModal :show="showSync" @close="showSync = false" @synced="reload" />
<ImportDataModal :show="showImportData" @close="showImportData = false" @imported="handleDataImported" /> <ImportDataModal :show="showImportData" @close="showImportData = false" @imported="handleDataImported" />
<BulkEditAccountModal :show="showBulkEdit" :account-ids="selIds" :selected-platforms="selPlatforms" :selected-types="selTypes" :proxies="proxies" :groups="groups" @close="showBulkEdit = false" @updated="handleBulkUpdated" /> <BulkEditAccountModal :show="showBulkEdit" :account-ids="selIds" :selected-platforms="selPlatforms" :selected-types="selTypes" :proxies="proxies" :groups="groups" @close="showBulkEdit = false" @updated="handleBulkUpdated" />
@@ -572,16 +572,17 @@ const resetAutoRefreshCache = () => {
const isFirstLoad = ref(true) const isFirstLoad = ref(true)
const load = async () => { const load = async () => {
const requestParams = params as any
hasPendingListSync.value = false hasPendingListSync.value = false
resetAutoRefreshCache() resetAutoRefreshCache()
pendingTodayStatsRefresh.value = false pendingTodayStatsRefresh.value = false
if (isFirstLoad.value) { if (isFirstLoad.value) {
;(params as any).lite = '1' requestParams.lite = '1'
} }
await baseLoad() await baseLoad()
if (isFirstLoad.value) { if (isFirstLoad.value) {
isFirstLoad.value = false isFirstLoad.value = false
delete (params as any).lite delete requestParams.lite
} }
await refreshTodayStatsBatch() await refreshTodayStatsBatch()
} }
@@ -1116,24 +1117,15 @@ const handleRefresh = async (a: Account) => {
console.error('Failed to refresh credentials:', error) console.error('Failed to refresh credentials:', error)
} }
} }
const handleResetStatus = async (a: Account) => { const handleRecoverState = async (a: Account) => {
try { try {
const updated = await adminAPI.accounts.clearError(a.id) const updated = await adminAPI.accounts.recoverState(a.id)
patchAccountInList(updated) patchAccountInList(updated)
enterAutoRefreshSilentWindow() enterAutoRefreshSilentWindow()
appStore.showSuccess(t('common.success')) appStore.showSuccess(t('admin.accounts.recoverStateSuccess'))
} catch (error) { } catch (error: any) {
console.error('Failed to reset status:', error) console.error('Failed to recover account state:', error)
} appStore.showError(error?.message || t('admin.accounts.recoverStateFailed'))
}
const handleClearRateLimit = async (a: Account) => {
try {
const updated = await adminAPI.accounts.clearRateLimit(a.id)
patchAccountInList(updated)
enterAutoRefreshSilentWindow()
appStore.showSuccess(t('common.success'))
} catch (error) {
console.error('Failed to clear rate limit:', error)
} }
} }
const handleResetQuota = async (a: Account) => { const handleResetQuota = async (a: Account) => {
@@ -1163,17 +1155,11 @@ const handleToggleSchedulable = async (a: Account) => {
} }
} }
const handleShowTempUnsched = (a: Account) => { tempUnschedAcc.value = a; showTempUnsched.value = true } const handleShowTempUnsched = (a: Account) => { tempUnschedAcc.value = a; showTempUnsched.value = true }
const handleTempUnschedReset = async () => { const handleTempUnschedReset = async (updated: Account) => {
if(!tempUnschedAcc.value) return showTempUnsched.value = false
try { tempUnschedAcc.value = null
const updated = await adminAPI.accounts.clearError(tempUnschedAcc.value.id) patchAccountInList(updated)
showTempUnsched.value = false enterAutoRefreshSilentWindow()
tempUnschedAcc.value = null
patchAccountInList(updated)
enterAutoRefreshSilentWindow()
} catch (error) {
console.error('Failed to reset temp unscheduled:', error)
}
} }
const formatExpiresAt = (value: number | null) => { const formatExpiresAt = (value: number | null) => {
if (!value) return '-' if (!value) return '-'

View File

@@ -552,9 +552,10 @@ const loadDashboardSnapshot = async (includeStats: boolean) => {
appStore.showError(t('admin.dashboard.failedToLoad')) appStore.showError(t('admin.dashboard.failedToLoad'))
console.error('Error loading dashboard snapshot:', error) console.error('Error loading dashboard snapshot:', error)
} finally { } finally {
if (currentSeq !== chartLoadSeq) return if (currentSeq === chartLoadSeq) {
loading.value = false loading.value = false
chartsLoading.value = false chartsLoading.value = false
}
} }
} }
@@ -575,8 +576,9 @@ const loadUsersTrend = async () => {
console.error('Error loading users trend:', error) console.error('Error loading users trend:', error)
userTrend.value = [] userTrend.value = []
} finally { } finally {
if (currentSeq !== usersTrendLoadSeq) return if (currentSeq === usersTrendLoadSeq) {
userTrendLoading.value = false userTrendLoading.value = false
}
} }
} }