mirror of
https://gitee.com/wanwujie/sub2api
synced 2026-04-03 06:52:13 +08:00
Merge pull request #854 from james-6-23/main
feat(admin): 支持定时测试自动恢复并统一账号恢复入口
This commit is contained in:
@@ -229,7 +229,7 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) {
|
||||
tokenRefreshService := service.ProvideTokenRefreshService(accountRepository, soraAccountRepository, oAuthService, openAIOAuthService, geminiOAuthService, antigravityOAuthService, compositeTokenCacheInvalidator, schedulerCache, configConfig, tempUnschedCache)
|
||||
accountExpiryService := service.ProvideAccountExpiryService(accountRepository)
|
||||
subscriptionExpiryService := service.ProvideSubscriptionExpiryService(userSubscriptionRepository)
|
||||
scheduledTestRunnerService := service.ProvideScheduledTestRunnerService(scheduledTestPlanRepository, scheduledTestService, accountTestService, configConfig)
|
||||
scheduledTestRunnerService := service.ProvideScheduledTestRunnerService(scheduledTestPlanRepository, scheduledTestService, accountTestService, rateLimitService, configConfig)
|
||||
v := provideCleanup(client, redisClient, opsMetricsCollector, opsAggregationService, opsAlertEvaluatorService, opsCleanupService, opsScheduledReportService, opsSystemLogSink, soraMediaCleanupService, schedulerSnapshotService, tokenRefreshService, accountExpiryService, subscriptionExpiryService, usageCleanupService, idempotencyCleanupService, pricingService, emailQueueService, billingCacheService, usageRecordWorkerPool, subscriptionService, oAuthService, openAIOAuthService, geminiOAuthService, antigravityOAuthService, openAIGatewayService, scheduledTestRunnerService)
|
||||
application := &Application{
|
||||
Server: httpServer,
|
||||
|
||||
@@ -660,6 +660,42 @@ func (h *AccountHandler) Test(c *gin.Context) {
|
||||
// Error already sent via SSE, just log
|
||||
return
|
||||
}
|
||||
|
||||
if h.rateLimitService != nil {
|
||||
if _, err := h.rateLimitService.RecoverAccountAfterSuccessfulTest(c.Request.Context(), accountID); err != nil {
|
||||
_ = c.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RecoverState handles unified recovery of recoverable account runtime state.
|
||||
// POST /api/v1/admin/accounts/:id/recover-state
|
||||
func (h *AccountHandler) RecoverState(c *gin.Context) {
|
||||
accountID, err := strconv.ParseInt(c.Param("id"), 10, 64)
|
||||
if err != nil {
|
||||
response.BadRequest(c, "Invalid account ID")
|
||||
return
|
||||
}
|
||||
|
||||
if h.rateLimitService == nil {
|
||||
response.Error(c, http.StatusServiceUnavailable, "Rate limit service unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := h.rateLimitService.RecoverAccountState(c.Request.Context(), accountID, service.AccountRecoveryOptions{
|
||||
InvalidateToken: true,
|
||||
}); err != nil {
|
||||
response.ErrorFrom(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
account, err := h.adminService.GetAccount(c.Request.Context(), accountID)
|
||||
if err != nil {
|
||||
response.ErrorFrom(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
response.Success(c, h.buildAccountResponseWithRuntime(c.Request.Context(), account))
|
||||
}
|
||||
|
||||
// SyncFromCRS handles syncing accounts from claude-relay-service (CRS)
|
||||
|
||||
@@ -25,6 +25,7 @@ type createScheduledTestPlanRequest struct {
|
||||
CronExpression string `json:"cron_expression" binding:"required"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
MaxResults int `json:"max_results"`
|
||||
AutoRecover *bool `json:"auto_recover"`
|
||||
}
|
||||
|
||||
type updateScheduledTestPlanRequest struct {
|
||||
@@ -32,6 +33,7 @@ type updateScheduledTestPlanRequest struct {
|
||||
CronExpression string `json:"cron_expression"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
MaxResults int `json:"max_results"`
|
||||
AutoRecover *bool `json:"auto_recover"`
|
||||
}
|
||||
|
||||
// ListByAccount GET /admin/accounts/:id/scheduled-test-plans
|
||||
@@ -68,6 +70,9 @@ func (h *ScheduledTestHandler) Create(c *gin.Context) {
|
||||
if req.Enabled != nil {
|
||||
plan.Enabled = *req.Enabled
|
||||
}
|
||||
if req.AutoRecover != nil {
|
||||
plan.AutoRecover = *req.AutoRecover
|
||||
}
|
||||
|
||||
created, err := h.scheduledTestSvc.CreatePlan(c.Request.Context(), plan)
|
||||
if err != nil {
|
||||
@@ -109,6 +114,9 @@ func (h *ScheduledTestHandler) Update(c *gin.Context) {
|
||||
if req.MaxResults > 0 {
|
||||
existing.MaxResults = req.MaxResults
|
||||
}
|
||||
if req.AutoRecover != nil {
|
||||
existing.AutoRecover = *req.AutoRecover
|
||||
}
|
||||
|
||||
updated, err := h.scheduledTestSvc.UpdatePlan(c.Request.Context(), existing)
|
||||
if err != nil {
|
||||
|
||||
@@ -659,13 +659,10 @@ func (r *accountRepository) ClearError(ctx context.Context, id int64) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 清除临时不可调度状态,重置 401 升级链
|
||||
_, _ = r.sql.ExecContext(ctx, `
|
||||
UPDATE accounts
|
||||
SET temp_unschedulable_until = NULL,
|
||||
temp_unschedulable_reason = NULL
|
||||
WHERE id = $1 AND deleted_at IS NULL
|
||||
`, id)
|
||||
if err := enqueueSchedulerOutbox(ctx, r.sql, service.SchedulerOutboxEventAccountChanged, &id, nil, nil); err != nil {
|
||||
logger.LegacyPrintf("repository.account", "[SchedulerOutbox] enqueue clear error failed: account=%d err=%v", id, err)
|
||||
}
|
||||
r.syncSchedulerAccountSnapshot(ctx, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -558,6 +558,26 @@ func (s *AccountRepoSuite) TestSetError() {
|
||||
s.Require().Equal("something went wrong", got.ErrorMessage)
|
||||
}
|
||||
|
||||
func (s *AccountRepoSuite) TestClearError_SyncSchedulerSnapshotOnRecovery() {
|
||||
account := mustCreateAccount(s.T(), s.client, &service.Account{
|
||||
Name: "acc-clear-err",
|
||||
Status: service.StatusError,
|
||||
ErrorMessage: "temporary error",
|
||||
})
|
||||
cacheRecorder := &schedulerCacheRecorder{}
|
||||
s.repo.schedulerCache = cacheRecorder
|
||||
|
||||
s.Require().NoError(s.repo.ClearError(s.ctx, account.ID))
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, account.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Require().Equal(service.StatusActive, got.Status)
|
||||
s.Require().Empty(got.ErrorMessage)
|
||||
s.Require().Len(cacheRecorder.setAccounts, 1)
|
||||
s.Require().Equal(account.ID, cacheRecorder.setAccounts[0].ID)
|
||||
s.Require().Equal(service.StatusActive, cacheRecorder.setAccounts[0].Status)
|
||||
}
|
||||
|
||||
// --- UpdateSessionWindow ---
|
||||
|
||||
func (s *AccountRepoSuite) TestUpdateSessionWindow() {
|
||||
|
||||
@@ -20,16 +20,16 @@ func NewScheduledTestPlanRepository(db *sql.DB) service.ScheduledTestPlanReposit
|
||||
|
||||
func (r *scheduledTestPlanRepository) Create(ctx context.Context, plan *service.ScheduledTestPlan) (*service.ScheduledTestPlan, error) {
|
||||
row := r.db.QueryRowContext(ctx, `
|
||||
INSERT INTO scheduled_test_plans (account_id, model_id, cron_expression, enabled, max_results, next_run_at, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, NOW(), NOW())
|
||||
RETURNING id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at
|
||||
`, plan.AccountID, plan.ModelID, plan.CronExpression, plan.Enabled, plan.MaxResults, plan.NextRunAt)
|
||||
INSERT INTO scheduled_test_plans (account_id, model_id, cron_expression, enabled, max_results, auto_recover, next_run_at, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW())
|
||||
RETURNING id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
|
||||
`, plan.AccountID, plan.ModelID, plan.CronExpression, plan.Enabled, plan.MaxResults, plan.AutoRecover, plan.NextRunAt)
|
||||
return scanPlan(row)
|
||||
}
|
||||
|
||||
func (r *scheduledTestPlanRepository) GetByID(ctx context.Context, id int64) (*service.ScheduledTestPlan, error) {
|
||||
row := r.db.QueryRowContext(ctx, `
|
||||
SELECT id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at
|
||||
SELECT id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
|
||||
FROM scheduled_test_plans WHERE id = $1
|
||||
`, id)
|
||||
return scanPlan(row)
|
||||
@@ -37,7 +37,7 @@ func (r *scheduledTestPlanRepository) GetByID(ctx context.Context, id int64) (*s
|
||||
|
||||
func (r *scheduledTestPlanRepository) ListByAccountID(ctx context.Context, accountID int64) ([]*service.ScheduledTestPlan, error) {
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at
|
||||
SELECT id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
|
||||
FROM scheduled_test_plans WHERE account_id = $1
|
||||
ORDER BY created_at DESC
|
||||
`, accountID)
|
||||
@@ -50,7 +50,7 @@ func (r *scheduledTestPlanRepository) ListByAccountID(ctx context.Context, accou
|
||||
|
||||
func (r *scheduledTestPlanRepository) ListDue(ctx context.Context, now time.Time) ([]*service.ScheduledTestPlan, error) {
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at
|
||||
SELECT id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
|
||||
FROM scheduled_test_plans
|
||||
WHERE enabled = true AND next_run_at <= $1
|
||||
ORDER BY next_run_at ASC
|
||||
@@ -65,10 +65,10 @@ func (r *scheduledTestPlanRepository) ListDue(ctx context.Context, now time.Time
|
||||
func (r *scheduledTestPlanRepository) Update(ctx context.Context, plan *service.ScheduledTestPlan) (*service.ScheduledTestPlan, error) {
|
||||
row := r.db.QueryRowContext(ctx, `
|
||||
UPDATE scheduled_test_plans
|
||||
SET model_id = $2, cron_expression = $3, enabled = $4, max_results = $5, next_run_at = $6, updated_at = NOW()
|
||||
SET model_id = $2, cron_expression = $3, enabled = $4, max_results = $5, auto_recover = $6, next_run_at = $7, updated_at = NOW()
|
||||
WHERE id = $1
|
||||
RETURNING id, account_id, model_id, cron_expression, enabled, max_results, last_run_at, next_run_at, created_at, updated_at
|
||||
`, plan.ID, plan.ModelID, plan.CronExpression, plan.Enabled, plan.MaxResults, plan.NextRunAt)
|
||||
RETURNING id, account_id, model_id, cron_expression, enabled, max_results, auto_recover, last_run_at, next_run_at, created_at, updated_at
|
||||
`, plan.ID, plan.ModelID, plan.CronExpression, plan.Enabled, plan.MaxResults, plan.AutoRecover, plan.NextRunAt)
|
||||
return scanPlan(row)
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ type scannable interface {
|
||||
func scanPlan(row scannable) (*service.ScheduledTestPlan, error) {
|
||||
p := &service.ScheduledTestPlan{}
|
||||
if err := row.Scan(
|
||||
&p.ID, &p.AccountID, &p.ModelID, &p.CronExpression, &p.Enabled, &p.MaxResults,
|
||||
&p.ID, &p.AccountID, &p.ModelID, &p.CronExpression, &p.Enabled, &p.MaxResults, &p.AutoRecover,
|
||||
&p.LastRunAt, &p.NextRunAt, &p.CreatedAt, &p.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -244,6 +244,7 @@ func registerAccountRoutes(admin *gin.RouterGroup, h *handler.Handlers) {
|
||||
accounts.PUT("/:id", h.Admin.Account.Update)
|
||||
accounts.DELETE("/:id", h.Admin.Account.Delete)
|
||||
accounts.POST("/:id/test", h.Admin.Account.Test)
|
||||
accounts.POST("/:id/recover-state", h.Admin.Account.RecoverState)
|
||||
accounts.POST("/:id/refresh", h.Admin.Account.Refresh)
|
||||
accounts.POST("/:id/refresh-tier", h.Admin.Account.RefreshTier)
|
||||
accounts.GET("/:id/stats", h.Admin.Account.GetStats)
|
||||
|
||||
@@ -1723,16 +1723,10 @@ func (s *adminServiceImpl) RefreshAccountCredentials(ctx context.Context, id int
|
||||
}
|
||||
|
||||
func (s *adminServiceImpl) ClearAccountError(ctx context.Context, id int64) (*Account, error) {
|
||||
account, err := s.accountRepo.GetByID(ctx, id)
|
||||
if err != nil {
|
||||
if err := s.accountRepo.ClearError(ctx, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
account.Status = StatusActive
|
||||
account.ErrorMessage = ""
|
||||
if err := s.accountRepo.Update(ctx, account); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return account, nil
|
||||
return s.accountRepo.GetByID(ctx, id)
|
||||
}
|
||||
|
||||
func (s *adminServiceImpl) SetAccountError(ctx context.Context, id int64, errorMsg string) error {
|
||||
|
||||
@@ -28,6 +28,17 @@ type RateLimitService struct {
|
||||
usageCache map[int64]*geminiUsageCacheEntry
|
||||
}
|
||||
|
||||
// SuccessfulTestRecoveryResult 表示测试成功后恢复了哪些运行时状态。
|
||||
type SuccessfulTestRecoveryResult struct {
|
||||
ClearedError bool
|
||||
ClearedRateLimit bool
|
||||
}
|
||||
|
||||
// AccountRecoveryOptions 控制账号恢复时的附加行为。
|
||||
type AccountRecoveryOptions struct {
|
||||
InvalidateToken bool
|
||||
}
|
||||
|
||||
type geminiUsageCacheEntry struct {
|
||||
windowStart time.Time
|
||||
cachedAt time.Time
|
||||
@@ -1040,6 +1051,42 @@ func (s *RateLimitService) ClearRateLimit(ctx context.Context, accountID int64)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecoverAccountState 按需恢复账号的可恢复运行时状态。
|
||||
func (s *RateLimitService) RecoverAccountState(ctx context.Context, accountID int64, options AccountRecoveryOptions) (*SuccessfulTestRecoveryResult, error) {
|
||||
account, err := s.accountRepo.GetByID(ctx, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := &SuccessfulTestRecoveryResult{}
|
||||
if account.Status == StatusError {
|
||||
if err := s.accountRepo.ClearError(ctx, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.ClearedError = true
|
||||
if options.InvalidateToken && s.tokenCacheInvalidator != nil && account.IsOAuth() {
|
||||
if invalidateErr := s.tokenCacheInvalidator.InvalidateToken(ctx, account); invalidateErr != nil {
|
||||
slog.Warn("recover_account_state_invalidate_token_failed", "account_id", accountID, "error", invalidateErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasRecoverableRuntimeState(account) {
|
||||
if err := s.ClearRateLimit(ctx, accountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.ClearedRateLimit = true
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// RecoverAccountAfterSuccessfulTest 将一次成功测试视为正常请求,
|
||||
// 按需恢复 error / rate-limit / overload / temp-unsched / model-rate-limit 等运行时状态。
|
||||
func (s *RateLimitService) RecoverAccountAfterSuccessfulTest(ctx context.Context, accountID int64) (*SuccessfulTestRecoveryResult, error) {
|
||||
return s.RecoverAccountState(ctx, accountID, AccountRecoveryOptions{})
|
||||
}
|
||||
|
||||
func (s *RateLimitService) ClearTempUnschedulable(ctx context.Context, accountID int64) error {
|
||||
if err := s.accountRepo.ClearTempUnschedulable(ctx, accountID); err != nil {
|
||||
return err
|
||||
@@ -1056,6 +1103,36 @@ func (s *RateLimitService) ClearTempUnschedulable(ctx context.Context, accountID
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasRecoverableRuntimeState(account *Account) bool {
|
||||
if account == nil {
|
||||
return false
|
||||
}
|
||||
if account.RateLimitedAt != nil || account.RateLimitResetAt != nil || account.OverloadUntil != nil || account.TempUnschedulableUntil != nil {
|
||||
return true
|
||||
}
|
||||
if len(account.Extra) == 0 {
|
||||
return false
|
||||
}
|
||||
return hasNonEmptyMapValue(account.Extra, "model_rate_limits") || hasNonEmptyMapValue(account.Extra, "antigravity_quota_scopes")
|
||||
}
|
||||
|
||||
func hasNonEmptyMapValue(extra map[string]any, key string) bool {
|
||||
raw, ok := extra[key]
|
||||
if !ok || raw == nil {
|
||||
return false
|
||||
}
|
||||
switch typed := raw.(type) {
|
||||
case map[string]any:
|
||||
return len(typed) > 0
|
||||
case map[string]string:
|
||||
return len(typed) > 0
|
||||
case []any:
|
||||
return len(typed) > 0
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (s *RateLimitService) GetTempUnschedStatus(ctx context.Context, accountID int64) (*TempUnschedState, error) {
|
||||
now := time.Now().Unix()
|
||||
if s.tempUnschedCache != nil {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -13,16 +14,34 @@ import (
|
||||
|
||||
type rateLimitClearRepoStub struct {
|
||||
mockAccountRepoForGemini
|
||||
getByIDAccount *Account
|
||||
getByIDErr error
|
||||
getByIDCalls int
|
||||
clearErrorCalls int
|
||||
clearRateLimitCalls int
|
||||
clearAntigravityCalls int
|
||||
clearModelRateLimitCalls int
|
||||
clearTempUnschedCalls int
|
||||
clearErrorErr error
|
||||
clearRateLimitErr error
|
||||
clearAntigravityErr error
|
||||
clearModelRateLimitErr error
|
||||
clearTempUnschedulableErr error
|
||||
}
|
||||
|
||||
func (r *rateLimitClearRepoStub) GetByID(ctx context.Context, id int64) (*Account, error) {
|
||||
r.getByIDCalls++
|
||||
if r.getByIDErr != nil {
|
||||
return nil, r.getByIDErr
|
||||
}
|
||||
return r.getByIDAccount, nil
|
||||
}
|
||||
|
||||
func (r *rateLimitClearRepoStub) ClearError(ctx context.Context, id int64) error {
|
||||
r.clearErrorCalls++
|
||||
return r.clearErrorErr
|
||||
}
|
||||
|
||||
func (r *rateLimitClearRepoStub) ClearRateLimit(ctx context.Context, id int64) error {
|
||||
r.clearRateLimitCalls++
|
||||
return r.clearRateLimitErr
|
||||
@@ -48,6 +67,11 @@ type tempUnschedCacheRecorder struct {
|
||||
deleteErr error
|
||||
}
|
||||
|
||||
type recoverTokenInvalidatorStub struct {
|
||||
accounts []*Account
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *tempUnschedCacheRecorder) SetTempUnsched(ctx context.Context, accountID int64, state *TempUnschedState) error {
|
||||
return nil
|
||||
}
|
||||
@@ -61,6 +85,11 @@ func (c *tempUnschedCacheRecorder) DeleteTempUnsched(ctx context.Context, accoun
|
||||
return c.deleteErr
|
||||
}
|
||||
|
||||
func (s *recoverTokenInvalidatorStub) InvalidateToken(ctx context.Context, account *Account) error {
|
||||
s.accounts = append(s.accounts, account)
|
||||
return s.err
|
||||
}
|
||||
|
||||
func TestRateLimitService_ClearRateLimit_AlsoClearsTempUnschedulable(t *testing.T) {
|
||||
repo := &rateLimitClearRepoStub{}
|
||||
cache := &tempUnschedCacheRecorder{}
|
||||
@@ -170,3 +199,108 @@ func TestRateLimitService_ClearRateLimit_WithoutTempUnschedCache(t *testing.T) {
|
||||
require.Equal(t, 1, repo.clearModelRateLimitCalls)
|
||||
require.Equal(t, 1, repo.clearTempUnschedCalls)
|
||||
}
|
||||
|
||||
func TestRateLimitService_RecoverAccountAfterSuccessfulTest_ClearsErrorAndRateLimitRelatedState(t *testing.T) {
|
||||
now := time.Now()
|
||||
repo := &rateLimitClearRepoStub{
|
||||
getByIDAccount: &Account{
|
||||
ID: 42,
|
||||
Status: StatusError,
|
||||
RateLimitedAt: &now,
|
||||
TempUnschedulableUntil: &now,
|
||||
Extra: map[string]any{
|
||||
"model_rate_limits": map[string]any{
|
||||
"claude-sonnet-4-5": map[string]any{
|
||||
"rate_limit_reset_at": now.Format(time.RFC3339),
|
||||
},
|
||||
},
|
||||
"antigravity_quota_scopes": map[string]any{"gemini": true},
|
||||
},
|
||||
},
|
||||
}
|
||||
cache := &tempUnschedCacheRecorder{}
|
||||
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, cache)
|
||||
|
||||
result, err := svc.RecoverAccountAfterSuccessfulTest(context.Background(), 42)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.True(t, result.ClearedError)
|
||||
require.True(t, result.ClearedRateLimit)
|
||||
|
||||
require.Equal(t, 1, repo.getByIDCalls)
|
||||
require.Equal(t, 1, repo.clearErrorCalls)
|
||||
require.Equal(t, 1, repo.clearRateLimitCalls)
|
||||
require.Equal(t, 1, repo.clearAntigravityCalls)
|
||||
require.Equal(t, 1, repo.clearModelRateLimitCalls)
|
||||
require.Equal(t, 1, repo.clearTempUnschedCalls)
|
||||
require.Equal(t, []int64{42}, cache.deletedIDs)
|
||||
}
|
||||
|
||||
func TestRateLimitService_RecoverAccountAfterSuccessfulTest_NoRecoverableStateIsNoop(t *testing.T) {
|
||||
repo := &rateLimitClearRepoStub{
|
||||
getByIDAccount: &Account{
|
||||
ID: 7,
|
||||
Status: StatusActive,
|
||||
Schedulable: true,
|
||||
Extra: map[string]any{},
|
||||
},
|
||||
}
|
||||
cache := &tempUnschedCacheRecorder{}
|
||||
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, cache)
|
||||
|
||||
result, err := svc.RecoverAccountAfterSuccessfulTest(context.Background(), 7)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.False(t, result.ClearedError)
|
||||
require.False(t, result.ClearedRateLimit)
|
||||
|
||||
require.Equal(t, 1, repo.getByIDCalls)
|
||||
require.Equal(t, 0, repo.clearErrorCalls)
|
||||
require.Equal(t, 0, repo.clearRateLimitCalls)
|
||||
require.Equal(t, 0, repo.clearAntigravityCalls)
|
||||
require.Equal(t, 0, repo.clearModelRateLimitCalls)
|
||||
require.Equal(t, 0, repo.clearTempUnschedCalls)
|
||||
require.Empty(t, cache.deletedIDs)
|
||||
}
|
||||
|
||||
func TestRateLimitService_RecoverAccountAfterSuccessfulTest_ClearErrorFailed(t *testing.T) {
|
||||
repo := &rateLimitClearRepoStub{
|
||||
getByIDAccount: &Account{
|
||||
ID: 9,
|
||||
Status: StatusError,
|
||||
},
|
||||
clearErrorErr: errors.New("clear error failed"),
|
||||
}
|
||||
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
|
||||
result, err := svc.RecoverAccountAfterSuccessfulTest(context.Background(), 9)
|
||||
require.Error(t, err)
|
||||
require.Nil(t, result)
|
||||
require.Equal(t, 1, repo.getByIDCalls)
|
||||
require.Equal(t, 1, repo.clearErrorCalls)
|
||||
require.Equal(t, 0, repo.clearRateLimitCalls)
|
||||
}
|
||||
|
||||
func TestRateLimitService_RecoverAccountState_InvalidatesOAuthTokenOnErrorRecovery(t *testing.T) {
|
||||
repo := &rateLimitClearRepoStub{
|
||||
getByIDAccount: &Account{
|
||||
ID: 21,
|
||||
Type: AccountTypeOAuth,
|
||||
Status: StatusError,
|
||||
},
|
||||
}
|
||||
invalidator := &recoverTokenInvalidatorStub{}
|
||||
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
svc.SetTokenCacheInvalidator(invalidator)
|
||||
|
||||
result, err := svc.RecoverAccountState(context.Background(), 21, AccountRecoveryOptions{
|
||||
InvalidateToken: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.True(t, result.ClearedError)
|
||||
require.False(t, result.ClearedRateLimit)
|
||||
require.Equal(t, 1, repo.clearErrorCalls)
|
||||
require.Len(t, invalidator.accounts, 1)
|
||||
require.Equal(t, int64(21), invalidator.accounts[0].ID)
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ type ScheduledTestPlan struct {
|
||||
CronExpression string `json:"cron_expression"`
|
||||
Enabled bool `json:"enabled"`
|
||||
MaxResults int `json:"max_results"`
|
||||
AutoRecover bool `json:"auto_recover"`
|
||||
LastRunAt *time.Time `json:"last_run_at"`
|
||||
NextRunAt *time.Time `json:"next_run_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
|
||||
@@ -17,6 +17,7 @@ type ScheduledTestRunnerService struct {
|
||||
planRepo ScheduledTestPlanRepository
|
||||
scheduledSvc *ScheduledTestService
|
||||
accountTestSvc *AccountTestService
|
||||
rateLimitSvc *RateLimitService
|
||||
cfg *config.Config
|
||||
|
||||
cron *cron.Cron
|
||||
@@ -29,12 +30,14 @@ func NewScheduledTestRunnerService(
|
||||
planRepo ScheduledTestPlanRepository,
|
||||
scheduledSvc *ScheduledTestService,
|
||||
accountTestSvc *AccountTestService,
|
||||
rateLimitSvc *RateLimitService,
|
||||
cfg *config.Config,
|
||||
) *ScheduledTestRunnerService {
|
||||
return &ScheduledTestRunnerService{
|
||||
planRepo: planRepo,
|
||||
scheduledSvc: scheduledSvc,
|
||||
accountTestSvc: accountTestSvc,
|
||||
rateLimitSvc: rateLimitSvc,
|
||||
cfg: cfg,
|
||||
}
|
||||
}
|
||||
@@ -127,6 +130,11 @@ func (s *ScheduledTestRunnerService) runOnePlan(ctx context.Context, plan *Sched
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d SaveResult error: %v", plan.ID, err)
|
||||
}
|
||||
|
||||
// Auto-recover account if test succeeded and auto_recover is enabled.
|
||||
if result.Status == "success" && plan.AutoRecover {
|
||||
s.tryRecoverAccount(ctx, plan.AccountID, plan.ID)
|
||||
}
|
||||
|
||||
nextRun, err := computeNextRun(plan.CronExpression, time.Now())
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d computeNextRun error: %v", plan.ID, err)
|
||||
@@ -137,3 +145,26 @@ func (s *ScheduledTestRunnerService) runOnePlan(ctx context.Context, plan *Sched
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d UpdateAfterRun error: %v", plan.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// tryRecoverAccount attempts to recover an account from recoverable runtime state.
|
||||
func (s *ScheduledTestRunnerService) tryRecoverAccount(ctx context.Context, accountID int64, planID int64) {
|
||||
if s.rateLimitSvc == nil {
|
||||
return
|
||||
}
|
||||
|
||||
recovery, err := s.rateLimitSvc.RecoverAccountAfterSuccessfulTest(ctx, accountID)
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d auto-recover failed: %v", planID, err)
|
||||
return
|
||||
}
|
||||
if recovery == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if recovery.ClearedError {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d auto-recover: account=%d recovered from error status", planID, accountID)
|
||||
}
|
||||
if recovery.ClearedRateLimit {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d auto-recover: account=%d cleared rate-limit/runtime state", planID, accountID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,9 +287,10 @@ func ProvideScheduledTestRunnerService(
|
||||
planRepo ScheduledTestPlanRepository,
|
||||
scheduledSvc *ScheduledTestService,
|
||||
accountTestSvc *AccountTestService,
|
||||
rateLimitSvc *RateLimitService,
|
||||
cfg *config.Config,
|
||||
) *ScheduledTestRunnerService {
|
||||
svc := NewScheduledTestRunnerService(planRepo, scheduledSvc, accountTestSvc, cfg)
|
||||
svc := NewScheduledTestRunnerService(planRepo, scheduledSvc, accountTestSvc, rateLimitSvc, cfg)
|
||||
svc.Start()
|
||||
return svc
|
||||
}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- 070: Add auto_recover column to scheduled_test_plans
|
||||
-- When enabled, automatically recovers account from error/rate-limited state on successful test
|
||||
|
||||
ALTER TABLE scheduled_test_plans ADD COLUMN IF NOT EXISTS auto_recover BOOLEAN NOT NULL DEFAULT false;
|
||||
@@ -240,6 +240,16 @@ export async function clearRateLimit(id: number): Promise<Account> {
|
||||
return data
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover account runtime state in one call
|
||||
* @param id - Account ID
|
||||
* @returns Updated account
|
||||
*/
|
||||
export async function recoverState(id: number): Promise<Account> {
|
||||
const { data } = await apiClient.post<Account>(`/admin/accounts/${id}/recover-state`)
|
||||
return data
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset account quota usage
|
||||
* @param id - Account ID
|
||||
@@ -588,6 +598,7 @@ export const accountsAPI = {
|
||||
getTodayStats,
|
||||
getBatchTodayStats,
|
||||
clearRateLimit,
|
||||
recoverState,
|
||||
resetAccountQuota,
|
||||
getTempUnschedulableStatus,
|
||||
resetTempUnschedulable,
|
||||
|
||||
@@ -29,6 +29,10 @@
|
||||
</div>
|
||||
|
||||
<div v-else class="space-y-4">
|
||||
<div class="rounded-lg border border-emerald-200 bg-emerald-50 p-3 text-sm text-emerald-800 dark:border-emerald-500/30 dark:bg-emerald-500/10 dark:text-emerald-300">
|
||||
{{ t('admin.accounts.recoverStateHint') }}
|
||||
</div>
|
||||
|
||||
<div class="rounded-lg border border-gray-200 p-4 dark:border-dark-600">
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400">
|
||||
{{ t('admin.accounts.tempUnschedulable.accountName') }}
|
||||
@@ -131,7 +135,7 @@
|
||||
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
|
||||
></path>
|
||||
</svg>
|
||||
{{ t('admin.accounts.tempUnschedulable.reset') }}
|
||||
{{ t('admin.accounts.recoverState') }}
|
||||
</button>
|
||||
</div>
|
||||
</template>
|
||||
@@ -154,7 +158,7 @@ const props = defineProps<{
|
||||
|
||||
const emit = defineEmits<{
|
||||
close: []
|
||||
reset: []
|
||||
reset: [account: Account]
|
||||
}>()
|
||||
|
||||
const { t } = useI18n()
|
||||
@@ -225,12 +229,12 @@ const handleReset = async () => {
|
||||
if (!props.account) return
|
||||
resetting.value = true
|
||||
try {
|
||||
await adminAPI.accounts.resetTempUnschedulable(props.account.id)
|
||||
appStore.showSuccess(t('admin.accounts.tempUnschedulable.resetSuccess'))
|
||||
emit('reset')
|
||||
const updated = await adminAPI.accounts.recoverState(props.account.id)
|
||||
appStore.showSuccess(t('admin.accounts.recoverStateSuccess'))
|
||||
emit('reset', updated)
|
||||
handleClose()
|
||||
} catch (error: any) {
|
||||
appStore.showError(error?.message || t('admin.accounts.tempUnschedulable.resetFailed'))
|
||||
appStore.showError(error?.message || t('admin.accounts.recoverStateFailed'))
|
||||
} finally {
|
||||
resetting.value = false
|
||||
}
|
||||
|
||||
@@ -32,14 +32,10 @@
|
||||
{{ t('admin.accounts.refreshToken') }}
|
||||
</button>
|
||||
</template>
|
||||
<div v-if="account.status === 'error' || isRateLimited || isOverloaded" class="my-1 border-t border-gray-100 dark:border-dark-700"></div>
|
||||
<button v-if="account.status === 'error'" @click="$emit('reset-status', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-yellow-600 hover:bg-gray-100 dark:hover:bg-dark-700">
|
||||
<div v-if="hasRecoverableState" class="my-1 border-t border-gray-100 dark:border-dark-700"></div>
|
||||
<button v-if="hasRecoverableState" @click="$emit('recover-state', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-emerald-600 hover:bg-gray-100 dark:hover:bg-dark-700">
|
||||
<Icon name="sync" size="sm" />
|
||||
{{ t('admin.accounts.resetStatus') }}
|
||||
</button>
|
||||
<button v-if="isRateLimited || isOverloaded" @click="$emit('clear-rate-limit', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-amber-600 hover:bg-gray-100 dark:hover:bg-dark-700">
|
||||
<Icon name="clock" size="sm" />
|
||||
{{ t('admin.accounts.clearRateLimit') }}
|
||||
{{ t('admin.accounts.recoverState') }}
|
||||
</button>
|
||||
<button v-if="hasQuotaLimit" @click="$emit('reset-quota', account); $emit('close')" class="flex w-full items-center gap-2 px-4 py-2 text-sm text-teal-600 hover:bg-gray-100 dark:hover:bg-dark-700">
|
||||
<Icon name="refresh" size="sm" />
|
||||
@@ -59,7 +55,7 @@ import { Icon } from '@/components/icons'
|
||||
import type { Account } from '@/types'
|
||||
|
||||
const props = defineProps<{ show: boolean; account: Account | null; position: { top: number; left: number } | null }>()
|
||||
const emit = defineEmits(['close', 'test', 'stats', 'schedule', 'reauth', 'refresh-token', 'reset-status', 'clear-rate-limit', 'reset-quota'])
|
||||
const emit = defineEmits(['close', 'test', 'stats', 'schedule', 'reauth', 'refresh-token', 'recover-state', 'reset-quota'])
|
||||
const { t } = useI18n()
|
||||
const isRateLimited = computed(() => {
|
||||
if (props.account?.rate_limit_reset_at && new Date(props.account.rate_limit_reset_at) > new Date()) {
|
||||
@@ -75,6 +71,10 @@ const isRateLimited = computed(() => {
|
||||
return false
|
||||
})
|
||||
const isOverloaded = computed(() => props.account?.overload_until && new Date(props.account.overload_until) > new Date())
|
||||
const isTempUnschedulable = computed(() => props.account?.temp_unschedulable_until && new Date(props.account.temp_unschedulable_until) > new Date())
|
||||
const hasRecoverableState = computed(() => {
|
||||
return props.account?.status === 'error' || Boolean(isRateLimited.value) || Boolean(isOverloaded.value) || Boolean(isTempUnschedulable.value)
|
||||
})
|
||||
const hasQuotaLimit = computed(() => {
|
||||
return props.account?.type === 'apikey' && (
|
||||
(props.account?.quota_limit ?? 0) > 0 ||
|
||||
|
||||
@@ -41,8 +41,24 @@
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
<label class="mb-1 flex items-center gap-1 text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
{{ t('admin.scheduledTests.cronExpression') }}
|
||||
<HelpTooltip>
|
||||
<template #trigger>
|
||||
<span class="inline-flex h-4 w-4 cursor-help items-center justify-center rounded-full border border-gray-400/70 text-[10px] font-semibold text-gray-400 transition-colors hover:border-primary-500 hover:text-primary-600 dark:border-gray-500 dark:text-gray-500 dark:hover:border-primary-400 dark:hover:text-primary-400">
|
||||
?
|
||||
</span>
|
||||
</template>
|
||||
<div class="space-y-1.5">
|
||||
<p class="font-medium">{{ t('admin.scheduledTests.cronTooltipTitle') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipMeaning') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipExampleEvery30Min') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipExampleHourly') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipExampleDaily') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipExampleWeekly') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipRange') }}</p>
|
||||
</div>
|
||||
</HelpTooltip>
|
||||
</label>
|
||||
<Input
|
||||
v-model="newPlan.cron_expression"
|
||||
@@ -51,8 +67,22 @@
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
<label class="mb-1 flex items-center gap-1 text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
{{ t('admin.scheduledTests.maxResults') }}
|
||||
<HelpTooltip>
|
||||
<template #trigger>
|
||||
<span class="inline-flex h-4 w-4 cursor-help items-center justify-center rounded-full border border-gray-400/70 text-[10px] font-semibold text-gray-400 transition-colors hover:border-primary-500 hover:text-primary-600 dark:border-gray-500 dark:text-gray-500 dark:hover:border-primary-400 dark:hover:text-primary-400">
|
||||
?
|
||||
</span>
|
||||
</template>
|
||||
<div class="space-y-1.5">
|
||||
<p class="font-medium">{{ t('admin.scheduledTests.maxResultsTooltipTitle') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.maxResultsTooltipMeaning') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.maxResultsTooltipBody') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.maxResultsTooltipExample') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.maxResultsTooltipRange') }}</p>
|
||||
</div>
|
||||
</HelpTooltip>
|
||||
</label>
|
||||
<Input
|
||||
v-model="newPlan.max_results"
|
||||
@@ -66,6 +96,17 @@
|
||||
{{ t('admin.scheduledTests.enabled') }}
|
||||
</label>
|
||||
</div>
|
||||
<div class="flex items-end">
|
||||
<div>
|
||||
<label class="flex items-center gap-2 text-sm text-gray-700 dark:text-gray-300">
|
||||
<Toggle v-model="newPlan.auto_recover" />
|
||||
{{ t('admin.scheduledTests.autoRecover') }}
|
||||
</label>
|
||||
<p class="mt-0.5 text-xs text-gray-400 dark:text-gray-500">
|
||||
{{ t('admin.scheduledTests.autoRecoverHelp') }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-3 flex justify-end gap-2">
|
||||
<button
|
||||
@@ -135,6 +176,14 @@
|
||||
{{ plan.enabled ? t('admin.scheduledTests.enabled') : '' }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<!-- Auto Recover Badge -->
|
||||
<span
|
||||
v-if="plan.auto_recover"
|
||||
class="inline-flex items-center rounded-full bg-emerald-100 px-2 py-0.5 text-xs font-medium text-emerald-700 dark:bg-emerald-500/20 dark:text-emerald-400"
|
||||
>
|
||||
{{ t('admin.scheduledTests.autoRecover') }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="flex items-center gap-3">
|
||||
@@ -202,8 +251,24 @@
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
<label class="mb-1 flex items-center gap-1 text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
{{ t('admin.scheduledTests.cronExpression') }}
|
||||
<HelpTooltip>
|
||||
<template #trigger>
|
||||
<span class="inline-flex h-4 w-4 cursor-help items-center justify-center rounded-full border border-gray-400/70 text-[10px] font-semibold text-gray-400 transition-colors hover:border-primary-500 hover:text-primary-600 dark:border-gray-500 dark:text-gray-500 dark:hover:border-primary-400 dark:hover:text-primary-400">
|
||||
?
|
||||
</span>
|
||||
</template>
|
||||
<div class="space-y-1.5">
|
||||
<p class="font-medium">{{ t('admin.scheduledTests.cronTooltipTitle') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipMeaning') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipExampleEvery30Min') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipExampleHourly') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipExampleDaily') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipExampleWeekly') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.cronTooltipRange') }}</p>
|
||||
</div>
|
||||
</HelpTooltip>
|
||||
</label>
|
||||
<Input
|
||||
v-model="editForm.cron_expression"
|
||||
@@ -212,8 +277,22 @@
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
<label class="mb-1 flex items-center gap-1 text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
{{ t('admin.scheduledTests.maxResults') }}
|
||||
<HelpTooltip>
|
||||
<template #trigger>
|
||||
<span class="inline-flex h-4 w-4 cursor-help items-center justify-center rounded-full border border-gray-400/70 text-[10px] font-semibold text-gray-400 transition-colors hover:border-primary-500 hover:text-primary-600 dark:border-gray-500 dark:text-gray-500 dark:hover:border-primary-400 dark:hover:text-primary-400">
|
||||
?
|
||||
</span>
|
||||
</template>
|
||||
<div class="space-y-1.5">
|
||||
<p class="font-medium">{{ t('admin.scheduledTests.maxResultsTooltipTitle') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.maxResultsTooltipMeaning') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.maxResultsTooltipBody') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.maxResultsTooltipExample') }}</p>
|
||||
<p>{{ t('admin.scheduledTests.maxResultsTooltipRange') }}</p>
|
||||
</div>
|
||||
</HelpTooltip>
|
||||
</label>
|
||||
<Input
|
||||
v-model="editForm.max_results"
|
||||
@@ -227,6 +306,17 @@
|
||||
{{ t('admin.scheduledTests.enabled') }}
|
||||
</label>
|
||||
</div>
|
||||
<div class="flex items-end">
|
||||
<div>
|
||||
<label class="flex items-center gap-2 text-sm text-gray-700 dark:text-gray-300">
|
||||
<Toggle v-model="editForm.auto_recover" />
|
||||
{{ t('admin.scheduledTests.autoRecover') }}
|
||||
</label>
|
||||
<p class="mt-0.5 text-xs text-gray-400 dark:text-gray-500">
|
||||
{{ t('admin.scheduledTests.autoRecoverHelp') }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-3 flex justify-end gap-2">
|
||||
<button
|
||||
@@ -377,6 +467,7 @@ import { ref, reactive, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import BaseDialog from '@/components/common/BaseDialog.vue'
|
||||
import ConfirmDialog from '@/components/common/ConfirmDialog.vue'
|
||||
import HelpTooltip from '@/components/common/HelpTooltip.vue'
|
||||
import Select, { type SelectOption } from '@/components/common/Select.vue'
|
||||
import Input from '@/components/common/Input.vue'
|
||||
import Toggle from '@/components/common/Toggle.vue'
|
||||
@@ -416,14 +507,16 @@ const editForm = reactive({
|
||||
model_id: '' as string,
|
||||
cron_expression: '' as string,
|
||||
max_results: '100' as string,
|
||||
enabled: true
|
||||
enabled: true,
|
||||
auto_recover: false
|
||||
})
|
||||
|
||||
const newPlan = reactive({
|
||||
model_id: '' as string,
|
||||
cron_expression: '' as string,
|
||||
max_results: '100' as string,
|
||||
enabled: true
|
||||
enabled: true,
|
||||
auto_recover: false
|
||||
})
|
||||
|
||||
const resetNewPlan = () => {
|
||||
@@ -431,6 +524,7 @@ const resetNewPlan = () => {
|
||||
newPlan.cron_expression = ''
|
||||
newPlan.max_results = '100'
|
||||
newPlan.enabled = true
|
||||
newPlan.auto_recover = false
|
||||
}
|
||||
|
||||
// Load plans when dialog opens
|
||||
@@ -472,7 +566,8 @@ const handleCreate = async () => {
|
||||
model_id: newPlan.model_id,
|
||||
cron_expression: newPlan.cron_expression,
|
||||
enabled: newPlan.enabled,
|
||||
max_results: maxResults
|
||||
max_results: maxResults,
|
||||
auto_recover: newPlan.auto_recover
|
||||
})
|
||||
appStore.showSuccess(t('admin.scheduledTests.createSuccess'))
|
||||
showAddForm.value = false
|
||||
@@ -504,6 +599,7 @@ const startEdit = (plan: ScheduledTestPlan) => {
|
||||
editForm.cron_expression = plan.cron_expression
|
||||
editForm.max_results = String(plan.max_results)
|
||||
editForm.enabled = plan.enabled
|
||||
editForm.auto_recover = plan.auto_recover
|
||||
}
|
||||
|
||||
const cancelEdit = () => {
|
||||
@@ -518,7 +614,8 @@ const handleEdit = async () => {
|
||||
model_id: editForm.model_id,
|
||||
cron_expression: editForm.cron_expression,
|
||||
max_results: Number(editForm.max_results) || 100,
|
||||
enabled: editForm.enabled
|
||||
enabled: editForm.enabled,
|
||||
auto_recover: editForm.auto_recover
|
||||
})
|
||||
const index = plans.value.findIndex((p) => p.id === editingPlanId.value)
|
||||
if (index !== -1) {
|
||||
|
||||
@@ -1775,9 +1775,9 @@ export default {
|
||||
remaining: 'Remaining',
|
||||
matchedKeyword: 'Matched Keyword',
|
||||
errorMessage: 'Error Details',
|
||||
reset: 'Reset Status',
|
||||
resetSuccess: 'Temp unschedulable status reset',
|
||||
resetFailed: 'Failed to reset temp unschedulable status',
|
||||
reset: 'Recover State',
|
||||
resetSuccess: 'Account state recovered successfully',
|
||||
resetFailed: 'Failed to recover account state',
|
||||
failedToLoad: 'Failed to load temp unschedulable status',
|
||||
notActive: 'This account is not temporarily unschedulable.',
|
||||
expired: 'Expired',
|
||||
@@ -1849,6 +1849,10 @@ export default {
|
||||
bulkDeleteSuccess: 'Deleted {count} account(s)',
|
||||
bulkDeletePartial: 'Partially deleted: {success} succeeded, {failed} failed',
|
||||
bulkDeleteFailed: 'Bulk delete failed',
|
||||
recoverState: 'Recover State',
|
||||
recoverStateHint: 'Used to recover error, rate-limit, and temporary unschedulable runtime state.',
|
||||
recoverStateSuccess: 'Account state recovered successfully',
|
||||
recoverStateFailed: 'Failed to recover account state',
|
||||
resetStatus: 'Reset Status',
|
||||
statusReset: 'Account status reset successfully',
|
||||
failedToResetStatus: 'Failed to reset account status',
|
||||
@@ -2480,7 +2484,21 @@ export default {
|
||||
failed: 'Failed',
|
||||
running: 'Running',
|
||||
schedule: 'Schedule',
|
||||
cronHelp: 'Standard 5-field cron expression (e.g., */30 * * * *)'
|
||||
cronHelp: 'Standard 5-field cron expression (e.g., */30 * * * *)',
|
||||
cronTooltipTitle: 'Cron expression examples:',
|
||||
cronTooltipMeaning: 'Defines when the test runs automatically. The 5 fields are: minute, hour, day, month, and weekday.',
|
||||
cronTooltipExampleEvery30Min: '*/30 * * * *: run every 30 minutes',
|
||||
cronTooltipExampleHourly: '0 * * * *: run at the start of every hour',
|
||||
cronTooltipExampleDaily: '0 9 * * *: run every day at 09:00',
|
||||
cronTooltipExampleWeekly: '0 9 * * 1: run every Monday at 09:00',
|
||||
cronTooltipRange: 'Recommended range: use standard 5-field cron. For health checks, start with a moderate frequency such as every 30 minutes, every hour, or once a day instead of running too often.',
|
||||
maxResultsTooltipTitle: 'What Max Results means:',
|
||||
maxResultsTooltipMeaning: 'Sets how many historical test results are kept for a single plan so the result list does not grow without limit.',
|
||||
maxResultsTooltipBody: 'Only the newest test results are kept. Once the number of saved results exceeds this value, older records are pruned automatically so the history list and storage stay under control.',
|
||||
maxResultsTooltipExample: 'For example, 100 means keeping at most the latest 100 test results. When the 101st result is saved, the oldest one is removed.',
|
||||
maxResultsTooltipRange: 'Recommended range: usually 20 to 200. Use 20-50 when you only care about recent health status, or 100-200 if you want a longer trend history.',
|
||||
autoRecover: 'Auto Recover',
|
||||
autoRecoverHelp: 'Automatically recover account from error/rate-limited state on successful test'
|
||||
},
|
||||
|
||||
// Proxies
|
||||
|
||||
@@ -1892,9 +1892,9 @@ export default {
|
||||
remaining: '剩余时间',
|
||||
matchedKeyword: '匹配关键词',
|
||||
errorMessage: '错误详情',
|
||||
reset: '重置状态',
|
||||
resetSuccess: '临时不可调度已重置',
|
||||
resetFailed: '重置临时不可调度失败',
|
||||
reset: '恢复状态',
|
||||
resetSuccess: '账号状态已恢复',
|
||||
resetFailed: '恢复账号状态失败',
|
||||
failedToLoad: '加载临时不可调度状态失败',
|
||||
notActive: '当前账号未处于临时不可调度状态。',
|
||||
expired: '已到期',
|
||||
@@ -1995,6 +1995,10 @@ export default {
|
||||
bulkDeleteSuccess: '成功删除 {count} 个账号',
|
||||
bulkDeletePartial: '部分删除成功:成功 {success} 个,失败 {failed} 个',
|
||||
bulkDeleteFailed: '批量删除失败',
|
||||
recoverState: '恢复状态',
|
||||
recoverStateHint: '用于恢复错误、限流和临时不可调度等可恢复状态。',
|
||||
recoverStateSuccess: '账号状态已恢复',
|
||||
recoverStateFailed: '恢复账号状态失败',
|
||||
resetStatus: '重置状态',
|
||||
statusReset: '账号状态已重置',
|
||||
failedToResetStatus: '重置账号状态失败',
|
||||
@@ -2587,7 +2591,21 @@ export default {
|
||||
failed: '失败',
|
||||
running: '运行中',
|
||||
schedule: '定时测试',
|
||||
cronHelp: '标准 5 字段 cron 表达式(例如 */30 * * * *)'
|
||||
cronHelp: '标准 5 字段 cron 表达式(例如 */30 * * * *)',
|
||||
cronTooltipTitle: 'Cron 表达式示例:',
|
||||
cronTooltipMeaning: '用于定义自动执行测试的时间规则,格式依次为:分钟 小时 日 月 星期。',
|
||||
cronTooltipExampleEvery30Min: '*/30 * * * *:每 30 分钟运行一次',
|
||||
cronTooltipExampleHourly: '0 * * * *:每小时整点运行一次',
|
||||
cronTooltipExampleDaily: '0 9 * * *:每天 09:00 运行一次',
|
||||
cronTooltipExampleWeekly: '0 9 * * 1:每周一 09:00 运行一次',
|
||||
cronTooltipRange: '推荐填写范围:使用标准 5 字段 cron;如果只是健康检查,建议从每 30 分钟、每 1 小时或每天固定时间开始,不建议一开始就设置过高频率。',
|
||||
maxResultsTooltipTitle: '最大结果数说明:',
|
||||
maxResultsTooltipMeaning: '用于限制单个计划最多保留多少条历史测试结果,避免结果列表无限增长。',
|
||||
maxResultsTooltipBody: '系统只会保留最近的测试结果;当保存数量超过这个值时,更早的历史记录会自动清理,避免列表过长和存储持续增长。',
|
||||
maxResultsTooltipExample: '例如填写 100,表示最多保存最近 100 次测试结果;第 101 次结果写入后,最早的一条会被清理。',
|
||||
maxResultsTooltipRange: '推荐填写范围:一般可填 20 到 200。只关注近期可用性时可填 20-50;需要回看较长时间的波动趋势时可填 100-200。',
|
||||
autoRecover: '自动恢复',
|
||||
autoRecoverHelp: '测试成功后自动恢复异常状态的账号'
|
||||
},
|
||||
|
||||
// Proxies Management
|
||||
|
||||
@@ -1491,6 +1491,7 @@ export interface ScheduledTestPlan {
|
||||
cron_expression: string
|
||||
enabled: boolean
|
||||
max_results: number
|
||||
auto_recover: boolean
|
||||
last_run_at: string | null
|
||||
next_run_at: string | null
|
||||
created_at: string
|
||||
@@ -1515,6 +1516,7 @@ export interface CreateScheduledTestPlanRequest {
|
||||
cron_expression: string
|
||||
enabled?: boolean
|
||||
max_results?: number
|
||||
auto_recover?: boolean
|
||||
}
|
||||
|
||||
export interface UpdateScheduledTestPlanRequest {
|
||||
@@ -1522,4 +1524,5 @@ export interface UpdateScheduledTestPlanRequest {
|
||||
cron_expression?: string
|
||||
enabled?: boolean
|
||||
max_results?: number
|
||||
auto_recover?: boolean
|
||||
}
|
||||
|
||||
@@ -263,7 +263,7 @@
|
||||
<AccountTestModal :show="showTest" :account="testingAcc" @close="closeTestModal" />
|
||||
<AccountStatsModal :show="showStats" :account="statsAcc" @close="closeStatsModal" />
|
||||
<ScheduledTestsPanel :show="showSchedulePanel" :account-id="scheduleAcc?.id ?? null" :model-options="scheduleModelOptions" @close="closeSchedulePanel" />
|
||||
<AccountActionMenu :show="menu.show" :account="menu.acc" :position="menu.pos" @close="menu.show = false" @test="handleTest" @stats="handleViewStats" @schedule="handleSchedule" @reauth="handleReAuth" @refresh-token="handleRefresh" @reset-status="handleResetStatus" @clear-rate-limit="handleClearRateLimit" @reset-quota="handleResetQuota" />
|
||||
<AccountActionMenu :show="menu.show" :account="menu.acc" :position="menu.pos" @close="menu.show = false" @test="handleTest" @stats="handleViewStats" @schedule="handleSchedule" @reauth="handleReAuth" @refresh-token="handleRefresh" @recover-state="handleRecoverState" @reset-quota="handleResetQuota" />
|
||||
<SyncFromCrsModal :show="showSync" @close="showSync = false" @synced="reload" />
|
||||
<ImportDataModal :show="showImportData" @close="showImportData = false" @imported="handleDataImported" />
|
||||
<BulkEditAccountModal :show="showBulkEdit" :account-ids="selIds" :selected-platforms="selPlatforms" :selected-types="selTypes" :proxies="proxies" :groups="groups" @close="showBulkEdit = false" @updated="handleBulkUpdated" />
|
||||
@@ -572,16 +572,17 @@ const resetAutoRefreshCache = () => {
|
||||
const isFirstLoad = ref(true)
|
||||
|
||||
const load = async () => {
|
||||
const requestParams = params as any
|
||||
hasPendingListSync.value = false
|
||||
resetAutoRefreshCache()
|
||||
pendingTodayStatsRefresh.value = false
|
||||
if (isFirstLoad.value) {
|
||||
;(params as any).lite = '1'
|
||||
requestParams.lite = '1'
|
||||
}
|
||||
await baseLoad()
|
||||
if (isFirstLoad.value) {
|
||||
isFirstLoad.value = false
|
||||
delete (params as any).lite
|
||||
delete requestParams.lite
|
||||
}
|
||||
await refreshTodayStatsBatch()
|
||||
}
|
||||
@@ -1116,24 +1117,15 @@ const handleRefresh = async (a: Account) => {
|
||||
console.error('Failed to refresh credentials:', error)
|
||||
}
|
||||
}
|
||||
const handleResetStatus = async (a: Account) => {
|
||||
const handleRecoverState = async (a: Account) => {
|
||||
try {
|
||||
const updated = await adminAPI.accounts.clearError(a.id)
|
||||
const updated = await adminAPI.accounts.recoverState(a.id)
|
||||
patchAccountInList(updated)
|
||||
enterAutoRefreshSilentWindow()
|
||||
appStore.showSuccess(t('common.success'))
|
||||
} catch (error) {
|
||||
console.error('Failed to reset status:', error)
|
||||
}
|
||||
}
|
||||
const handleClearRateLimit = async (a: Account) => {
|
||||
try {
|
||||
const updated = await adminAPI.accounts.clearRateLimit(a.id)
|
||||
patchAccountInList(updated)
|
||||
enterAutoRefreshSilentWindow()
|
||||
appStore.showSuccess(t('common.success'))
|
||||
} catch (error) {
|
||||
console.error('Failed to clear rate limit:', error)
|
||||
appStore.showSuccess(t('admin.accounts.recoverStateSuccess'))
|
||||
} catch (error: any) {
|
||||
console.error('Failed to recover account state:', error)
|
||||
appStore.showError(error?.message || t('admin.accounts.recoverStateFailed'))
|
||||
}
|
||||
}
|
||||
const handleResetQuota = async (a: Account) => {
|
||||
@@ -1163,17 +1155,11 @@ const handleToggleSchedulable = async (a: Account) => {
|
||||
}
|
||||
}
|
||||
const handleShowTempUnsched = (a: Account) => { tempUnschedAcc.value = a; showTempUnsched.value = true }
|
||||
const handleTempUnschedReset = async () => {
|
||||
if(!tempUnschedAcc.value) return
|
||||
try {
|
||||
const updated = await adminAPI.accounts.clearError(tempUnschedAcc.value.id)
|
||||
showTempUnsched.value = false
|
||||
tempUnschedAcc.value = null
|
||||
patchAccountInList(updated)
|
||||
enterAutoRefreshSilentWindow()
|
||||
} catch (error) {
|
||||
console.error('Failed to reset temp unscheduled:', error)
|
||||
}
|
||||
const handleTempUnschedReset = async (updated: Account) => {
|
||||
showTempUnsched.value = false
|
||||
tempUnschedAcc.value = null
|
||||
patchAccountInList(updated)
|
||||
enterAutoRefreshSilentWindow()
|
||||
}
|
||||
const formatExpiresAt = (value: number | null) => {
|
||||
if (!value) return '-'
|
||||
|
||||
@@ -552,9 +552,10 @@ const loadDashboardSnapshot = async (includeStats: boolean) => {
|
||||
appStore.showError(t('admin.dashboard.failedToLoad'))
|
||||
console.error('Error loading dashboard snapshot:', error)
|
||||
} finally {
|
||||
if (currentSeq !== chartLoadSeq) return
|
||||
loading.value = false
|
||||
chartsLoading.value = false
|
||||
if (currentSeq === chartLoadSeq) {
|
||||
loading.value = false
|
||||
chartsLoading.value = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -575,8 +576,9 @@ const loadUsersTrend = async () => {
|
||||
console.error('Error loading users trend:', error)
|
||||
userTrend.value = []
|
||||
} finally {
|
||||
if (currentSeq !== usersTrendLoadSeq) return
|
||||
userTrendLoading.value = false
|
||||
if (currentSeq === usersTrendLoadSeq) {
|
||||
userTrendLoading.value = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user