diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c009a..b2ecaeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ## Unreleased +- 管理后台会异步汇总工具健康大数，并避免短时间内重复请求；多人同时查看或自动刷新时不再反复扫描历史记录。 + - 单用户可同时运行的重型任务由 2 个提升到 3 个；任务等待执行容量时，对话会直接说明是当前用户、整机或宿主内存限制，获得槽位后自动继续。 - 管理员现在可在管理后台安全录入、测试和更换模型、媒体、检索及语音服务凭据，并查看来源、脱敏尾号和可用状态；数据库凭据可随时删除并回退原有环境配置。DeepSeek 余额低于 30 元、额度耗尽或认证失败时会主动提醒。 diff --git a/PROGRESS.md b/PROGRESS.md index f8a2429..9bef23d 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,7 +2,7 @@ > 配合 `DESIGN.md`。本文件只记 phase 状态、决策偏差、文件量、下一步。每条 1-2 句:做了啥 + 关键判断;细节查 `git log` / `git diff` / `DESIGN §7.9`。 -最后更新:2026-09-03（Unreleased：管理后台紧凑总览与分组工作区） +最后更新:2026-09-03（Unreleased：工具健康按需刷新与短时缓存） --- @@ -20,6 +20,8 @@ --- ## 已完成关键能力 +- **09-03 / Unreleased / 工具健康监控请求收敛**：Admin 首次进入时异步汇总工具健康大数，之后固定按 90 秒节流刷新并对进行中请求单飞，不再每 10 秒重复扫描近 7 天记录。服务端按查询参数增加 90 秒进程内单飞缓存，多页面或多管理员并发时共享一次扫描，日巡检继续直读不受缓存影响。未新增表、字段、migration 或外部配置，结构化普通工具失败事件因双写会与 messages 历史口径重复而留待独立迁移。 + - **09-03 / Unreleased / 管理后台紧凑总览与分组工作区**：Admin 由纵向长页改为顶部常驻的两级总览和“运行、用量、用户、资源、集成、质量”六个页签，四项运行状态使用主卡、四项运营指标收进紧凑指标栏；点击总览可切换页签并精确定位同口径详情，任务、容量、用户活跃、Token/成本和存储均补齐详情指标。页签 hash 支持刷新与前进后退，桌面端页签吸顶，移动端顶栏操作收进菜单、运营指标与页签可横向滑动；1440px/390px 浏览器渲染、六页签与深链交互检查通过，无 API、schema、migration 或运行方式变化。 - **09-02 / Unreleased / 单用户重型执行容量与排队提示**：共享执行容量保持整机 6、后台 4，单用户上限由 2 提升到 3；前台重型任务首次等待时通过 SSE 告知单用户、整机、内存压力或队列顺序原因，放行后恢复正常执行提示，排队仍不计入命令 timeout 且可取消。无 schema、migration 或 API 变化。 diff --git a/core/toolfail.py b/core/toolfail.py index 51886d1..4140079 100644 --- a/core/toolfail.py +++ b/core/toolfail.py @@ -34,6 +34,8 @@ from __future__ import annotations import json import math import re +import threading +import time from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple @@ -47,6 +49,21 @@ from core.storage.telemetry import ( KIND_TOOL_SALVAGED, ) + +_CACHE_TTL_SECONDS = 90.0 +_failure_cache_lock = threading.Lock() +_wire_cache_lock = threading.Lock() +_failure_cache: dict[tuple[float, int, int], tuple[float, List[Dict[str, Any]]]] = {} +_wire_cache: dict[int, tuple[float, Dict[str, Any]]] = {} + + +def _clear_tool_health_cache() -> None: + """测试/运维进程内失效入口；不触碰数据库。""" + with _failure_cache_lock: + _failure_cache.clear() + with _wire_cache_lock: + _wire_cache.clear() + # 签名归一:同一类错误在不同 task/参数下的差异(路径/数字/uuid/十六进制)抹平, # 让 "figures/a.png doesn't exist" 和 "figures/b.png doesn't exist" 聚成一条。 _RE_UUID = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") @@ -321,6 +338,30 @@ def scan_tool_failures( return out +def scan_tool_failures_cached( + days: float = 7, + min_count: int = 5, + min_tasks: int = 2, +) -> List[Dict[str, Any]]: + """管理页短时缓存；锁覆盖扫描，使同参数并发请求只扫库一次。""" + key = (float(days), int(min_count), int(min_tasks)) + now = time.monotonic() + with _failure_cache_lock: + cached = _failure_cache.get(key) + if cached and cached[0] > now: + return cached[1] + for stale_key, (expires_at, _value) in list(_failure_cache.items()): + if expires_at <= now: + _failure_cache.pop(stale_key, None) + value = scan_tool_failures( + days=days, + min_count=min_count, + min_tasks=min_tasks, + ) + _failure_cache[key] = (time.monotonic() + _CACHE_TTL_SECONDS, value) + return value + + def scan_tool_wire_health(days: int = 7) -> Dict[str, Any]: """聚合 provider 工具参数损坏的抢救/残余比例；纯只读、无派生状态。 @@ -407,6 +448,22 @@ def scan_tool_wire_health(days: int = 7) -> Dict[str, Any]: return {"days": days, "rows": out, "total": totals} +def scan_tool_wire_health_cached(days: int = 7) -> Dict[str, Any]: + """管理页链路健康短时缓存；同窗口并发请求共享一次聚合。""" + key = min(90, max(1, int(days))) + now = time.monotonic() + with _wire_cache_lock: + cached = _wire_cache.get(key) + if cached and cached[0] > now: + return cached[1] + for stale_key, (expires_at, _value) in list(_wire_cache.items()): + if expires_at <= now: + _wire_cache.pop(stale_key, None) + value = scan_tool_wire_health(days=key) + _wire_cache[key] = (time.monotonic() + _CACHE_TTL_SECONDS, value) + return value + + # ── provider 级致命错误即时告警 ── # 命中判据:错误文案含余额/配额/认证类关键词 —— 这类错误不是单任务偶发,而是该 # provider 上所有 run 全挂(如 Zai 余额不足),等日巡检的 5 次/2 task 阈值太慢。 diff --git a/tests/test_static_vendor.py b/tests/test_static_vendor.py index 3315a3f..8544241 100644 --- a/tests/test_static_vendor.py +++ b/tests/test_static_vendor.py @@ -145,6 +145,10 @@ class StaticVendorTests(unittest.TestCase): self.assertIn('data-summary-tab="${escapeHtml(tab)}"', admin_js) self.assertIn("function renderCapacityDrawer()", admin_js) self.assertIn("function openCapacityDrawer()", admin_js) + self.assertIn("const TOOL_HEALTH_REFRESH_MS = 90000", admin_js) + self.assertIn("if (toolFailuresRequest) return toolFailuresRequest", admin_js) + self.assertIn("loadToolFailures();", admin_js) + self.assertNotIn('activeTab === "quality"', admin_js) self.assertIn('id="capacity-drawer" class="capacity-drawer"', html) self.assertIn(".capacity-drawer-panel {", html) diff --git a/tests/test_toolfail_malformed.py b/tests/test_toolfail_malformed.py index 8839160..a22db64 100644 --- a/tests/test_toolfail_malformed.py +++ b/tests/test_toolfail_malformed.py @@ -226,6 +226,34 @@ class TestToolWireHealth(unittest.TestCase): self.assertIsNone(out["total"]["recovery_rate_24h"]) +class TestToolHealthCache(unittest.TestCase): + def setUp(self): + tf._clear_tool_health_cache() + + def tearDown(self): + tf._clear_tool_health_cache() + + def test_failure_scan_is_reused_within_ttl(self): + expected = [{"tool": "shell"}] + with patch.object(tf, "scan_tool_failures", return_value=expected) as scan: + first = tf.scan_tool_failures_cached(days=7, min_count=3, min_tasks=1) + second = tf.scan_tool_failures_cached(days=7, min_count=3, min_tasks=1) + + self.assertIs(first, expected) + self.assertIs(second, expected) + scan.assert_called_once_with(days=7, min_count=3, min_tasks=1) + + def test_wire_scan_normalizes_window_before_caching(self): + expected = {"days": 90, "rows": [], "total": {}} + with patch.object(tf, "scan_tool_wire_health", return_value=expected) as scan: + first = tf.scan_tool_wire_health_cached(days=365) + second = tf.scan_tool_wire_health_cached(days=90) + + self.assertIs(first, expected) + self.assertIs(second, expected) + scan.assert_called_once_with(days=90) + + class TestProviderCriticalAlert(unittest.TestCase): def setUp(self): tf._alerted_at.clear() diff --git a/web/admin.py b/web/admin.py index 7ad7fb1..711fe5d 100644 --- a/web/admin.py +++ b/web/admin.py @@ -533,11 +533,11 @@ def register_admin_routes(app: FastAPI, require_admin) -> None: ): """工具失败聚集(core/toolfail 同款扫描,页面用低阈值看全量; 巡检推送走 5 次/2 task 的高阈值)。admin-only。""" - from core.toolfail import scan_tool_failures + from core.toolfail import scan_tool_failures_cached days = min(90, max(1, days)) return { "days": days, - "clusters": scan_tool_failures( + "clusters": scan_tool_failures_cached( days=days, min_count=max(1, min_count), min_tasks=max(1, min_tasks) ), } @@ -547,8 +547,8 @@ def register_admin_routes(app: FastAPI, require_admin) -> None: days: int = 7, user_id: UUID = Depends(require_admin), ): """工具参数 wire 损坏的抢救率，按模型档+工具聚合；admin-only、纯只读。""" - from core.toolfail import scan_tool_wire_health - return scan_tool_wire_health(days=days) + from core.toolfail import scan_tool_wire_health_cached + return scan_tool_wire_health_cached(days=days) @app.get("/v1/admin/tiers", tags=["admin"]) def admin_tiers(user_id: UUID = Depends(require_admin)): diff --git a/web/static/js/admin.js b/web/static/js/admin.js index bfeda66..2e3ac77 100644 --- a/web/static/js/admin.js +++ b/web/static/js/admin.js @@ -8,6 +8,9 @@ import { dialogConfirm, dialogPrompt, message } from "./dialog.js"; const LS_TOKEN = "zcbot.token"; const REFRESH_MS = 10000; +// 工具健康需要聚合近 7 天日志：进入 Admin 立即异步读取，之后最多 90s 刷新一次； +// 后端另有同周期缓存，多个管理员也不会重复扫库。 +const TOOL_HEALTH_REFRESH_MS = 90000; const PAGE_SIZE = 20; const RANGE_OPTS = [["all", "全部"], ["7d", "近7天"], ["30d", "近30天"]]; const SORT_OPTS = [["cost", "按成本"], ["tokens", "按用量"]]; @@ -65,6 +68,8 @@ let sandboxCapacityData = null; let storageSummaryData = null; let toolFailuresData = null; let toolWireData = null; +let toolFailuresRequest = null; +let toolFailuresLastAttemptAt = 0; let capacityDrawerTrigger = null; // ───── 格式化 ───── @@ -1480,21 +1485,32 @@ async function loadStorageSummary() { } catch (e) { /* 保留最近一次成功的摘要 */ } } -async function loadToolFailures() { +async function loadToolFailures(force = false) { + const now = Date.now(); + if (toolFailuresRequest) return toolFailuresRequest; + if (!force && now - toolFailuresLastAttemptAt < TOOL_HEALTH_REFRESH_MS) return; + toolFailuresLastAttemptAt = now; + toolFailuresRequest = (async () => { + try { + const [failures, wire] = await Promise.allSettled([ + apiGet("/v1/admin/tool-failures?days=7&min_count=3&min_tasks=1"), + apiGet("/v1/admin/tool-wire-health?days=7"), + ]); + if (failures.status !== "fulfilled") throw failures.reason; + toolFailuresData = failures.value; + toolWireData = wire.status === "fulfilled" ? wire.value : null; + renderToolFailures( + failures.value, + toolWireData, + ); + renderOpsSummary(); + } catch (e) { /* 同上 */ } + })(); try { - const [failures, wire] = await Promise.allSettled([ - apiGet("/v1/admin/tool-failures?days=7&min_count=3&min_tasks=1"), - apiGet("/v1/admin/tool-wire-health?days=7"), - ]); - if (failures.status !== "fulfilled") throw failures.reason; - toolFailuresData = failures.value; - toolWireData = wire.status === "fulfilled" ? wire.value : null; - renderToolFailures( - failures.value, - toolWireData, - ); - renderOpsSummary(); - } catch (e) { /* 同上 */ } + await toolFailuresRequest; + } finally { + toolFailuresRequest = null; + } } async function loadExternalDefinitions(force = false) {