"""工具失败聚集巡检:同签名的工具报错反复出现 → 主动冒头。 背景(2026-07,task 9dcae061 终案的结构性教训):mermaid 渲染在生产挂了 90 天 0 成功(67 次超时 + 26 次 launch fail、烧掉数十万 token),没有任何机制发现, 靠人工扫 DB 才挖出来。本模块把「失败聚集」变成信号:统一扫描 usage_events 中的结构化异常事件,按 (工具名 + 归一化错误签名) 聚合,超阈值即算聚集。 普通工具失败从 2026-09-03 起在 tool result 落 messages 后同步写 tool_failure； 切换前只存在于 messages 的历史故障不再回扫。纯只读查询、无派生状态；告警通道 由调用方决定(web/app.py 的巡检 loop 发开发者邮箱,admin API 直接返给前端表格)。 第二数据源(0.58.19):被丢弃的畸形 tool_call 参数(kind=malformed)——这类失败 整轮不入 messages(防投毒级联),loop 落 usage_events(kind=tool_malformed), 在此并入同一聚合口径(signature=归一化 JSON 报错,sample=损坏参数首尾片段)。 第三数据源(0.58.21):run 级终态错误(kind=run)—— LLM 请求层/构建期直接抛异常 (RateLimitError 余额不足、认证失败等),整轮无 tool 消息,_run_agent_bg 落 usage_events(kind=run_error),在此按归一化错误签名聚合(tool 名固定 "(run)")。 其中 provider 级致命错误(余额/认证)另走 `alert_provider_critical` 即时邮件, 不等日巡检 —— 这类错误会让该 provider 上所有用户的所有 run 全挂。 第四数据源(0.58.32):provider 吐空(kind=empty_response)—— assistant 轮既无 tool_calls 又无正文,会被 run loop 当正常收尾静默 done(task 2a1bc25d 案:网关把 tool_use 漏成正文后丢空)。loop 落 usage_events(kind=empty_response),在此按固定签名 聚合(tool 名固定 "(empty)",sample=model_profile,看哪个网关档在吐空)。 失败判定(tool content 的三类标记,形态见 executor_docker/_host): - `[Error` 开头 —— 执行器/工具层报错([Error]、[Error executing ...]) - `command timed out` —— shell/run_python 超时 - 尾部 `[exit N]` 且 N != 0 —— shell 非零退出 `[exit 0]` 但语义失败(如 "No mermaid charts found")不判 —— 无通用判据,不猜。 """ from __future__ import annotations import math import re import threading import time from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple from sqlalchemy import text from core.storage import session_scope from core.storage.telemetry import ( KIND_AGENT_GUARD, KIND_CONTEXT_FOLD_FAILURE, KIND_EMPTY_RESPONSE, KIND_QUALITY_GATE, KIND_RUN_ERROR, KIND_RUN_STOPPED, KIND_TOOL_FAILURE, KIND_TOOL_MALFORMED, KIND_TOOL_SALVAGED, ) from core.tool_failure import ( failure_category as _failure_category, normalize_failure_signature as _normalize, ) _CACHE_TTL_SECONDS = 90.0 _failure_cache_lock = threading.Lock() _wire_cache_lock = threading.Lock() _failure_cache: dict[tuple[float, int, int], tuple[float, List[Dict[str, Any]]]] = {} _wire_cache: dict[int, tuple[float, Dict[str, Any]]] = {} def _clear_tool_health_cache() -> None: """测试/运维进程内失效入口；不触碰数据库。""" with _failure_cache_lock: _failure_cache.clear() with _wire_cache_lock: _wire_cache.clear() def scan_tool_failures( days: float = 7, min_count: int = 5, min_tasks: int = 2, ) -> List[Dict[str, Any]]: """扫近 `days` 天的结构化异常事件,返回超阈值的聚集。 阈值语义:同签名 >= min_count 次 且 跨 >= min_tasks 个 task —— 单 task 内 模型试错几次就自愈的正常噪音不触发;跨 task 复现的才是平台性问题。 时间分布:每个聚集带 `daily`(从 now 往回按 24h 分桶的次数,旧→新, 非日历日)和 `count_24h`(= daily 尾桶)—— 修复部署后看尾桶是否归零, 区分「还在发生」和「窗口内的存量记录」。排序:近 24h 活跃的在前 (count_24h 降序),其后按 count 降序。 同步阻塞(DB 查询),asyncio 调用方放 to_thread/executor。 """ now = datetime.now(timezone.utc) cutoff = now - timedelta(days=days) n_buckets = max(1, math.ceil(days)) with session_scope() as s: event_rows = s.execute( text( "select kind, task_id, user_id, created_at, model_profile, " " units->>'tool' as tool_name, units->>'err' as err, " " units->>'head' as head, units->>'tail' as tail, " " units->>'failure_kind' as failure_kind, " " units->>'signature' as signature, " " units->>'category' as category, " " units->>'sample' as sample " "from usage_events " f"where kind in ('{KIND_TOOL_FAILURE}', '{KIND_TOOL_MALFORMED}', " f"'{KIND_RUN_ERROR}', '{KIND_EMPTY_RESPONSE}', '{KIND_RUN_STOPPED}', " f"'{KIND_AGENT_GUARD}', '{KIND_CONTEXT_FOLD_FAILURE}', " f"'{KIND_QUALITY_GATE}') " " and created_at >= :cutoff" ), {"cutoff": cutoff}, ).fetchall() agg: Dict[Tuple[str, str], Dict[str, Any]] = {} def _add( tool_name: str, kind: str, sig_line: str, sample: str, task_id: Any, user_id: Any, created_at: datetime, *, category: Optional[str] = None, signature_normalized: bool = False, ) -> None: # DB 列若是 naive timestamp(存 UTC),补 tzinfo 才能和 now 做减法 ts = created_at if created_at.tzinfo else created_at.replace(tzinfo=timezone.utc) key = ( tool_name or "?", sig_line if signature_normalized else _normalize(sig_line), ) resolved_category = category or _failure_category(kind, sig_line, sample) c = agg.get(key) if c is None: c = agg[key] = { "tool": key[0], "signature": key[1], "kind": kind, # 分类必须看未截断的完整结果；sample 只保留 300 字给前端悬浮。 "category": resolved_category, "count": 0, "tasks": set(), "users": set(), "first_at": ts, "last_at": ts, "sample": sample[:300], "daily": [0] * n_buckets, } elif resolved_category == "quality_gate": c["category"] = "quality_gate" c["count"] += 1 c["tasks"].add(task_id) c["users"].add(user_id) # 分桶:距 now 每满 24h 退一桶,尾桶 = 近 24h(时钟漂移/边界值 clamp 进首尾桶) age_days = int((now - ts).total_seconds() // 86400) c["daily"][n_buckets - 1 - min(n_buckets - 1, max(0, age_days))] += 1 if ts < c["first_at"]: c["first_at"] = ts if ts > c["last_at"]: c["last_at"] = ts c["sample"] = sample[:300] for ( event_kind, task_id, user_id, created_at, model_profile, tool_name, err, head, tail, failure_kind, signature, category, sample, ) in event_rows: if event_kind in { KIND_TOOL_FAILURE, KIND_RUN_STOPPED, KIND_AGENT_GUARD, KIND_CONTEXT_FOLD_FAILURE, KIND_QUALITY_GATE, }: _add( tool_name, failure_kind or "failure", signature or "?", sample or "", task_id, user_id, created_at, category=category or "failure", signature_normalized=True, ) elif event_kind == KIND_TOOL_MALFORMED: _add( tool_name, "malformed", err or "?", f"{head or ''} … {tail or ''}", task_id, user_id, created_at, ) elif event_kind == KIND_RUN_ERROR: _add("(run)", "run", err or "?", err or "", task_id, user_id, created_at) elif event_kind == KIND_EMPTY_RESPONSE: _add( "(empty)", "empty", "provider returned empty response", model_profile or "?", task_id, user_id, created_at, ) out = [] for c in agg.values(): if c["count"] < min_count or len(c["tasks"]) < min_tasks: continue out.append({ "tool": c["tool"], "signature": c["signature"], "kind": c["kind"], "category": c["category"], "count": c["count"], "count_24h": c["daily"][-1], "daily": c["daily"], "task_count": len(c["tasks"]), "user_count": len(c["users"]), "first_at": c["first_at"].isoformat(), "last_at": c["last_at"].isoformat(), "sample": c["sample"], }) # 活跃的(近 24h 还在发生)排前面,已安静的沉底 —— 面板/邮件都先看还在烧的 out.sort(key=lambda x: (x["count_24h"], x["count"]), reverse=True) return out def scan_tool_failures_cached( days: float = 7, min_count: int = 5, min_tasks: int = 2, ) -> List[Dict[str, Any]]: """管理页短时缓存；锁覆盖扫描，使同参数并发请求只扫库一次。""" key = (float(days), int(min_count), int(min_tasks)) now = time.monotonic() with _failure_cache_lock: cached = _failure_cache.get(key) if cached and cached[0] > now: return cached[1] for stale_key, (expires_at, _value) in list(_failure_cache.items()): if expires_at <= now: _failure_cache.pop(stale_key, None) value = scan_tool_failures( days=days, min_count=min_count, min_tasks=min_tasks, ) _failure_cache[key] = (time.monotonic() + _CACHE_TTL_SECONDS, value) return value def scan_tool_wire_health(days: int = 7) -> Dict[str, Any]: """聚合 provider 工具参数损坏的抢救/残余比例；纯只读、无派生状态。 ``tool_salvaged`` 与 ``tool_malformed`` 是同一类 wire 损坏的两个结局，二者之和 才是观测分母。按 model_profile + tool 展示近 24h 与窗口总量，避免只看 malformed 绝对数时把调用量上涨误判为 provider 恶化。 """ days = min(90, max(1, int(days))) now = datetime.now(timezone.utc) cutoff = now - timedelta(days=days) cutoff_24h = now - timedelta(hours=24) with session_scope() as s: rows = s.execute( text( "select model_profile, coalesce(units->>'tool', '?') as tool, " f" count(*) filter (where kind = '{KIND_TOOL_SALVAGED}') as salvaged, " f" count(*) filter (where kind = '{KIND_TOOL_MALFORMED}') as malformed, " f" count(*) filter (where kind = '{KIND_TOOL_SALVAGED}' " " and created_at >= :cutoff_24h) as salvaged_24h, " f" count(*) filter (where kind = '{KIND_TOOL_MALFORMED}' " " and created_at >= :cutoff_24h) as malformed_24h, " " max(created_at) as last_at " "from usage_events " f"where kind in ('{KIND_TOOL_SALVAGED}', '{KIND_TOOL_MALFORMED}') " " and created_at >= :cutoff " "group by model_profile, coalesce(units->>'tool', '?')" ), {"cutoff": cutoff, "cutoff_24h": cutoff_24h}, ).fetchall() out: list[dict[str, Any]] = [] counts: dict[str, int] = { "salvaged": 0, "malformed": 0, "salvaged_24h": 0, "malformed_24h": 0, } def _rate(saved: int, residual: int) -> Optional[float]: total = saved + residual return round(saved / total * 100, 1) if total else None for ( model_profile, tool, salvaged, malformed, salvaged_24h, malformed_24h, last_at, ) in rows: saved = int(salvaged or 0) residual = int(malformed or 0) saved_24h = int(salvaged_24h or 0) residual_24h = int(malformed_24h or 0) for key, value in ( ("salvaged", saved), ("malformed", residual), ("salvaged_24h", saved_24h), ("malformed_24h", residual_24h), ): counts[key] += value out.append({ "model_profile": model_profile or "?", "tool": tool or "?", "salvaged": saved, "malformed": residual, "recovery_rate": _rate(saved, residual), "salvaged_24h": saved_24h, "malformed_24h": residual_24h, "recovery_rate_24h": _rate(saved_24h, residual_24h), "last_at": last_at.isoformat() if last_at else None, }) out.sort( key=lambda x: ( x["malformed_24h"], x["salvaged_24h"] + x["malformed_24h"], x["malformed"], x["salvaged"] + x["malformed"], ), reverse=True, ) totals: dict[str, int | float | None] = dict(counts) totals["recovery_rate"] = _rate(counts["salvaged"], counts["malformed"]) totals["recovery_rate_24h"] = _rate( counts["salvaged_24h"], counts["malformed_24h"] ) return {"days": days, "rows": out, "total": totals} def scan_tool_wire_health_cached(days: int = 7) -> Dict[str, Any]: """管理页链路健康短时缓存；同窗口并发请求共享一次聚合。""" key = min(90, max(1, int(days))) now = time.monotonic() with _wire_cache_lock: cached = _wire_cache.get(key) if cached and cached[0] > now: return cached[1] for stale_key, (expires_at, _value) in list(_wire_cache.items()): if expires_at <= now: _wire_cache.pop(stale_key, None) value = scan_tool_wire_health(days=key) _wire_cache[key] = (time.monotonic() + _CACHE_TTL_SECONDS, value) return value # ── provider 级致命错误即时告警 ── # 命中判据:错误文案含余额/配额/认证类关键词 —— 这类错误不是单任务偶发,而是该 # provider 上所有 run 全挂(如 Zai 余额不足),等日巡检的 5 次/2 task 阈值太慢。 # 冷却:同归一化签名 6h 内只发一封(进程内存态,重启清零 —— 重启后再发一封可接受, # 比引入持久化状态表划算)。 _CRITICAL_RE = re.compile( r"余额不足|无可用资源包|请充值|欠费" r"|insufficient[_ ](?:quota|balance|funds)" r"|exceeded your current quota" r"|AuthenticationError|invalid[_ ]api[_ ]?key|api key.{0,20}(?:invalid|expired)", re.IGNORECASE, ) _ALERT_COOLDOWN_S = 6 * 3600 _alerted_at: Dict[str, datetime] = {} def alert_provider_critical(err: str, *, task_id: Any = None, model_profile: str = "") -> bool: """run 错误若属 provider 级致命(余额/认证)→ 立即邮件开发者;返回是否已发。 同步阻塞(SMTP),只应在 worker 线程调用(_run_agent_bg 的 except 路径本就在 to_thread 里)。所有失败静默 —— 告警绝不能反过来在错误路径上再抛异常。 """ try: if not err or not _CRITICAL_RE.search(err): return False sig = _normalize(err) now = datetime.now(timezone.utc) last = _alerted_at.get(sig) if last and (now - last).total_seconds() < _ALERT_COOLDOWN_S: return False _alerted_at[sig] = now import os from tools.send_email import send_email_smtp, smtp_configured print(f"[runerror] provider-critical task={task_id} mp={model_profile} " f"err={err[:200]}", flush=True) dev_email = os.getenv("ZCBOT_DEVELOPER_EMAIL", "").strip() if not (dev_email and smtp_configured()): print("[runerror] ZCBOT_DEVELOPER_EMAIL/SMTP 未配,仅日志", flush=True) return False body = "\n".join([ "检出 provider 级致命错误(余额/配额/认证),该 provider 上的 run 可能全部失败:", "", f"错误: {err[:500]}", f"模型档: {model_profile or '?'}", f"任务: {task_id or '?'}", "", f"同类签名 {_ALERT_COOLDOWN_S // 3600}h 内不再重复告警;" "完整聚合见 admin 工具失败面板(kind=run)。", ]) send_email_smtp(dev_email, f"[zcbot] provider 级错误:{sig[:60]}", body) return True except Exception: return False def format_alert(clusters: List[Dict[str, Any]], days: float) -> str: """聚集列表 → 告警邮件正文(纯文本)。""" lines = [f"近 {days:g} 天内检出 {len(clusters)} 类工具失败聚集(近 24h 活跃的在前):", ""] for c in clusters: n24 = c.get("count_24h", 0) lines.append( f"- [{c['tool']}/{c['kind']}] x{c['count']}" f"(近24h {n24} 次{',已安静' if not n24 else ''}," f"task {c['task_count']} 个 / 用户 {c['user_count']} 人," f"最近 {c['last_at']})" ) lines.append(f" 签名: {c['signature']}") lines.append(f" 样例: {c['sample'][:200]}") lines.append("") lines.append("排查入口:RUN.md 故障兜底表;历史案例:mermaid loopback DROP(0.58.10)。") return "\n".join(lines)