294 lines
10 KiB
Python
294 lines
10 KiB
Python
"""失败埋点(telemetry)—— 与计费(usage.py)分家的另一类 usage_events 写入。
|
|
|
|
这里的异常类 kind 均 cost 恒 0、tokens 不入 kind=chat 汇总,语义是**可观测性
|
|
留痕**而非用户花销 —— 它们是 admin「工具失败聚集」面板与巡检邮件(core/toolfail.py)
|
|
的唯一持久数据源。此前与真计费(chat/image/video/vision)混住 usage.py,污染
|
|
「usage=计费」的心智模型(架构审查数据层 Top4),2026-07-23 拆出。
|
|
|
|
kind 常量是 loop/llm_transport(写)与 toolfail(读 SQL)之间的契约单一事实源 ——
|
|
以前两侧靠字符串字面量约定,改名会静默漏读。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from decimal import Decimal
|
|
from typing import Any, Optional
|
|
from uuid import UUID
|
|
|
|
from .engine import session_scope
|
|
from .models import UsageEvent
|
|
|
|
# ── kind 契约常量(写侧本模块 / 读侧 core/toolfail.py 共用)──
|
|
KIND_TOOL_MALFORMED = "tool_malformed"
|
|
KIND_TOOL_SALVAGED = "tool_salvaged"
|
|
KIND_TOOL_FAILURE = "tool_failure"
|
|
KIND_EMPTY_RESPONSE = "empty_response"
|
|
KIND_RUN_ERROR = "run_error"
|
|
KIND_RUN_STOPPED = "run_stopped"
|
|
KIND_AGENT_GUARD = "agent_guard"
|
|
KIND_CONTEXT_FOLD_FAILURE = "context_fold_failure"
|
|
KIND_QUALITY_GATE = "quality_gate"
|
|
|
|
|
|
def _record_health_event(
|
|
*,
|
|
kind: str,
|
|
task_id: UUID,
|
|
user_id: UUID,
|
|
model_profile: str,
|
|
units: dict[str, Any],
|
|
message_id: Optional[UUID] = None,
|
|
) -> None:
|
|
with session_scope() as s:
|
|
s.add(UsageEvent(
|
|
user_id=user_id,
|
|
task_id=task_id,
|
|
message_id=message_id,
|
|
kind=kind,
|
|
model_profile=model_profile,
|
|
units=units,
|
|
cost_cny=Decimal("0"),
|
|
))
|
|
|
|
|
|
def record_tool_failure(
|
|
*,
|
|
task_id: UUID,
|
|
user_id: UUID,
|
|
message_id: UUID,
|
|
model_profile: str,
|
|
tool: str,
|
|
content: str,
|
|
arguments=None,
|
|
) -> bool:
|
|
"""结构化记录普通工具失败；正常结果不写入并返回 False。"""
|
|
from core.tool_failure import structured_failure
|
|
|
|
units = structured_failure(tool, content, arguments=arguments)
|
|
if units is None:
|
|
return False
|
|
event_kind = KIND_QUALITY_GATE if units["category"] == "quality_gate" else KIND_TOOL_FAILURE
|
|
_record_health_event(
|
|
kind=event_kind,
|
|
task_id=task_id,
|
|
user_id=user_id,
|
|
message_id=message_id,
|
|
model_profile=model_profile,
|
|
units=units,
|
|
)
|
|
return True
|
|
|
|
|
|
def record_run_stopped(
|
|
*, task_id: UUID, user_id: UUID, model_profile: str, reason: str,
|
|
) -> None:
|
|
_record_health_event(
|
|
kind=KIND_RUN_STOPPED,
|
|
task_id=task_id,
|
|
user_id=user_id,
|
|
model_profile=model_profile,
|
|
units={
|
|
"tool": "(run)", "failure_kind": "stopped", "signature": reason,
|
|
"category": "agent_control", "sample": reason,
|
|
},
|
|
)
|
|
|
|
|
|
def record_agent_guard(
|
|
*,
|
|
task_id: UUID,
|
|
user_id: UUID,
|
|
model_profile: str,
|
|
tool: str,
|
|
guard: str,
|
|
count: int,
|
|
signature: str = "",
|
|
) -> None:
|
|
from core.tool_failure import normalize_failure_signature
|
|
|
|
stable = normalize_failure_signature(f"{guard}: {signature}" if signature else guard)
|
|
_record_health_event(
|
|
kind=KIND_AGENT_GUARD,
|
|
task_id=task_id,
|
|
user_id=user_id,
|
|
model_profile=model_profile,
|
|
units={
|
|
"tool": tool, "failure_kind": "guard", "signature": stable,
|
|
"category": "agent_control", "sample": f"{guard}, count={count}",
|
|
"count": int(count),
|
|
},
|
|
)
|
|
|
|
|
|
def record_context_fold_failure(
|
|
*, task_id: UUID, user_id: UUID, model_profile: str, reason: str, detail: str = "",
|
|
) -> None:
|
|
from core.tool_failure import normalize_failure_signature
|
|
|
|
_record_health_event(
|
|
kind=KIND_CONTEXT_FOLD_FAILURE,
|
|
task_id=task_id,
|
|
user_id=user_id,
|
|
model_profile=model_profile,
|
|
units={
|
|
"tool": "(context)", "failure_kind": "context", "signature": reason,
|
|
"category": "agent_control",
|
|
"sample": normalize_failure_signature(detail) if detail else reason,
|
|
},
|
|
)
|
|
|
|
|
|
def record_quality_gate(
|
|
*, task_id: UUID, user_id: UUID, model_profile: str, tool: str, gate: str,
|
|
) -> None:
|
|
_record_health_event(
|
|
kind=KIND_QUALITY_GATE,
|
|
task_id=task_id,
|
|
user_id=user_id,
|
|
model_profile=model_profile,
|
|
units={
|
|
"tool": tool, "failure_kind": "gate", "signature": gate,
|
|
"category": "quality_gate", "sample": gate,
|
|
},
|
|
)
|
|
|
|
|
|
def record_malformed_tool_call(
|
|
*,
|
|
task_id: UUID,
|
|
user_id: UUID,
|
|
model_profile: str,
|
|
tool: str,
|
|
arg_len: int,
|
|
error: str,
|
|
head: str,
|
|
tail: str,
|
|
tokens_in: int = 0,
|
|
tokens_out: int = 0,
|
|
) -> None:
|
|
"""记一次被丢弃的畸形 tool_call(kind=tool_malformed)。
|
|
|
|
这不是记账而是留痕:畸形轮整轮丢弃、messages 无痕,此行是 admin「工具失败聚集」
|
|
面板与巡检邮件唯一的数据源(core/toolfail.py 第二段扫描按 units->>'tool'/'err'
|
|
聚合)。cost_cny 恒 0 —— cost 统计全 kind 合计且随任务展示,provider 抖动的浪费
|
|
不该算成用户花销;该轮真实 token 快照进 units,将来要算浪费成本可从 units 反推
|
|
(token 汇总只算 kind=chat,此处 tokens_in/out 不会混进 token 统计)。
|
|
"""
|
|
with session_scope() as s:
|
|
s.add(UsageEvent(
|
|
user_id=user_id,
|
|
task_id=task_id,
|
|
message_id=None, # 畸形轮不入 messages,无可关联
|
|
kind=KIND_TOOL_MALFORMED,
|
|
model_profile=model_profile,
|
|
units={
|
|
"tool": tool,
|
|
"len": int(arg_len),
|
|
"err": str(error)[:200],
|
|
"head": head,
|
|
"tail": tail,
|
|
"tokens_in": int(tokens_in),
|
|
"tokens_out": int(tokens_out),
|
|
},
|
|
cost_cny=Decimal("0"),
|
|
))
|
|
|
|
|
|
def record_salvaged_tool_call(
|
|
*,
|
|
task_id: UUID,
|
|
user_id: UUID,
|
|
model_profile: str,
|
|
tool: str,
|
|
arg_len: int,
|
|
salvaged_len: int,
|
|
head: str,
|
|
) -> None:
|
|
"""记一次从畸形 arguments 里抢救成功的 tool_call(kind=tool_salvaged)。
|
|
|
|
与 record_malformed_tool_call 是一对:畸形轮本会整轮丢弃 + 非流式重 roll,salvage
|
|
命中时改成就地抠出尾部完好 JSON、当轮照常执行,省掉一次重试。此行是留痕(cost 恒 0,
|
|
tokens 不入 kind=chat 汇总),供「工具失败聚集」面板 / 事后核对区分「真丢弃」与「抢救回」:
|
|
arg_len=损坏前缀+JSON 的总长,salvaged_len=抠出 JSON 的长,差值即被丢弃的 wire 垃圾前缀长。
|
|
head 存前 300 字(过 ascii 转义,消费端编码不可控)供核验前缀形态是否仍是 char-0 型。
|
|
"""
|
|
with session_scope() as s:
|
|
s.add(UsageEvent(
|
|
user_id=user_id,
|
|
task_id=task_id,
|
|
message_id=None, # 抢救出的 tool_call 正常执行并入 messages,但此留痕行不与之关联
|
|
kind=KIND_TOOL_SALVAGED,
|
|
model_profile=model_profile,
|
|
units={
|
|
"tool": tool,
|
|
"len": int(arg_len),
|
|
"salvaged_len": int(salvaged_len),
|
|
"head": head,
|
|
},
|
|
cost_cny=Decimal("0"),
|
|
))
|
|
|
|
|
|
def record_empty_response(
|
|
*,
|
|
task_id: UUID,
|
|
user_id: UUID,
|
|
model_profile: str,
|
|
attempt: int,
|
|
tokens_in: int = 0,
|
|
tokens_out: int = 0,
|
|
finish_reason: str = "",
|
|
) -> None:
|
|
"""记一次 provider 吐空(kind=empty_response):assistant 轮既无 tool_calls 又无正文。
|
|
|
|
背景(task 2a1bc25d 案):unifyllm 网关对某档 Claude 偶发把 tool_use 漏成正文 / 直接吐空,
|
|
run loop 见 tool_calls 空即当「模型答完」静默 done —— 无报错、run_status=idle,只能人肉
|
|
挖 DB 才发现。此行是留痕(cost 恒 0,tokens 不入 kind=chat 汇总):即使当轮非流式重试救回,
|
|
也留一条供「工具失败聚集」面板看到「哪个模型档在吐空」(第四段扫描,tool 名固定 "(empty)")。
|
|
attempt=第几次尝试(1=首个流式轮),tokens 快照供估算浪费。单条转瞬即逝不触发面板(阈值
|
|
min_count/min_tasks),跨 task 系统性吐空才冒头 —— 正是要抓的网关级抖动。
|
|
"""
|
|
with session_scope() as s:
|
|
s.add(UsageEvent(
|
|
user_id=user_id,
|
|
task_id=task_id,
|
|
message_id=None, # 空响应轮不入 messages(丢弃重试),无可关联
|
|
kind=KIND_EMPTY_RESPONSE,
|
|
model_profile=model_profile,
|
|
units={
|
|
"attempt": int(attempt),
|
|
"tokens_in": int(tokens_in),
|
|
"tokens_out": int(tokens_out),
|
|
# finish_reason 区分「网关 wire 吐空」(stop/其他)与「输出达上限被截断」
|
|
# (length)—— 后者是我方输出预算/推理失控,同上下文重试无效(见 loop 处理)。
|
|
"finish_reason": finish_reason or "",
|
|
},
|
|
cost_cny=Decimal("0"),
|
|
))
|
|
|
|
|
|
def record_run_error(
|
|
*,
|
|
task_id: UUID,
|
|
user_id: UUID,
|
|
model_profile: str,
|
|
error: str,
|
|
) -> None:
|
|
"""记一次 run 级终态错误(kind=run_error)。
|
|
|
|
留痕而非记账(cost 恒 0):run 在 LLM 请求层/构建期直接抛异常(RateLimitError
|
|
余额不足、认证失败等)时,整轮没有 tool 消息,错误只写 tasks.run_error 一列 ——
|
|
而该列只留最后一次,下次成功即被清掉。此行是「工具失败聚集」面板 + 巡检邮件
|
|
看到 run 级错误的唯一持久数据源(core/toolfail.py 第三段扫描按 units->>'err'
|
|
聚合)。task 2a1bc25d 案:Zai 余额不足连挂 3 次续跑,前端无提示、admin 无感知。
|
|
"""
|
|
with session_scope() as s:
|
|
s.add(UsageEvent(
|
|
user_id=user_id,
|
|
task_id=task_id,
|
|
message_id=None, # 错误轮通常无 assistant message 落库,无可关联
|
|
kind=KIND_RUN_ERROR,
|
|
model_profile=model_profile,
|
|
units={"err": str(error)[:500]},
|
|
cost_cny=Decimal("0"),
|
|
))
|