From 1685004f96ee6e3a2984ae1295ee69f28de065cc Mon Sep 17 00:00:00 2001 From: caoqianming Date: Wed, 15 Jul 2026 15:29:49 +0800 Subject: [PATCH] =?UTF-8?q?feat(loop):=20=E7=A9=BA=E5=93=8D=E5=BA=94?= =?UTF-8?q?=E9=98=B2=E5=BE=A1=E2=80=94=E2=80=94provider=20=E5=90=90?= =?UTF-8?q?=E7=A9=BA=E8=87=AA=E5=8A=A8=E9=87=8D=E8=AF=95=20+=20warn=20?= =?UTF-8?q?=E8=87=AA=E5=81=9C=E4=B8=8D=E9=9D=99=E9=BB=98=20done=20+=20?= =?UTF-8?q?=E9=9D=A2=E6=9D=BF=E7=95=99=E7=97=95(bump=200.58.32)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit task 2a1bc25d 复盘:unifyllm 网关对某档 Claude 偶发把 tool_use 漏成正文/直接吐空,回来的轮 tool_calls=[] 且正文空,loop 当"模型答完"静默 done(run_status=idle、无报错、无终态失败), 表现为"自己中断",只能人肉挖 DB 才发现。复测该 bug 现已消失/瞬态,但"空 tool_calls 直接 done"这个失败模式本身太隐蔽,任何 provider 未来吐一次空都会静默卡死,故加防御(对称既有畸形 salvage 链)。 - _is_empty_response(tc 空且 content 去空白为空;纯 tool_call 轮不误判)挂进 _stream_llm 的 attempt 循环:空-空丢弃本轮走非流式重试(多数瞬态重发一次即恢复,用户无感) - run() 收尾点分空-空分支:重试耗尽仍空不发静默 done,改发可见 warn「模型返回空响应,已停止, 回复继续可重试」+ done(复用 stall 熔断自停话术,run_status 落 idle 可续),不引入终态 error - _log_empty_response stdout + usage_events(kind=empty_response,cost 0)双写留痕;core/toolfail 加第四段扫描聚成 tool=(empty)/kind=empty cluster(sample=model_profile 看哪个网关档在吐空), admin 面板 + 巡检邮件零改动自动带上(单次瞬态 <阈值不触发,跨 task 系统性吐空才冒头) - 刻意不做内容嗅探式"narrated tool_call 特征→重试":marker 每次变且与正常正文高度重叠,假阳性 (合法回答判坏反复重试)比罕见静默停更糟,那类只靠面板留痕兜、不改热路径行为 - tests: test_loop_empty_response.py 新增 7 件 + test_toolfail_malformed.py +2;顺手补回 test_loop_malformed_retry.py 因 salvage(0.58.24)落地后失修的 loop.executor 桩。全 247 测试绿 Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 + PROGRESS.md | 3 +- core/__init__.py | 2 +- core/loop.py | 80 +++++++++++++- core/storage/__init__.py | 2 + core/storage/usage.py | 34 ++++++ core/toolfail.py | 22 ++++ scripts/diag_narrated_toolcall_2a1bc25d.py | 122 +++++++++++++++++++++ scripts/diag_task_2a1bc25d.py | 72 ++++++++++++ tests/test_loop_empty_response.py | 110 +++++++++++++++++++ tests/test_loop_malformed_retry.py | 3 + tests/test_toolfail_malformed.py | 35 +++++- 12 files changed, 482 insertions(+), 7 deletions(-) create mode 100644 scripts/diag_narrated_toolcall_2a1bc25d.py create mode 100644 scripts/diag_task_2a1bc25d.py create mode 100644 tests/test_loop_empty_response.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ac5b3b..43222d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ > 所以不是每个版本号都有条目。条目格式 `## <版本> — <日期>`,新条目加在最上面。 > 工程口径的完整记录见 `PROGRESS.md` / git log。 +## 0.58.32 — 2026-07-15 + +- 模型偶发返回空内容时不再"无声停下":此前极少数情况下模型会回一个空响应,系统当成"已答完"直接结束,任务看起来像自己中断了却没有任何提示。现在遇到空响应会自动重试(多数一次即恢复,你多半察觉不到);万一重试后仍为空,会明确显示「模型返回空响应,已停止,回复『继续』可重试」,不再让你对着停住的对话摸不着头脑。 + ## 0.58.29 — 2026-07-15 - 修两处对话里文件相关的小毛病:助手回复正文里点文件路径链接不再整页报错(现在直接弹出预览,PPT 自动转成 PDF 看),文件名里带间隔号「·」的产物也能正常显示可点的文件小标签了。 diff --git a/PROGRESS.md b/PROGRESS.md index b0db212..a23f8b0 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,7 +2,7 @@ > 配合 `DESIGN.md`。本文件只记 phase 状态、决策偏差、文件量、下一步。每条 1-2 句:做了啥 + 关键判断;细节查 `git log` / `git diff` / `DESIGN §7.9`。 -最后更新:2026-07-15(用户停止后「已停止」持久提示,补齐 cancel 终态与 error 卡对称,bump 0.58.26) +最后更新:2026-07-15(空响应防御:provider 吐空自动重试 + warn 自停不静默 done + 面板留痕,bump 0.58.32) --- @@ -23,6 +23,7 @@ ### 2026-07 +- **07-15 / 0.58.32**:**空响应防御:provider 吐空不再被当正常收尾静默 done**(复盘 task 2a1bc25d "ppt3":unifyllm 网关对某档 Claude 偶发把 tool_use 漏成正文 / 直接吐空,回来的轮 `tool_calls=[]` 且正文空,`loop.py` 的 `if not tool_calls: done` 当"模型答完"收尾——run_status=idle、无报错、无终态失败,表现为"自己中断",只能人肉挖 DB 才发现;复测该 bug 现已消失/瞬态,但"空 tool_calls 直接 done"这个失败模式本身太隐蔽,任何 provider 未来吐一次空都会静默卡死,故加防御)。**两层**(对称既有畸形 salvage 链):① 自动重试——`_is_empty_response`(tc 空且 content 去空白为空;纯 tool_call 轮不误判)接进 `_stream_llm` 的 attempt 循环,和畸形同类:丢弃本轮走非流式重试(多数瞬态重发一次即好,用户无感),`_log_empty_response` stdout+`usage_events`(kind=`empty_response`,cost 0)双写留痕、DB 失败静默不阻塞;② 重试耗尽仍空——`run()` 收尾点分空-空分支:不发静默 `done`,改发**可见 warn**「模型返回空响应(已自动重试仍为空),已停止,回复『继续』可重试」+ done(复用 stall 熔断的 warn+自停话术,run_status 落 idle 可续),**不引入终态 error**(空响应可恢复,红 error 过重且与既有路径不对称)。③ 可观测——`core/toolfail.py` 加第四段扫描 kind=empty_response 聚成 tool=`(empty)`/kind=`empty` 的 cluster(sample=model_profile 看哪个网关档在吐空),即便自动救回也留痕、跨 task 系统性吐空才冒头(单次瞬态 <阈值不触发)。**刻意不做**内容嗅探式"narrated tool_call 特征→重试":marker 每次变、且与助手正常解释/贴代码的正文高度重叠,假阳性(把合法回答判坏反复重试)比它防的罕见静默停更糟——那类只靠面板留痕兜、不改热路径行为。测试:`tests/test_loop_empty_response.py`(_is_empty_response 四态含纯 tool_call 不误判 + 重试恢复/耗尽返空/正常文本不重试)、`tests/test_toolfail_malformed.py` +2(空响应聚集 / 单次瞬态不触发),顺手补回 `test_loop_malformed_retry.py` 因 salvage(0.58.24)落地后失修的 `loop.executor` 桩(空 schemas → salvage 返 False 走既有重试)。全 247 测试绿。盲区:provider-wire 抖动本机复现不出、/verify 端到端跑不了,靠单测+生产灰度。additive 内部机制,不碰对外 API/DB 契约。相关 memory:unifyllm tool_use 漏正文案 / deepseek 畸形 salvage 案。 - **07-15 / 0.58.31**:**撤回 §8.11 delegate(0.58.30 曾整体落地,revert + force-push 抹除)**。delegate(隔离检索子循环:FilteredExecutor + 内存态子 Session + loop 拦截 + 降 flash)机制本身没错、对标 Claude Code subagent 也成立,但**触发信号没坐实就上了**:自评时回看 `diag_search_args.py`,motivating 案子 `document_search` 122 次呈"一批批不同材料体系并行搜"形态,更像**批量扇出而非结果驱动探索**——若属实其正解与 mp_search 同为**批量工具 `document_search_batch`**(便宜/可预测/可诊断),delegate 是过度设计;叠加子循环 transcript 不落盘(诊断驱动的功能反不可诊断)、20 轮上限对 122 次负载偏低、强制 flash 对难检索可能降质等未验证坑。故整体撤回,只留验证过的批量工具(0.58.28)。**重建前置收紧**:先用 diag 确认某检索 task 的 query 是真探索(依赖前序结果、无法一次列全)而非可枚举批量,是批量走批量工具、只有真探索才 delegate(DESIGN §8.11 记该教训)。撤法:`git reset --hard` 到 0.58.29 + `--force-with-lease` 推,delegate commit 从历史抹除(远程 tip 确认无他人后续提交才 force);保留本条 PROGRESS 记录"试过并撤回"免日后盲目重建。 - **07-15 / 0.58.29**:**修对话正文文件锚点两处回归(含 `·` 的 pptx 丢 chip + 正文文件链接点击整页 404)**。① chip 丢失:`media.js` 的 `_TAIL_CLS` 把间隔号 `·`(U+00B7)当尾部排除字符,而它是文件名合法分隔符(`AI赋能陶瓷产业·…-图标增强版.pptx`)→ 在 `·` 处截断成目录不挂 chip。去掉 `·`(尾部标点另有 strip 兜底);node 验证 OLD→`[]`、NEW→两路径全出。② 链接 404:模型把相对路径写成 `[]()`,`renderMd` 出 ``,但 `chat.js` 的 `#chat-stream` 点击委托无 `` 拦截 → 原生跳转打到 API 不存在路由返 FastAPI 默认 `{"detail":"Not Found"}`。加拦截 `.msg .body a[href]`:外链开新标签,内部相对路径 `openFilePreview(rel)`(复用预览 modal,pptx 转 PDF),`#` 锚点走默认。纯前端修复,不碰对外契约。 - **07-15 / 0.58.28**:**`mp_search_summary` 加批量入参 `formulas=[...]`(检索型烧 token 头号根因的定点解)**(诊断:`diag_tool_repeat.py` 全库扫,pymatgen task `fe2d8b73` 687 次 tool_call 里 `mp_search_summary` 占 **562 次**、重复率 91%——逐化学式一 formula 一轮走 agent loop 建氧化物表,562 份中间 JSON 全流经主上下文且每轮重发)。根因不是"检索发散"而是**工具只收单 `formula` + host-side 持 MP key(§7.5 #7 禁在 run_python 里循环)→ 被迫一式一调**。**这类"已知清单批量扇出"的正解是工具批量化,不是 delegate 子循环**(delegate 只把 562 轮搬进子上下文、基础 token 一分不省;批量化直接把 562 次并成 1 次工具调用+全部中间数据落盘不进对话)。实现:`formulas: list[str]`(≤300/次,超则报错分批)触发 batch 分支,复用**单个 MPRester 会话**逐式查(`num_chunks=1` fair-use 护栏不变)、每式按 `energy_above_hull` 升序(稳定相优先)、**单式出错不连坐整批**(记 `{"formula","error"}` 继续);结果聚合写 `working_dir/materials/mp_search_batch_.json`(内容寻址、同批幂等)、**只回紧凑 per-formula 摘要 + 路径**(N 个有结果/M 个空错 + 空错清单前 20),引导模型用 `read`/`run_python` 直读文件、勿逐条 dump。单查询路径(`formula`/`material_ids`/`elements`)**字节不变**保向后兼容;`working_dir` 设可选(裸构造 `MaterialsProjectSearchSummaryTool()` 不破,测试/smoke 无 working_dir 时降级内联返回)。`agent_builder` 注入 `working_dir_path`(同 `mp_get_structure`);pymatgen SKILL.md 加"多式用 `formulas` 批量"引导(WHY+WHAT,不写死步骤);`tests/test_secret_host_tools.py` +1(去重/落盘/稳定相排序/单式错误隔离/无 key 泄漏),原单查询测试补回 `chunk_size` 断言。additive 内部工具能力,不碰对外 API/DB 契约。**背景**:这是"检索型烧 token"三头(mp_search 562 批量扇出 / document_search 122 探索检索 / research code 试错抖动)里第一头的定点解;探索检索型留待 §8.11 delegate,代码试错归 RepeatGuard,不混为一谈。 diff --git a/core/__init__.py b/core/__init__.py index 6796229..674afc0 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,3 +1,3 @@ # zcbot 版本号单一事实源:web/app.py 的 FastAPI version、/healthz 返回、前端展示都引这里。 # 改版本只动这一行。 -__version__ = "0.58.31" +__version__ = "0.58.32" diff --git a/core/loop.py b/core/loop.py index e5d8698..7e0d852 100644 --- a/core/loop.py +++ b/core/loop.py @@ -29,6 +29,7 @@ from .salvage import salvage_tool_arguments from .session import Session from .storage import ( record_chat_usage, + record_empty_response, record_malformed_tool_call, record_salvaged_tool_call, ) @@ -177,6 +178,56 @@ def _malformed_tool_calls(response: Any) -> List[str]: return bad +def _is_empty_response(response: Any) -> bool: + """检出「空响应」:assistant 轮既无 tool_calls 又无正文(去空白后为空)。 + + 背景(task 2a1bc25d 案):provider wire 偶发吐空 —— 截断流 / finish_reason 无内容 / + 网关把 tool_use 漏成正文后又丢空,回来的这一轮 tool_calls 空且 content 空。run loop + 见 tool_calls 空即当「模型答完」静默 done、返回空串,无报错、run_status=idle,与卡死 + 无法区分。故和畸形同类对待:丢弃本轮走非流式重试。 + 注意:纯 tool_call 轮(tc 非空、content 空)不算空响应 —— 那是正常的工具调用轮。 + """ + try: + msg = response.choices[0].message + except (AttributeError, IndexError, TypeError): + return False + if getattr(msg, "tool_calls", None): + return False + content = getattr(msg, "content", None) or "" + return not content.strip() + + +def _log_empty_response( + task_id: Any, user_id: Any, model_profile: str, response: Any, attempt: int, +) -> None: + """空响应留痕:stdout + usage_events(kind=empty_response)双写,任何一路失败都静默。 + + 与 _log_malformed_args 对称:空响应轮同样整轮丢弃、messages 无痕,这里是唯一留痕 —— + 供事后定性(哪个模型档在吐空)与「工具失败聚集」面板第四段聚合。留痕绝不能反过来打断 + 重试主路径(单测无 DB 时 record_* 落库失败也吞掉)。 + """ + try: + usage = _extract_usage_details(getattr(response, "usage", None)) + print( + f"[empty_response] task={task_id} mp={model_profile} attempt={attempt} " + f"tok={usage['tokens_in']}/{usage['tokens_out']}", + flush=True, + ) + try: + record_empty_response( + task_id=task_id, + user_id=user_id, + model_profile=model_profile, + attempt=attempt, + tokens_in=usage["tokens_in"], + tokens_out=usage["tokens_out"], + ) + except Exception: + pass # DB 不可用(如单测无 DB)不影响 stdout 留痕与重试 + except Exception: + pass + + def _log_malformed_args( task_id: Any, user_id: Any, model_profile: str, response: Any, ) -> None: @@ -388,8 +439,19 @@ class AgentLoop: # content 已通过 stream 流式 emit 过 delta,这里不再 emit 整段 text 事件。 if not tool_calls: + content = getattr(msg, "content", None) or "" + # 空-空且重试后仍空(_stream_llm 已耗尽非流式重试):不能当正常收尾静默 done, + # 否则表现为「无报错自己停」(task 2a1bc25d 案)。发可见 warn + 留「继续」入口, + # 复用 stall 熔断那套自停话术(warn + done,run_status 落 idle,用户可续)。 + if not content.strip(): + self._emit({ + "type": "warn", + "msg": "模型返回空响应(已自动重试仍为空),已停止,回复「继续」可重试。", + }) + self._emit({"type": "done"}) + return "[stopped: empty response]" self._emit({"type": "done"}) - return getattr(msg, "content", None) or "" + return content step_productive = False for i, tc in enumerate(tool_calls): @@ -531,6 +593,22 @@ class AgentLoop: bad = _malformed_tool_calls(response) if not bad: + # 空响应(tc 空且正文空):provider wire 吐空,和畸形同类瞬态故障 —— + # 丢弃本轮走非流式重试(多数瞬态重发一次即好,用户无感),留痕供面板可见。 + if _is_empty_response(response): + _log_empty_response( + self.session.task_id, self.user_id, + f"{self.caps.family}.{self.caps.variant}", response, attempt + 1, + ) + self._emit({ + "type": "warn", + "msg": ( + "模型返回空响应,丢弃本轮" + f"{'重试' if use_nonstream else ',改非流式重试'}" + f" ({attempt + 1}/{self._MAX_MALFORMED_ATTEMPTS})" + ), + }) + continue return response, False # 先尝试就地抢救:畸形是 char-0 垃圾前缀 + 尾部完好 JSON(定层已证 provider-wire), # 全部畸形 tool_call 都能抠出干净 JSON 才改写并当轮继续,省掉一次非流式重试; diff --git a/core/storage/__init__.py b/core/storage/__init__.py index f08481e..244ad7f 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -14,6 +14,7 @@ from .engine import ( ) from .usage import ( record_chat_usage, + record_empty_response, record_malformed_tool_call, record_salvaged_tool_call, ) @@ -33,6 +34,7 @@ __all__ = [ "get_engine", "get_task", "record_chat_usage", + "record_empty_response", "record_malformed_tool_call", "record_salvaged_tool_call", "session_scope", diff --git a/core/storage/usage.py b/core/storage/usage.py index d79f643..ec06a6e 100644 --- a/core/storage/usage.py +++ b/core/storage/usage.py @@ -225,6 +225,40 @@ def record_salvaged_tool_call( )) +def record_empty_response( + *, + task_id: UUID, + user_id: UUID, + model_profile: str, + attempt: int, + tokens_in: int = 0, + tokens_out: int = 0, +) -> None: + """记一次 provider 吐空(kind=empty_response):assistant 轮既无 tool_calls 又无正文。 + + 背景(task 2a1bc25d 案):unifyllm 网关对某档 Claude 偶发把 tool_use 漏成正文 / 直接吐空, + run loop 见 tool_calls 空即当「模型答完」静默 done —— 无报错、run_status=idle,只能人肉 + 挖 DB 才发现。此行是留痕(cost 恒 0,tokens 不入 kind=chat 汇总):即使当轮非流式重试救回, + 也留一条供「工具失败聚集」面板看到「哪个模型档在吐空」(第四段扫描,tool 名固定 "(empty)")。 + attempt=第几次尝试(1=首个流式轮),tokens 快照供估算浪费。单条转瞬即逝不触发面板(阈值 + min_count/min_tasks),跨 task 系统性吐空才冒头 —— 正是要抓的网关级抖动。 + """ + with session_scope() as s: + s.add(UsageEvent( + user_id=user_id, + task_id=task_id, + message_id=None, # 空响应轮不入 messages(丢弃重试),无可关联 + kind="empty_response", + model_profile=model_profile, + units={ + "attempt": int(attempt), + "tokens_in": int(tokens_in), + "tokens_out": int(tokens_out), + }, + cost_cny=Decimal("0"), + )) + + def record_run_error( *, task_id: UUID, diff --git a/core/toolfail.py b/core/toolfail.py index d3375a5..fa5e915 100644 --- a/core/toolfail.py +++ b/core/toolfail.py @@ -18,6 +18,11 @@ usage_events(kind=run_error),在此按归一化错误签名聚合(tool 名固定 其中 provider 级致命错误(余额/认证)另走 `alert_provider_critical` 即时邮件, 不等日巡检 —— 这类错误会让该 provider 上所有用户的所有 run 全挂。 +第四数据源(0.58.32):provider 吐空(kind=empty_response)—— assistant 轮既无 +tool_calls 又无正文,会被 run loop 当正常收尾静默 done(task 2a1bc25d 案:网关把 +tool_use 漏成正文后丢空)。loop 落 usage_events(kind=empty_response),在此按固定签名 +聚合(tool 名固定 "(empty)",sample=model_profile,看哪个网关档在吐空)。 + 失败判定(tool content 的三类标记,形态见 executor_docker/_host): - `[Error` 开头 —— 执行器/工具层报错([Error]、[Error executing ...]) - `command timed out` —— shell/run_python 超时 @@ -136,6 +141,17 @@ def scan_tool_failures( ), {"cutoff": cutoff}, ).fetchall() + # 第四段:provider 吐空(kind=empty_response)。assistant 轮既无 tool_calls 又无正文, + # 会被 run loop 当正常收尾静默 done;loop 落 usage_events(kind=empty_response)是唯一 + # 留痕。tool 名固定 "(empty)",签名固定,sample=model_profile —— 看是哪个网关档在吐空。 + erows = s.execute( + text( + "select task_id, user_id, created_at, model_profile " + "from usage_events " + "where kind = 'empty_response' and created_at >= :cutoff" + ), + {"cutoff": cutoff}, + ).fetchall() agg: Dict[Tuple[str, str], Dict[str, Any]] = {} @@ -191,6 +207,12 @@ def scan_tool_failures( for task_id, user_id, created_at, err in rrows: _add("(run)", "run", err or "?", err or "", task_id, user_id, created_at) + for task_id, user_id, created_at, model_profile in erows: + _add( + "(empty)", "empty", "provider returned empty response", + model_profile or "?", task_id, user_id, created_at, + ) + out = [] for c in agg.values(): if c["count"] < min_count or len(c["tasks"]) < min_tasks: diff --git a/scripts/diag_narrated_toolcall_2a1bc25d.py b/scripts/diag_narrated_toolcall_2a1bc25d.py new file mode 100644 index 0000000..9e5aa2b --- /dev/null +++ b/scripts/diag_narrated_toolcall_2a1bc25d.py @@ -0,0 +1,122 @@ +"""复现 task 2a1bc25d 的「工具意图漏成正文」现象。 + +用 build_agent(resume=True) 复原真实 system prompt + 全套工具 schema(只读 DB,不跑 +loop 不写库),再拿原始用户消息 + 真实 tools 直接打 opus48,看网关回不回结构化 +tool_calls。stream / non-stream 两条路径各打一发。输出 ASCII(Windows GBK 安全), +完整正文写文件。 +""" +import os +import sys +from pathlib import Path +from uuid import UUID + +ROOT = Path(__file__).resolve().parent.parent +env = ROOT / ".env" +if env.exists(): + for line in env.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") + +sys.path.insert(0, str(ROOT)) +import litellm # noqa: E402 +from core.agent_builder import build_agent # noqa: E402 + +_tag = (sys.argv[1] if len(sys.argv) > 1 else "default").replace(".", "_").replace("/", "_") +OUT = open(ROOT / "scripts" / f"_narrated_probe_{_tag}.txt", "w", encoding="utf-8") + + +def w(*a): + print(*a, file=OUT) + + +USER_ID = UUID("5c8af5a4-48cb-4515-bf88-822816e383c9") +TASK_ID = "2a1bc25d-d045-4ecd-acc9-b3ee2c8ccf09" +USER_MSG = "基于中国建材总院对接景德镇市汇报材料 制作ppt" + +# 可传档位覆盖(同 task 的 system prompt + 31 工具,只换模型层),默认用 task 自身档。 +MODEL_OVERRIDE = sys.argv[1] if len(sys.argv) > 1 else None + +agent, session, sid, tstate, wd = build_agent( + user_id=USER_ID, session_id=TASK_ID, resume=True, + model_name=MODEL_OVERRIDE, +) +schemas = agent.executor.schemas() +caps = agent.caps +llm = agent.llm + +sys_msg = next((m for m in session.messages if m.get("role") == "system"), None) +sys_txt = sys_msg["content"] if sys_msg else "" + +w("[SETUP]") +w(f" model_id={caps.model_id} profile={caps.family}.{caps.variant}") +w(f" api_base={llm.api_base} temp={caps.optimal_temperature} " + f"parallel_tools={caps.parallel_tools}") +w(f" system_prompt chars={len(sys_txt)}") +w(f" tools count={len(schemas)}") +w(f" tool names={[s['function']['name'] for s in schemas]}") +w("") + +messages = [ + {"role": "system", "content": sys_txt}, + {"role": "user", "content": USER_MSG}, +] + +kwargs_base = dict( + model=caps.model_id, + api_base=llm.api_base, + api_key=llm.api_key, + messages=messages, + tools=schemas, + temperature=caps.optimal_temperature, + timeout=300, +) +if caps.parallel_tools: + kwargs_base["parallel_tool_calls"] = True + +stdout_lines = [] + + +def probe(tag, stream): + w(f"========== {tag} ==========") + try: + if stream: + chunks = list(litellm.completion( + **kwargs_base, stream=True, + stream_options={"include_usage": True})) + resp = litellm.stream_chunk_builder(chunks) + else: + resp = litellm.completion(**kwargs_base) + except Exception as e: + w(f"[FAIL] {type(e).__name__}: {str(e)[:300]}") + stdout_lines.append(f"{tag}: FAIL {type(e).__name__}") + return + msg = resp.choices[0].message + tc = msg.tool_calls or [] + content = msg.content or "" + u = resp.usage + w(f" usage prompt={u.prompt_tokens} completion={u.completion_tokens}") + w(f" tool_calls count={len(tc)}") + for i, t in enumerate(tc): + w(f" [{i}] {t.function.name}({(t.function.arguments or '')[:200]})") + w(f" content chars={len(content)}") + w(" content:") + w(content[:2000]) + w("") + # narration 特征:无结构化 tool_calls 但正文里出现反引号工具名 bullet 参数 + narrated = (not tc) and ("`" in content) and ("\n- " in content or "No result received" in content) + verdict = ("STRUCTURED tool_calls" if tc + else "NARRATED-as-text" if narrated + else "plain-text-no-tools") + stdout_lines.append(f"{tag}: {verdict} (tc={len(tc)}, content={len(content)}c)") + + +probe("NON-STREAM", stream=False) +probe("STREAM", stream=True) + +OUT.close() +for ln in stdout_lines: + print(ln) +print("full -> scripts/_narrated_probe.txt") diff --git a/scripts/diag_task_2a1bc25d.py b/scripts/diag_task_2a1bc25d.py new file mode 100644 index 0000000..b70c15a --- /dev/null +++ b/scripts/diag_task_2a1bc25d.py @@ -0,0 +1,72 @@ +"""Dump task 2a1bc25d full row + messages (ASCII safe, prefix match on task_id).""" +import json +import os +import sys +from pathlib import Path + +env = Path(__file__).resolve().parent.parent / ".env" +for line in env.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("ZCBOT_DB_URL="): + os.environ["ZCBOT_DB_URL"] = line.split("=", 1)[1].strip() +from sqlalchemy import create_engine, text # noqa: E402 + +import builtins # noqa: E402 + +_out = open(Path(__file__).resolve().parent / "_task_2a1bc25d.txt", "w", encoding="utf-8") + + +def print(*a, **k): # noqa: A001 + builtins.print(*a, **k, file=_out) + + +engine = create_engine(os.environ["ZCBOT_DB_URL"]) +PREFIX = "2a1bc25d" + + +def s(x, n=4000): + t = str(x or "") + return t if len(t) <= n else t[:n] + f"...[+{len(t)-n} chars]" + + +with engine.connect() as conn: + row = conn.execute( + text("select * from tasks where task_id::text like :p"), {"p": PREFIX + "%"} + ).mappings().fetchone() + if not row: + print("[NO TASK]", PREFIX) + sys.exit(1) + TID = row["task_id"] + print("[TASK ROW]") + for k, v in dict(row).items(): + print(f" {k} = {s(v, 2000)}") + + msgs = conn.execute( + text("select idx,payload,model_profile,tokens_in,tokens_out,created_at from messages " + "where task_id=:t order by idx"), + {"t": TID}, + ).fetchall() + print(f"\n[MESSAGES] count={len(msgs)}\n") + for idx, p, mp, ti, to, ca in msgs: + if isinstance(p, str): + try: + p = json.loads(p) + except Exception as e: + print(f"[{idx}] !! payload JSON parse fail: {e}; raw head: {s(p, 500)}") + continue + role = p.get("role") + head = f"[{idx}] {role}" + if mp: + head += f" ({mp})" + if ti or to: + head += f" tok={ti}/{to}" + head += f" at={ca}" + print(head) + content = p.get("content") + if content is not None: + print(" content:", s(content, 2000)) + for tc in p.get("tool_calls") or []: + fn = tc.get("function") or {} + print(f" CALL {fn.get('name')}({s(fn.get('arguments'), 800)})") + if role == "tool": + print(f" TOOL[{p.get('name')}] id={p.get('tool_call_id')}:", s(content, 1200)) + print() diff --git a/tests/test_loop_empty_response.py b/tests/test_loop_empty_response.py new file mode 100644 index 0000000..34f7701 --- /dev/null +++ b/tests/test_loop_empty_response.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from core.loop import AgentLoop, _is_empty_response # noqa: E402 + + +def _resp_empty(content=None): + """空响应桩:无 tool_calls,content 为空/None/空白。""" + msg = SimpleNamespace(tool_calls=[], content=content) + return SimpleNamespace(choices=[SimpleNamespace(message=msg)], usage=None) + + +def _resp_text(text): + """正常文本收尾桩:无 tool_calls 但有正文。""" + msg = SimpleNamespace(tool_calls=[], content=text) + return SimpleNamespace(choices=[SimpleNamespace(message=msg)], usage=None) + + +def _resp_tool(): + """纯 tool_call 轮桩:有 tool_calls、content 空 —— 不算空响应。""" + tc = SimpleNamespace(function=SimpleNamespace(name="glob", arguments='{"pattern":"*"}')) + msg = SimpleNamespace(tool_calls=[tc], content=None) + return SimpleNamespace(choices=[SimpleNamespace(message=msg)], usage=None) + + +def _make_loop(stream_results, nonstream_results): + loop = object.__new__(AgentLoop) + loop.caps = SimpleNamespace(reliable_context=64_000, family="test", variant="t") + loop.session = SimpleNamespace(messages=[], task_id="test-task") + loop.user_id = "test-user" # 无 DB:_log_empty_response 落库路径静默跳过 + loop.events = [] + loop._emit = loop.events.append + calls = {"stream": 0, "nonstream": 0} + + def collect_stream_once(llm_messages): + calls["stream"] += 1 + return stream_results.pop(0), False + + def nonstream_once(llm_messages): + calls["nonstream"] += 1 + return nonstream_results.pop(0) + + loop._collect_stream_once = collect_stream_once + loop._nonstream_once = nonstream_once + return loop, calls + + +class TestIsEmptyResponse(unittest.TestCase): + def test_empty_empty_is_empty(self): + self.assertTrue(_is_empty_response(_resp_empty(None))) + self.assertTrue(_is_empty_response(_resp_empty(""))) + self.assertTrue(_is_empty_response(_resp_empty(" \n\t "))) + + def test_text_answer_not_empty(self): + self.assertFalse(_is_empty_response(_resp_text("这是最终回答"))) + + def test_pure_tool_call_round_not_empty(self): + # 纯工具调用轮(content 空但有 tool_calls)是正常轮,绝不能误判为空响应 + self.assertFalse(_is_empty_response(_resp_tool())) + + def test_malformed_response_shape_is_safe(self): + # 畸形 / 缺字段的桩不应抛,只返 False + self.assertFalse(_is_empty_response(SimpleNamespace(choices=[]))) + self.assertFalse(_is_empty_response(SimpleNamespace())) + + +class TestEmptyResponseRetry(unittest.TestCase): + def test_empty_stream_falls_back_to_nonstream(self): + """空响应首败即降级非流式,重发拿到正文 → 恢复,1 条 warn 提「非流式」。""" + loop, calls = _make_loop([_resp_empty()], [_resp_text("ok")]) + resp, cancelled = loop._stream_llm() + self.assertFalse(cancelled) + self.assertFalse(_is_empty_response(resp)) + self.assertEqual(calls, {"stream": 1, "nonstream": 1}) + warns = [e for e in loop.events if e.get("type") == "warn"] + self.assertEqual(len(warns), 1) + self.assertIn("空响应", warns[0]["msg"]) + self.assertIn("非流式", warns[0]["msg"]) + # 每 attempt 重发 llm_start,让前端占位段重建 + starts = [e for e in loop.events if e.get("type") == "llm_start"] + self.assertEqual(len(starts), 2) + + def test_all_attempts_empty_returns_empty(self): + """全部尝试仍空 → 返回最后一次空 response(交 run() 收尾分支 warn 自停),不死循环。""" + n_nonstream = AgentLoop._MAX_MALFORMED_ATTEMPTS - 1 + loop, calls = _make_loop([_resp_empty()], [_resp_empty()] * n_nonstream) + resp, cancelled = loop._stream_llm() + self.assertFalse(cancelled) + self.assertTrue(_is_empty_response(resp)) + self.assertEqual(calls, {"stream": 1, "nonstream": n_nonstream}) + warns = [e for e in loop.events if e.get("type") == "warn"] + self.assertEqual(len(warns), AgentLoop._MAX_MALFORMED_ATTEMPTS) + + def test_clean_text_no_retry(self): + """正常文本收尾轮不触发重试。""" + loop, calls = _make_loop([_resp_text("done")], []) + resp, cancelled = loop._stream_llm() + self.assertFalse(cancelled) + self.assertEqual(calls, {"stream": 1, "nonstream": 0}) + self.assertEqual([e for e in loop.events if e.get("type") == "warn"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_loop_malformed_retry.py b/tests/test_loop_malformed_retry.py index d398e6f..9e339c3 100644 --- a/tests/test_loop_malformed_retry.py +++ b/tests/test_loop_malformed_retry.py @@ -29,6 +29,9 @@ def _make_loop(stream_results, nonstream_results): loop.caps = SimpleNamespace(reliable_context=64_000, family="test", variant="t") loop.session = SimpleNamespace(messages=[], task_id="test-task") loop.user_id = "test-user" # 无 DB 环境:_log_malformed_args 的落库路径应静默跳过 + # salvage(0.58.24)接在畸形重试前:空 schemas → 未知工具无白名单 → _try_salvage 返 False, + # 这些用例本就测「salvage 救不了 → 落非流式重试」,给个空 executor 即可走到该分支。 + loop.executor = SimpleNamespace(schemas=lambda: []) loop.events = [] loop._emit = loop.events.append calls = {"stream": 0, "nonstream": 0} diff --git a/tests/test_toolfail_malformed.py b/tests/test_toolfail_malformed.py index 0191cd8..6285ac8 100644 --- a/tests/test_toolfail_malformed.py +++ b/tests/test_toolfail_malformed.py @@ -16,12 +16,13 @@ NOW = datetime.now(timezone.utc) class _FakeSession: - """按 SQL 关键字分流三段查询:messages / tool_malformed / run_error。""" + """按 SQL 关键字分流四段查询:messages / tool_malformed / run_error / empty_response。""" - def __init__(self, msg_rows, malformed_rows, run_rows): + def __init__(self, msg_rows, malformed_rows, run_rows, empty_rows): self._msg_rows = msg_rows self._malformed_rows = malformed_rows self._run_rows = run_rows + self._empty_rows = empty_rows def execute(self, clause, params=None): sql = str(clause) @@ -29,15 +30,17 @@ class _FakeSession: rows = self._malformed_rows elif "run_error" in sql: rows = self._run_rows + elif "empty_response" in sql: + rows = self._empty_rows else: rows = self._msg_rows return SimpleNamespace(fetchall=lambda: rows) -def _scan(msg_rows, malformed_rows, run_rows=None, **kw): +def _scan(msg_rows, malformed_rows, run_rows=None, empty_rows=None, **kw): @contextmanager def fake_scope(): - yield _FakeSession(msg_rows, malformed_rows, run_rows or []) + yield _FakeSession(msg_rows, malformed_rows, run_rows or [], empty_rows or []) with patch.object(tf, "session_scope", fake_scope): return tf.scan_tool_failures(**kw) @@ -116,6 +119,30 @@ class TestToolfailRunError(unittest.TestCase): self.assertIn("余额不足", c["sample"]) +class TestToolfailEmptyResponse(unittest.TestCase): + def test_empty_response_rows_cluster(self): + """usage_events 的 empty_response 行聚成 kind=empty / tool=(empty) 的 cluster, + sample=model_profile(看哪个网关档在吐空)。""" + rows = [ + ("t1", "u1", NOW - timedelta(hours=1), "unifyllm.opus48"), + ("t2", "u2", NOW - timedelta(minutes=20), "unifyllm.opus48"), + ] + out = _scan([], [], empty_rows=rows, days=7, min_count=2, min_tasks=2) + self.assertEqual(len(out), 1) + c = out[0] + self.assertEqual(c["tool"], "(empty)") + self.assertEqual(c["kind"], "empty") + self.assertEqual(c["count"], 2) + self.assertEqual(c["task_count"], 2) + self.assertEqual(c["sample"], "unifyllm.opus48") + + def test_single_transient_below_threshold(self): + """单次瞬态吐空(