123 lines
4.1 KiB
Python
123 lines
4.1 KiB
Python
"""复现 task 2a1bc25d 的「工具意图漏成正文」现象。
|
|
|
|
用 build_agent(resume=True) 复原真实 system prompt + 全套工具 schema(只读 DB,不跑
|
|
loop 不写库),再拿原始用户消息 + 真实 tools 直接打 opus48,看网关回不回结构化
|
|
tool_calls。stream / non-stream 两条路径各打一发。输出 ASCII(Windows GBK 安全),
|
|
完整正文写文件。
|
|
"""
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
env = ROOT / ".env"
|
|
if env.exists():
|
|
for line in env.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if line and not line.startswith("#") and "=" in line:
|
|
k, v = line.split("=", 1)
|
|
os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))
|
|
os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
|
|
|
sys.path.insert(0, str(ROOT))
|
|
import litellm # noqa: E402
|
|
from core.agent_builder import build_agent # noqa: E402
|
|
|
|
_tag = (sys.argv[1] if len(sys.argv) > 1 else "default").replace(".", "_").replace("/", "_")
|
|
OUT = open(ROOT / "scripts" / f"_narrated_probe_{_tag}.txt", "w", encoding="utf-8")
|
|
|
|
|
|
def w(*a):
|
|
print(*a, file=OUT)
|
|
|
|
|
|
USER_ID = UUID("5c8af5a4-48cb-4515-bf88-822816e383c9")
|
|
TASK_ID = "2a1bc25d-d045-4ecd-acc9-b3ee2c8ccf09"
|
|
USER_MSG = "基于中国建材总院对接景德镇市汇报材料 制作ppt"
|
|
|
|
# 可传档位覆盖(同 task 的 system prompt + 31 工具,只换模型层),默认用 task 自身档。
|
|
MODEL_OVERRIDE = sys.argv[1] if len(sys.argv) > 1 else None
|
|
|
|
agent, session, sid, tstate, wd = build_agent(
|
|
user_id=USER_ID, session_id=TASK_ID, resume=True,
|
|
model_name=MODEL_OVERRIDE,
|
|
)
|
|
schemas = agent.executor.schemas()
|
|
caps = agent.caps
|
|
llm = agent.llm
|
|
|
|
sys_msg = next((m for m in session.messages if m.get("role") == "system"), None)
|
|
sys_txt = sys_msg["content"] if sys_msg else ""
|
|
|
|
w("[SETUP]")
|
|
w(f" model_id={caps.model_id} profile={caps.family}.{caps.variant}")
|
|
w(f" api_base={llm.api_base} temp={caps.optimal_temperature} "
|
|
f"parallel_tools={caps.parallel_tools}")
|
|
w(f" system_prompt chars={len(sys_txt)}")
|
|
w(f" tools count={len(schemas)}")
|
|
w(f" tool names={[s['function']['name'] for s in schemas]}")
|
|
w("")
|
|
|
|
messages = [
|
|
{"role": "system", "content": sys_txt},
|
|
{"role": "user", "content": USER_MSG},
|
|
]
|
|
|
|
kwargs_base = dict(
|
|
model=caps.model_id,
|
|
api_base=llm.api_base,
|
|
api_key=llm.api_key,
|
|
messages=messages,
|
|
tools=schemas,
|
|
temperature=caps.optimal_temperature,
|
|
timeout=300,
|
|
)
|
|
if caps.parallel_tools:
|
|
kwargs_base["parallel_tool_calls"] = True
|
|
|
|
stdout_lines = []
|
|
|
|
|
|
def probe(tag, stream):
|
|
w(f"========== {tag} ==========")
|
|
try:
|
|
if stream:
|
|
chunks = list(litellm.completion(
|
|
**kwargs_base, stream=True,
|
|
stream_options={"include_usage": True}))
|
|
resp = litellm.stream_chunk_builder(chunks)
|
|
else:
|
|
resp = litellm.completion(**kwargs_base)
|
|
except Exception as e:
|
|
w(f"[FAIL] {type(e).__name__}: {str(e)[:300]}")
|
|
stdout_lines.append(f"{tag}: FAIL {type(e).__name__}")
|
|
return
|
|
msg = resp.choices[0].message
|
|
tc = msg.tool_calls or []
|
|
content = msg.content or ""
|
|
u = resp.usage
|
|
w(f" usage prompt={u.prompt_tokens} completion={u.completion_tokens}")
|
|
w(f" tool_calls count={len(tc)}")
|
|
for i, t in enumerate(tc):
|
|
w(f" [{i}] {t.function.name}({(t.function.arguments or '')[:200]})")
|
|
w(f" content chars={len(content)}")
|
|
w(" content:")
|
|
w(content[:2000])
|
|
w("")
|
|
# narration 特征:无结构化 tool_calls 但正文里出现反引号工具名 bullet 参数
|
|
narrated = (not tc) and ("`" in content) and ("\n- " in content or "No result received" in content)
|
|
verdict = ("STRUCTURED tool_calls" if tc
|
|
else "NARRATED-as-text" if narrated
|
|
else "plain-text-no-tools")
|
|
stdout_lines.append(f"{tag}: {verdict} (tc={len(tc)}, content={len(content)}c)")
|
|
|
|
|
|
probe("NON-STREAM", stream=False)
|
|
probe("STREAM", stream=True)
|
|
|
|
OUT.close()
|
|
for ln in stdout_lines:
|
|
print(ln)
|
|
print("full -> scripts/_narrated_probe.txt")
|