113 lines
4.8 KiB
Python
113 lines
4.8 KiB
Python
"""窗口压力诊断:线上是否真的逼近 / 撞过模型上下文上限(DESIGN §8.8 垫子够不够厚)。
|
|
|
|
三个问题,全部按 DB 实测回答:
|
|
1. usage_events(kind=chat)的 units.tokens_in = provider 实报 prompt tokens,
|
|
逐 model_profile 对照 config/models/*.yaml 的 reliable_context / max_context:
|
|
有多少调用越过 50% 压缩线 / 85% 折叠线 / 100% reliable / 硬上限 max_context。
|
|
2. tasks.run_error 里有没有 context/length 类的 provider 超限报错(终态错误)。
|
|
3. kind=context_fold 的事件量 = 折叠实际触发了多少次(85% 线被踩的次数)。
|
|
|
|
结论口径:若 (1) 无越过 100% reliable 的调用且 (2) 无超限终态错误 → 现有
|
|
50%/85% 双层门槛 + reliable 与 max 之间的余量足够,run 中途折叠 / 超限自愈
|
|
继续搁置(§5 无信号不实施)。
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
|
if line.strip().startswith("ZCBOT_DB_URL="):
|
|
os.environ["ZCBOT_DB_URL"] = line.split("=", 1)[1].strip()
|
|
from sqlalchemy import create_engine, text # noqa: E402
|
|
|
|
engine = create_engine(os.environ["ZCBOT_DB_URL"])
|
|
|
|
# model_profile("family.variant")→ (reliable_context, max_context),直读 yaml
|
|
caps: dict[str, tuple[int, int]] = {}
|
|
for f in (ROOT / "config" / "models").glob("*.yaml"):
|
|
data = yaml.safe_load(f.read_text(encoding="utf-8")) or {}
|
|
family = data.get("family", f.stem)
|
|
for variant, var in (data.get("variants") or {}).items():
|
|
caps[f"{family}.{variant}"] = (
|
|
int(var.get("reliable_context", 64_000)),
|
|
int(var.get("max_context", 128_000)),
|
|
)
|
|
|
|
with engine.connect() as conn:
|
|
rows = conn.execute(text(
|
|
"select model_profile, task_id, units, created_at::date"
|
|
" from usage_events where kind='chat'"
|
|
)).fetchall()
|
|
folds = conn.execute(text(
|
|
"select model_profile, count(*), min(created_at)::date, max(created_at)::date"
|
|
" from usage_events where kind='context_fold' group by 1"
|
|
)).fetchall()
|
|
errs = conn.execute(text(
|
|
"select task_id, model_profile, run_error, updated_at::date from tasks"
|
|
" where run_error is not null and run_error <> ''"
|
|
)).fetchall()
|
|
|
|
# ---- 1. prompt tokens vs 各模型门槛 ----
|
|
stat = defaultdict(lambda: {"n": 0, "max": 0, "max_task": "", "over50": 0,
|
|
"over85": 0, "over100": 0, "overmax": 0, "top": []})
|
|
for profile, task_id, units, day in rows:
|
|
tin = int((units or {}).get("tokens_in") or 0)
|
|
s = stat[profile]
|
|
s["n"] += 1
|
|
if tin > s["max"]:
|
|
s["max"], s["max_task"] = tin, str(task_id)[:8]
|
|
rel, mx = caps.get(profile, (64_000, 128_000))
|
|
if tin >= rel:
|
|
s["over100"] += 1
|
|
s["top"].append((tin, str(task_id)[:8], str(day)))
|
|
elif tin >= rel * 0.85:
|
|
s["over85"] += 1
|
|
elif tin >= rel * 0.5:
|
|
s["over50"] += 1
|
|
if tin >= mx:
|
|
s["overmax"] += 1
|
|
|
|
print("=== 1. chat 调用 prompt tokens vs 窗口门槛(usage_events 实报)===")
|
|
print(f"{'model_profile':<28}{'calls':>7}{'max_in':>9}{'(task)':>10}"
|
|
f"{'>=50%':>7}{'>=85%':>7}{'>=100%rel':>10}{'>=max':>7}")
|
|
for profile in sorted(stat, key=lambda p: -stat[p]["max"]):
|
|
s = stat[profile]
|
|
rel, mx = caps.get(profile, (0, 0))
|
|
known = "" if profile in caps else " [!] 无 yaml 档案,按默认 64k/128k 算"
|
|
print(f"{profile:<28}{s['n']:>7}{s['max']:>9}{s['max_task']:>10}"
|
|
f"{s['over50']:>7}{s['over85']:>7}{s['over100']:>10}{s['overmax']:>7}"
|
|
f" (rel={rel} max={mx}){known}")
|
|
|
|
hits = [(t, p) for p, s in stat.items() for t in s["top"]]
|
|
if hits:
|
|
print("\n[!] 越过 100% reliable_context 的调用样本(最多 10 条):")
|
|
for (tin, tid, day), profile in sorted(hits, key=lambda x: -x[0][0])[:10]:
|
|
print(f" {tin:>8} tokens task {tid} {day} {profile}")
|
|
else:
|
|
print("\n[ok] 没有任何调用越过 100% reliable_context。")
|
|
|
|
# ---- 2. 终态错误里的超限报错 ----
|
|
pat = re.compile(
|
|
r"context|length|token|exceed|too long|maximum|上下文|超(出|过).{0,6}(长度|上限)",
|
|
re.I,
|
|
)
|
|
ctx_errs = [(t, p, e, d) for t, p, e, d in errs if pat.search(e or "")]
|
|
print(f"\n=== 2. tasks.run_error 终态错误:总 {len(errs)} 条,疑似上下文超限 {len(ctx_errs)} 条 ===")
|
|
for t, p, e, d in ctx_errs[:10]:
|
|
print(f" task {str(t)[:8]} {d} [{p}] {(e or '')[:160]}")
|
|
if not ctx_errs and errs:
|
|
print(" [ok] 无一条匹配 context/length/超限类关键词。")
|
|
|
|
# ---- 3. 折叠实际触发量 ----
|
|
print("\n=== 3. context_fold 触发次数(85% 线被踩的实况)===")
|
|
if folds:
|
|
for profile, n, d1, d2 in folds:
|
|
print(f" {profile}: {n} 次({d1} ~ {d2})")
|
|
else:
|
|
print(" 0 次 —— 85% 折叠线从未被踩到。")
|