330 lines
13 KiB
Python
330 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import sys
|
|
import unittest
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import core.toolfail as tf # noqa: E402
|
|
from core.storage.telemetry import ( # noqa: E402
|
|
KIND_AGENT_GUARD,
|
|
KIND_EMPTY_RESPONSE,
|
|
KIND_RUN_ERROR,
|
|
KIND_TOOL_FAILURE,
|
|
KIND_TOOL_MALFORMED,
|
|
)
|
|
from core.tool_failure import structured_failure # noqa: E402
|
|
|
|
NOW = datetime.now(timezone.utc)
|
|
|
|
|
|
class _FakeSession:
|
|
"""工具健康只允许一次 usage_events 查询。"""
|
|
|
|
def __init__(self, rows):
|
|
self._rows = rows
|
|
self.calls = 0
|
|
|
|
def execute(self, clause, params=None):
|
|
sql = str(clause)
|
|
self.calls += 1
|
|
if "from usage_events" not in sql or "from messages" in sql:
|
|
raise AssertionError(sql)
|
|
return SimpleNamespace(fetchall=lambda: self._rows)
|
|
|
|
|
|
def _structured_row(row):
|
|
task_id, user_id, created_at, tool, content = row[:5]
|
|
arguments = None
|
|
if len(row) > 6 and isinstance(row[6], dict):
|
|
calls = row[6].get("tool_calls") or []
|
|
if calls:
|
|
arguments = (calls[0].get("function") or {}).get("arguments")
|
|
units = structured_failure(tool, content, arguments=arguments)
|
|
assert units is not None
|
|
return (
|
|
KIND_TOOL_FAILURE, task_id, user_id, created_at, None, tool,
|
|
None, None, None, units["failure_kind"], units["signature"],
|
|
units["category"], units["sample"],
|
|
)
|
|
|
|
|
|
def _scan(msg_rows, malformed_rows, run_rows=None, empty_rows=None, **kw):
|
|
event_rows = [_structured_row(row) for row in msg_rows]
|
|
event_rows.extend((
|
|
KIND_TOOL_MALFORMED, task_id, user_id, created_at, None, tool,
|
|
err, head, tail, None, None, None, None,
|
|
) for task_id, user_id, created_at, tool, err, head, tail in malformed_rows)
|
|
event_rows.extend((
|
|
KIND_RUN_ERROR, task_id, user_id, created_at, None, None,
|
|
err, None, None, None, None, None, None,
|
|
) for task_id, user_id, created_at, err in (run_rows or []))
|
|
event_rows.extend((
|
|
KIND_EMPTY_RESPONSE, task_id, user_id, created_at, model_profile, None,
|
|
None, None, None, None, None, None, None,
|
|
) for task_id, user_id, created_at, model_profile in (empty_rows or []))
|
|
|
|
@contextmanager
|
|
def fake_scope():
|
|
yield _FakeSession(event_rows)
|
|
|
|
with patch.object(tf, "session_scope", fake_scope):
|
|
return tf.scan_tool_failures(**kw)
|
|
|
|
|
|
class TestToolfailMalformed(unittest.TestCase):
|
|
def test_malformed_rows_cluster(self):
|
|
"""usage_events 的 tool_malformed 行聚成 kind=malformed 的 cluster。"""
|
|
err = "Expecting value: line 1 column 1 (char 0)"
|
|
rows = [
|
|
("t1", "u1", NOW - timedelta(hours=1), "write", err, "].cells[1]", '"}'),
|
|
("t2", "u1", NOW - timedelta(hours=2), "write", err, "].merge(", '"}'),
|
|
]
|
|
out = _scan([], rows, days=7, min_count=2, min_tasks=2)
|
|
self.assertEqual(len(out), 1)
|
|
c = out[0]
|
|
self.assertEqual(c["tool"], "write")
|
|
self.assertEqual(c["kind"], "malformed")
|
|
self.assertEqual(c["count"], 2)
|
|
self.assertEqual(c["count_24h"], 2)
|
|
self.assertEqual(c["task_count"], 2)
|
|
self.assertIn("…", c["sample"]) # sample = head … tail
|
|
|
|
def test_err_signature_normalized(self):
|
|
"""报错里的行号/列号数字被归一,不同位置的同类报错聚成一条。"""
|
|
rows = [
|
|
("t1", "u1", NOW, "write", "Expecting ',' delimiter: line 1 column 57 (char 56)", "h", "t"),
|
|
("t2", "u2", NOW, "write", "Expecting ',' delimiter: line 3 column 9 (char 88)", "h", "t"),
|
|
]
|
|
out = _scan([], rows, days=7, min_count=2, min_tasks=2)
|
|
self.assertEqual(len(out), 1)
|
|
|
|
def test_merges_structured_and_malformed_clusters(self):
|
|
"""同表不同 kind 并存互不干扰:普通失败与 malformed 各出一组。"""
|
|
msg_rows = [
|
|
("t1", "u1", NOW, "glob", "[Error] base path not found"),
|
|
("t2", "u2", NOW, "glob", "[Error] base path not found"),
|
|
]
|
|
mal_rows = [
|
|
("t3", "u1", NOW, "write", "Expecting value", "h", "t"),
|
|
("t4", "u2", NOW, "write", "Expecting value", "h", "t"),
|
|
]
|
|
out = _scan(msg_rows, mal_rows, days=7, min_count=2, min_tasks=2)
|
|
kinds = sorted(c["kind"] for c in out)
|
|
self.assertEqual(kinds, ["error", "malformed"])
|
|
|
|
def test_quiet_cluster_daily_bucket(self):
|
|
"""3 天前的畸形记录:count_24h=0(已安静),落在对应历史桶。"""
|
|
rows = [
|
|
("t1", "u1", NOW - timedelta(days=3, hours=1), "write", "Expecting value", "h", "t"),
|
|
("t2", "u2", NOW - timedelta(days=3, hours=2), "write", "Expecting value", "h", "t"),
|
|
]
|
|
out = _scan([], rows, days=7, min_count=2, min_tasks=2)
|
|
c = out[0]
|
|
self.assertEqual(c["count_24h"], 0)
|
|
self.assertEqual(sum(c["daily"]), 2)
|
|
self.assertEqual(c["daily"][-4], 2) # 尾桶=近24h,往前退 3 桶
|
|
|
|
def test_quality_gate_is_additively_categorized(self):
|
|
"""按设计拦截内容质量的 exit 1 单列，不冒充平台工具故障。"""
|
|
sample = (
|
|
"[质量检查] type=review\n" + ("检查明细\n" * 80)
|
|
+ "[篇幅核算] type=review lang=zh\n"
|
|
"[WARN] 0 项超出 / 1 项不足 (含摘要/正文)。回头调整。\n[exit 1]"
|
|
)
|
|
rows = [
|
|
("t1", "u1", NOW, "shell", sample),
|
|
("t1", "u1", NOW, "shell", sample),
|
|
]
|
|
out = _scan(rows, [], days=1, min_count=2, min_tasks=1)
|
|
self.assertEqual(out[0]["category"], "quality_gate")
|
|
|
|
def test_regular_failure_category_preserved(self):
|
|
rows = [
|
|
("t1", "u1", NOW, "glob", "[Error] base path not found"),
|
|
("t2", "u2", NOW, "glob", "[Error] base path not found"),
|
|
]
|
|
out = _scan(rows, [], days=1, min_count=2, min_tasks=2)
|
|
self.assertEqual(out[0]["category"], "failure")
|
|
|
|
def test_bare_shell_exit_gets_command_hint(self):
|
|
"""空输出 grep 未命中不再和其他 exit 1 混成无意义的 ``exit N``。"""
|
|
prior = {
|
|
"role": "assistant",
|
|
"tool_calls": [{
|
|
"id": "call-1",
|
|
"function": {
|
|
"name": "shell",
|
|
"arguments": '{"command":"cd /workspace/x && grep -n needle a.md"}',
|
|
},
|
|
}],
|
|
}
|
|
rows = [
|
|
("t1", "u1", NOW, "shell", "[exit 1]", "call-1", prior),
|
|
("t2", "u2", NOW, "shell", "[exit 1]", "call-1", prior),
|
|
]
|
|
out = _scan(rows, [], days=1, min_count=2, min_tasks=2)
|
|
self.assertEqual(out[0]["signature"], "exit N (search/no match)")
|
|
|
|
|
|
class TestToolfailRunError(unittest.TestCase):
|
|
def test_run_error_rows_cluster(self):
|
|
"""usage_events 的 run_error 行聚成 kind=run / tool=(run) 的 cluster。"""
|
|
err = "RateLimitError: litellm.RateLimitError: ZaiException - 余额不足或无可用资源包,请充值。"
|
|
rows = [
|
|
("t1", "u1", NOW - timedelta(hours=1), err),
|
|
("t1", "u1", NOW - timedelta(minutes=30), err),
|
|
("t2", "u2", NOW - timedelta(minutes=10), err),
|
|
]
|
|
out = _scan([], [], rows, days=7, min_count=3, min_tasks=2)
|
|
self.assertEqual(len(out), 1)
|
|
c = out[0]
|
|
self.assertEqual(c["tool"], "(run)")
|
|
self.assertEqual(c["kind"], "run")
|
|
self.assertEqual(c["count"], 3)
|
|
self.assertEqual(c["task_count"], 2)
|
|
self.assertIn("余额不足", c["sample"])
|
|
|
|
|
|
class TestToolfailEmptyResponse(unittest.TestCase):
|
|
def test_empty_response_rows_cluster(self):
|
|
"""usage_events 的 empty_response 行聚成 kind=empty / tool=(empty) 的 cluster,
|
|
sample=model_profile(看哪个网关档在吐空)。"""
|
|
rows = [
|
|
("t1", "u1", NOW - timedelta(hours=1), "unifyllm.opus48"),
|
|
("t2", "u2", NOW - timedelta(minutes=20), "unifyllm.opus48"),
|
|
]
|
|
out = _scan([], [], empty_rows=rows, days=7, min_count=2, min_tasks=2)
|
|
self.assertEqual(len(out), 1)
|
|
c = out[0]
|
|
self.assertEqual(c["tool"], "(empty)")
|
|
self.assertEqual(c["kind"], "empty")
|
|
self.assertEqual(c["count"], 2)
|
|
self.assertEqual(c["task_count"], 2)
|
|
self.assertEqual(c["sample"], "unifyllm.opus48")
|
|
|
|
def test_single_transient_below_threshold(self):
|
|
"""单次瞬态吐空(<min_count 或单 task)不触发面板 —— 只抓系统性吐空。"""
|
|
rows = [("t1", "u1", NOW, "unifyllm.opus48")]
|
|
out = _scan([], [], empty_rows=rows, days=7, min_count=2, min_tasks=2)
|
|
self.assertEqual(out, [])
|
|
|
|
|
|
class TestStructuredHealthEvents(unittest.TestCase):
|
|
def test_agent_control_is_separate_from_operational_failure(self):
|
|
rows = [(
|
|
KIND_AGENT_GUARD, f"t{i}", "u1", NOW, "deepseek_v4.pro", "edit",
|
|
None, None, None, "guard", "same_error: old_str not found",
|
|
"agent_control", "same_error, count=4",
|
|
) for i in range(2)]
|
|
|
|
@contextmanager
|
|
def fake_scope():
|
|
yield _FakeSession(rows)
|
|
|
|
with patch.object(tf, "session_scope", fake_scope):
|
|
out = tf.scan_tool_failures(days=1, min_count=2, min_tasks=2)
|
|
self.assertEqual(len(out), 1)
|
|
self.assertEqual(out[0]["category"], "agent_control")
|
|
self.assertEqual(out[0]["kind"], "guard")
|
|
|
|
|
|
class TestToolWireHealth(unittest.TestCase):
|
|
def test_aggregates_recovery_rates_and_totals(self):
|
|
rows = [
|
|
("deepseek_v4.flash", "write", 10, 2, 4, 1, NOW),
|
|
("deepseek_v4.flash", "edit", 3, 0, 0, 0, NOW - timedelta(hours=2)),
|
|
]
|
|
|
|
class FakeWireSession:
|
|
def execute(self, clause, params=None):
|
|
return SimpleNamespace(fetchall=lambda: rows)
|
|
|
|
@contextmanager
|
|
def fake_scope():
|
|
yield FakeWireSession()
|
|
|
|
with patch.object(tf, "session_scope", fake_scope):
|
|
out = tf.scan_tool_wire_health(days=7)
|
|
|
|
self.assertEqual(out["days"], 7)
|
|
self.assertEqual(out["rows"][0]["tool"], "write") # 24h 残余活跃在前
|
|
self.assertEqual(out["rows"][0]["recovery_rate"], 83.3)
|
|
self.assertEqual(out["rows"][0]["recovery_rate_24h"], 80.0)
|
|
self.assertEqual(out["total"]["salvaged"], 13)
|
|
self.assertEqual(out["total"]["malformed"], 2)
|
|
self.assertEqual(out["total"]["recovery_rate"], 86.7)
|
|
|
|
def test_empty_window_has_null_rate(self):
|
|
class FakeWireSession:
|
|
def execute(self, clause, params=None):
|
|
return SimpleNamespace(fetchall=lambda: [])
|
|
|
|
@contextmanager
|
|
def fake_scope():
|
|
yield FakeWireSession()
|
|
|
|
with patch.object(tf, "session_scope", fake_scope):
|
|
out = tf.scan_tool_wire_health(days=7)
|
|
self.assertEqual(out["rows"], [])
|
|
self.assertIsNone(out["total"]["recovery_rate"])
|
|
self.assertIsNone(out["total"]["recovery_rate_24h"])
|
|
|
|
|
|
class TestToolHealthCache(unittest.TestCase):
|
|
def setUp(self):
|
|
tf._clear_tool_health_cache()
|
|
|
|
def tearDown(self):
|
|
tf._clear_tool_health_cache()
|
|
|
|
def test_failure_scan_is_reused_within_ttl(self):
|
|
expected = [{"tool": "shell"}]
|
|
with patch.object(tf, "scan_tool_failures", return_value=expected) as scan:
|
|
first = tf.scan_tool_failures_cached(days=7, min_count=3, min_tasks=1)
|
|
second = tf.scan_tool_failures_cached(days=7, min_count=3, min_tasks=1)
|
|
|
|
self.assertIs(first, expected)
|
|
self.assertIs(second, expected)
|
|
scan.assert_called_once_with(days=7, min_count=3, min_tasks=1)
|
|
|
|
def test_wire_scan_normalizes_window_before_caching(self):
|
|
expected = {"days": 90, "rows": [], "total": {}}
|
|
with patch.object(tf, "scan_tool_wire_health", return_value=expected) as scan:
|
|
first = tf.scan_tool_wire_health_cached(days=365)
|
|
second = tf.scan_tool_wire_health_cached(days=90)
|
|
|
|
self.assertIs(first, expected)
|
|
self.assertIs(second, expected)
|
|
scan.assert_called_once_with(days=90)
|
|
|
|
|
|
class TestProviderCriticalAlert(unittest.TestCase):
|
|
def setUp(self):
|
|
tf._alerted_at.clear()
|
|
|
|
def test_pattern_hit_and_cooldown(self):
|
|
"""余额类错误命中;同签名冷却期内第二次不再发。"""
|
|
err = "RateLimitError: ZaiException - 余额不足或无可用资源包,请充值。"
|
|
sent = []
|
|
with patch("tools.send_email.smtp_configured", return_value=True), \
|
|
patch("tools.send_email.send_email_smtp",
|
|
side_effect=lambda *a, **k: sent.append(a)), \
|
|
patch.dict("os.environ", {"ZCBOT_DEVELOPER_EMAIL": "dev@x.com"}):
|
|
self.assertTrue(tf.alert_provider_critical(err, task_id="t1"))
|
|
self.assertFalse(tf.alert_provider_critical(err, task_id="t2")) # 冷却
|
|
self.assertEqual(len(sent), 1)
|
|
|
|
def test_non_critical_ignored(self):
|
|
"""普通异常不告警。"""
|
|
self.assertFalse(tf.alert_provider_critical("ValueError: bad input"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|