105 lines
3.2 KiB
Python
105 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
import sys
|
|
import unittest
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import core.storage.telemetry as telemetry # noqa: E402
|
|
|
|
|
|
class _CaptureSession:
|
|
def __init__(self):
|
|
self.rows = []
|
|
|
|
def add(self, row):
|
|
self.rows.append(row)
|
|
|
|
|
|
class TestHealthTelemetry(unittest.TestCase):
|
|
def setUp(self):
|
|
self.session = _CaptureSession()
|
|
|
|
@contextmanager
|
|
def fake_scope():
|
|
yield self.session
|
|
|
|
self.scope_patch = patch.object(telemetry, "session_scope", fake_scope)
|
|
self.scope_patch.start()
|
|
|
|
def tearDown(self):
|
|
self.scope_patch.stop()
|
|
|
|
def _ids(self):
|
|
return {"task_id": uuid4(), "user_id": uuid4(), "message_id": uuid4()}
|
|
|
|
def test_normal_tool_result_does_not_write(self):
|
|
ids = self._ids()
|
|
written = telemetry.record_tool_failure(
|
|
**ids, model_profile="test.fast", tool="read", content="ok",
|
|
)
|
|
self.assertFalse(written)
|
|
self.assertEqual(self.session.rows, [])
|
|
|
|
def test_tool_failure_is_structured(self):
|
|
ids = self._ids()
|
|
written = telemetry.record_tool_failure(
|
|
**ids,
|
|
model_profile="test.fast",
|
|
tool="shell",
|
|
content="[exit 1]",
|
|
arguments={"command": "rg needle docs"},
|
|
)
|
|
self.assertTrue(written)
|
|
row = self.session.rows[0]
|
|
self.assertEqual(row.kind, telemetry.KIND_TOOL_FAILURE)
|
|
self.assertEqual(row.units["signature"], "exit N (search/no match)")
|
|
self.assertEqual(row.units["category"], "failure")
|
|
|
|
def test_quality_gate_uses_own_kind(self):
|
|
ids = self._ids()
|
|
telemetry.record_tool_failure(
|
|
**ids,
|
|
model_profile="test.fast",
|
|
tool="shell",
|
|
content="[GATE FAIL] svg_quality_checker: overlap\n[exit 1]",
|
|
)
|
|
self.assertEqual(self.session.rows[0].kind, telemetry.KIND_QUALITY_GATE)
|
|
self.assertEqual(self.session.rows[0].units["category"], "quality_gate")
|
|
|
|
def test_control_events_are_structured(self):
|
|
ids = self._ids()
|
|
common = {
|
|
"task_id": ids["task_id"], "user_id": ids["user_id"],
|
|
"model_profile": "test.fast",
|
|
}
|
|
telemetry.record_run_stopped(**common, reason="max_iterations")
|
|
telemetry.record_agent_guard(
|
|
**common, tool="edit", guard="same_error", count=4,
|
|
signature="old_str not found",
|
|
)
|
|
telemetry.record_context_fold_failure(
|
|
**common, reason="empty_summary",
|
|
)
|
|
telemetry.record_quality_gate(
|
|
**common, tool="shell", gate="pptx_full_page_image",
|
|
)
|
|
self.assertEqual(
|
|
[row.kind for row in self.session.rows],
|
|
[
|
|
telemetry.KIND_RUN_STOPPED,
|
|
telemetry.KIND_AGENT_GUARD,
|
|
telemetry.KIND_CONTEXT_FOLD_FAILURE,
|
|
telemetry.KIND_QUALITY_GATE,
|
|
],
|
|
)
|
|
self.assertEqual(self.session.rows[0].units["category"], "agent_control")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|