104 lines
3.0 KiB
Python
104 lines
3.0 KiB
Python
"""顺序执行评测任务。
|
|
|
|
第一版刻意不并发：避免压测与质量评测混在一起，也降低误连环境时的影响面。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from typing import Callable, Optional
|
|
|
|
from .client import ZcbotClient
|
|
from .models import (
|
|
CaseResult,
|
|
EvalCase,
|
|
EvalSuite,
|
|
RepetitionResult,
|
|
RunObservation,
|
|
safe_case_slug,
|
|
)
|
|
from .scoring import score_observation
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RunSettings:
|
|
repetitions: Optional[int] = None
|
|
timeout_s: float = 900.0
|
|
|
|
|
|
ProgressCallback = Callable[[str], None]
|
|
|
|
|
|
def run_suite(
|
|
suite: EvalSuite,
|
|
client: ZcbotClient,
|
|
*,
|
|
settings: RunSettings,
|
|
progress: ProgressCallback = lambda _message: None,
|
|
) -> list[CaseResult]:
|
|
stamp = (
|
|
dt.datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
+ "-"
|
|
+ uuid.uuid4().hex[:8]
|
|
)
|
|
results: list[CaseResult] = []
|
|
for case in suite.cases:
|
|
repetitions = (
|
|
settings.repetitions
|
|
if settings.repetitions is not None
|
|
else case.repetitions or suite.default_repetitions
|
|
)
|
|
run_results: list[RepetitionResult] = []
|
|
for index in range(1, repetitions + 1):
|
|
slug = safe_case_slug(case.id)
|
|
working_dir = f"eval-{slug}-{stamp}-{index}"
|
|
task_name = f"eval-{slug}-{index}"
|
|
progress(f"[INFO] {case.id} repetition {index}/{repetitions}")
|
|
task = client.create_task(
|
|
name=task_name,
|
|
working_dir=working_dir,
|
|
skill=case.skill,
|
|
model_profile=case.model_profile,
|
|
)
|
|
task_id = str(task["task_id"])
|
|
try:
|
|
observation = client.run_prompt(
|
|
task_id=task_id,
|
|
prompt=case.prompt,
|
|
working_dir=working_dir,
|
|
timeout_s=case.timeout_s or settings.timeout_s,
|
|
)
|
|
except Exception as exc:
|
|
observation = RunObservation(
|
|
run_status="error",
|
|
run_error=f"{type(exc).__name__}: {exc}",
|
|
task_id=task_id,
|
|
working_dir=working_dir,
|
|
)
|
|
score, assertions = score_observation(
|
|
case.assertions, observation
|
|
)
|
|
run_results.append(
|
|
RepetitionResult(
|
|
index=index,
|
|
observation=observation,
|
|
assertions=assertions,
|
|
score=score,
|
|
)
|
|
)
|
|
case_score = sum(item.score for item in run_results) / len(run_results)
|
|
passed = [
|
|
item.score >= suite.pass_threshold for item in run_results
|
|
]
|
|
results.append(
|
|
CaseResult(
|
|
case=case,
|
|
repetitions=run_results,
|
|
score=case_score,
|
|
pass_at_1=passed[0],
|
|
pass_all=all(passed),
|
|
)
|
|
)
|
|
return results
|