"""顺序执行评测任务。

第一版刻意不并发：避免压测与质量评测混在一起，也降低误连环境时的影响面。
"""
from __future__ import annotations

import datetime as dt
import uuid
from dataclasses import dataclass
from typing import Callable, Optional

from .client import ZcbotClient
from .models import (
    CaseResult,
    EvalCase,
    EvalSuite,
    RepetitionResult,
    RunObservation,
    safe_case_slug,
)
from .scoring import score_observation


@dataclass(frozen=True)
class RunSettings:
    repetitions: Optional[int] = None
    timeout_s: float = 900.0


ProgressCallback = Callable[[str], None]


def run_suite(
    suite: EvalSuite,
    client: ZcbotClient,
    *,
    settings: RunSettings,
    progress: ProgressCallback = lambda _message: None,
) -> list[CaseResult]:
    stamp = (
        dt.datetime.now().strftime("%Y%m%d-%H%M%S")
        + "-"
        + uuid.uuid4().hex[:8]
    )
    results: list[CaseResult] = []
    for case in suite.cases:
        repetitions = (
            settings.repetitions
            if settings.repetitions is not None
            else case.repetitions or suite.default_repetitions
        )
        run_results: list[RepetitionResult] = []
        for index in range(1, repetitions + 1):
            slug = safe_case_slug(case.id)
            working_dir = f"eval-{slug}-{stamp}-{index}"
            task_name = f"eval-{slug}-{index}"
            progress(f"[INFO] {case.id} repetition {index}/{repetitions}")
            task = client.create_task(
                name=task_name,
                working_dir=working_dir,
                skill=case.skill,
                model_profile=case.model_profile,
            )
            task_id = str(task["task_id"])
            try:
                observation = client.run_prompt(
                    task_id=task_id,
                    prompt=case.prompt,
                    working_dir=working_dir,
                    timeout_s=case.timeout_s or settings.timeout_s,
                )
            except Exception as exc:
                observation = RunObservation(
                    run_status="error",
                    run_error=f"{type(exc).__name__}: {exc}",
                    task_id=task_id,
                    working_dir=working_dir,
                )
            score, assertions = score_observation(
                case.assertions, observation
            )
            run_results.append(
                RepetitionResult(
                    index=index,
                    observation=observation,
                    assertions=assertions,
                    score=score,
                )
            )
        case_score = sum(item.score for item in run_results) / len(run_results)
        passed = [
            item.score >= suite.pass_threshold for item in run_results
        ]
        results.append(
            CaseResult(
                case=case,
                repetitions=run_results,
                score=case_score,
                pass_at_1=passed[0],
                pass_all=all(passed),
            )
        )
    return results