"""仓库工程质量自动化审计。 工程评分包含测试、编译、覆盖率、静态检查和类型检查;安全评分独立包含 代码安全扫描和依赖漏洞审计。 工具未安装按未得分处理,并在报告中明确标为 unavailable。 """ from __future__ import annotations import importlib.util import os import subprocess import sys import time from dataclasses import dataclass from pathlib import Path from .models import ( AssertionResult, AssertionSpec, CaseResult, EvalCase, RepetitionResult, RunObservation, ) @dataclass(frozen=True) class AuditCheck: id: str description: str weight: float module: str arguments: tuple[str, ...] ENGINEERING_CHECKS = ( AuditCheck( "ruff", "Ruff 阻断级静态检查", 10, "ruff", ( "-m", "ruff", "check", "--select", "E9,F63,F7,F82", "core", "web", "tools", "rendering", "evaluation", ), ), AuditCheck( "mypy", "Mypy 类型检查", 10, "mypy", ("-m", "mypy", "core", "web", "tools", "rendering", "evaluation"), ), ) SECURITY_CHECKS = ( AuditCheck( "bandit", "Bandit Python 安全扫描", 10, "bandit", ("-m", "bandit", "-r", "core", "web", "tools", "-lll", "-q"), ), AuditCheck( "pip-audit", "当前虚拟环境依赖漏洞审计", 15, "pip_audit", ( "-m", "pip_audit", "--local", "--cache-dir", "evaluation/.cache/pip-audit", "--progress-spinner", "off", ), ), ) def _case_result( *, case_id: str, name: str, dimension: str, assertions: list[AssertionResult], duration_s: float, ) -> CaseResult: denominator = sum(item.weight for item in assertions) score = ( sum(item.weight for item in assertions if item.passed) / denominator if denominator else 0.0 ) case = EvalCase( id=case_id, name=name, dimension=dimension, prompt="local repository audit", assertions=tuple( AssertionSpec(item.type, weight=item.weight) for item in assertions ), ) repetition = RepetitionResult( index=1, observation=RunObservation(duration_s=duration_s, run_status="idle"), assertions=assertions, score=score, ) return CaseResult( case=case, repetitions=[repetition], score=score, pass_at_1=score >= 0.8, pass_all=score >= 0.8, ) def _tail(text: str, limit: int = 1200) -> str: compact = text.strip() return compact[-limit:] if compact else "" def _run( arguments: tuple[str, ...], *, repo_root: Path, timeout_s: float, evidence_path: Path | None = None, ) -> tuple[bool, str, float]: started = time.monotonic() try: completed = subprocess.run( [sys.executable, *arguments], cwd=repo_root, env={ **os.environ, "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8", }, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout_s, check=False, ) output = "\n".join( part for part in (completed.stdout, completed.stderr) if part ) if evidence_path is not None: evidence_path.parent.mkdir(parents=True, exist_ok=True) evidence_path.write_text(output, encoding="utf-8") return completed.returncode == 0, _tail(output), time.monotonic() - started except subprocess.TimeoutExpired as exc: output = "\n".join( str(part) for part in (exc.stdout, exc.stderr) if part ) if evidence_path is not None: evidence_path.parent.mkdir(parents=True, exist_ok=True) evidence_path.write_text(output, encoding="utf-8") return False, f"timeout after {timeout_s}s\n{_tail(output)}", time.monotonic() - started except OSError as exc: return False, f"{type(exc).__name__}: {exc}", time.monotonic() - started def audit_repository( repo_root: Path, *, timeout_s: float = 900.0, progress=lambda _message: None, write_evidence: bool = True, ) -> CaseResult: assertions: list[AssertionResult] = [] total_duration = 0.0 evidence_dir = repo_root / "evaluation" / "reports" / "evidence" def evidence(name: str) -> Path | None: return evidence_dir / f"{name}.txt" if write_evidence else None progress("[INFO] engineering: compileall") passed, detail, duration = _run( ("-m", "compileall", "-q", "core", "web", "tools", "rendering", "evaluation"), repo_root=repo_root, timeout_s=timeout_s, evidence_path=evidence("compileall"), ) total_duration += duration assertions.append( AssertionResult("compileall", passed, 5, "Python 全量编译", detail) ) # coverage 已安装时让它承载同一轮单测;否则用 unittest 跑测试,并把覆盖率项明确 # 记为 unavailable。这样不会为了两个指标重复执行完整测试。 coverage_available = importlib.util.find_spec("coverage") is not None test_args = ( ("-m", "coverage", "run", "--source=core,web,tools,rendering", "-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py") if coverage_available else ("-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py") ) progress("[INFO] engineering: unit tests") tests_passed, tests_detail, duration = _run( test_args, repo_root=repo_root, timeout_s=timeout_s, evidence_path=evidence("unit-tests"), ) total_duration += duration assertions.append( AssertionResult( "unit_tests", tests_passed, 35, "自动化测试全量通过", tests_detail ) ) if coverage_available and tests_passed: progress("[INFO] engineering: coverage") coverage_passed, coverage_detail, duration = _run( ("-m", "coverage", "report", "--fail-under=70"), repo_root=repo_root, timeout_s=timeout_s, evidence_path=evidence("coverage"), ) total_duration += duration assertions.append( AssertionResult( "coverage", coverage_passed, 15, "语句覆盖率不低于 70%", coverage_detail, ) ) else: reason = ( "测试失败,未计算覆盖率" if coverage_available else "unavailable: coverage 未安装" ) assertions.append( AssertionResult("coverage", False, 15, "语句覆盖率不低于 70%", reason) ) for check in ENGINEERING_CHECKS: progress(f"[INFO] engineering: {check.id}") if importlib.util.find_spec(check.module) is None: assertions.append( AssertionResult( check.id, False, check.weight, check.description, f"unavailable: {check.module} 未安装", ) ) continue passed, detail, duration = _run( check.arguments, repo_root=repo_root, timeout_s=timeout_s, evidence_path=evidence(check.id), ) total_duration += duration assertions.append( AssertionResult( check.id, passed, check.weight, check.description, detail ) ) return _case_result( case_id="engineering-audit", name="仓库工程自动化审计", dimension="engineering", assertions=assertions, duration_s=total_duration, ) def audit_security_repository( repo_root: Path, *, timeout_s: float = 900.0, progress=lambda _message: None, write_evidence: bool = True, ) -> CaseResult: """运行不接触生产数据的静态代码与依赖安全审计。""" assertions: list[AssertionResult] = [] total_duration = 0.0 evidence_dir = repo_root / "evaluation" / "reports" / "evidence" for check in SECURITY_CHECKS: progress(f"[INFO] security: {check.id}") if importlib.util.find_spec(check.module) is None: assertions.append( AssertionResult( check.id, False, check.weight, check.description, f"unavailable: {check.module} 未安装", ) ) continue passed, detail, duration = _run( check.arguments, repo_root=repo_root, timeout_s=timeout_s, evidence_path=( evidence_dir / f"{check.id}.txt" if write_evidence else None ), ) total_duration += duration assertions.append( AssertionResult( check.id, passed, check.weight, check.description, detail ) ) return _case_result( case_id="security-static-audit", name="代码与依赖安全审计", dimension="security", assertions=assertions, duration_s=total_duration, )