322 lines
9.3 KiB
Python
322 lines
9.3 KiB
Python
"""仓库工程质量自动化审计。
|
|
|
|
工程评分包含测试、编译、覆盖率、静态检查和类型检查；安全评分独立包含
|
|
代码安全扫描和依赖漏洞审计。
|
|
工具未安装按未得分处理，并在报告中明确标为 unavailable。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
from .models import (
|
|
AssertionResult,
|
|
AssertionSpec,
|
|
CaseResult,
|
|
EvalCase,
|
|
RepetitionResult,
|
|
RunObservation,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AuditCheck:
|
|
id: str
|
|
description: str
|
|
weight: float
|
|
module: str
|
|
arguments: tuple[str, ...]
|
|
|
|
|
|
ENGINEERING_CHECKS = (
|
|
AuditCheck(
|
|
"ruff",
|
|
"Ruff 阻断级静态检查",
|
|
10,
|
|
"ruff",
|
|
(
|
|
"-m", "ruff", "check",
|
|
"--select", "E9,F63,F7,F82",
|
|
"core", "web", "tools", "rendering", "evaluation",
|
|
),
|
|
),
|
|
AuditCheck(
|
|
"mypy",
|
|
"Mypy 类型检查",
|
|
10,
|
|
"mypy",
|
|
("-m", "mypy", "core", "web", "tools", "rendering", "evaluation"),
|
|
),
|
|
)
|
|
|
|
SECURITY_CHECKS = (
|
|
AuditCheck(
|
|
"bandit",
|
|
"Bandit Python 安全扫描",
|
|
10,
|
|
"bandit",
|
|
("-m", "bandit", "-r", "core", "web", "tools", "-lll", "-q"),
|
|
),
|
|
AuditCheck(
|
|
"pip-audit",
|
|
"当前虚拟环境依赖漏洞审计",
|
|
15,
|
|
"pip_audit",
|
|
(
|
|
"-m", "pip_audit", "--local",
|
|
"--cache-dir", "evaluation/.cache/pip-audit",
|
|
"--progress-spinner", "off",
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
def _case_result(
|
|
*,
|
|
case_id: str,
|
|
name: str,
|
|
dimension: str,
|
|
assertions: list[AssertionResult],
|
|
duration_s: float,
|
|
) -> CaseResult:
|
|
denominator = sum(item.weight for item in assertions)
|
|
score = (
|
|
sum(item.weight for item in assertions if item.passed) / denominator
|
|
if denominator
|
|
else 0.0
|
|
)
|
|
case = EvalCase(
|
|
id=case_id,
|
|
name=name,
|
|
dimension=dimension,
|
|
prompt="local repository audit",
|
|
assertions=tuple(
|
|
AssertionSpec(item.type, weight=item.weight) for item in assertions
|
|
),
|
|
)
|
|
repetition = RepetitionResult(
|
|
index=1,
|
|
observation=RunObservation(duration_s=duration_s, run_status="idle"),
|
|
assertions=assertions,
|
|
score=score,
|
|
)
|
|
return CaseResult(
|
|
case=case,
|
|
repetitions=[repetition],
|
|
score=score,
|
|
pass_at_1=score >= 0.8,
|
|
pass_all=score >= 0.8,
|
|
)
|
|
|
|
|
|
def _tail(text: str, limit: int = 1200) -> str:
|
|
compact = text.strip()
|
|
return compact[-limit:] if compact else ""
|
|
|
|
|
|
def _run(
|
|
arguments: tuple[str, ...],
|
|
*,
|
|
repo_root: Path,
|
|
timeout_s: float,
|
|
evidence_path: Path | None = None,
|
|
) -> tuple[bool, str, float]:
|
|
started = time.monotonic()
|
|
try:
|
|
completed = subprocess.run(
|
|
[sys.executable, *arguments],
|
|
cwd=repo_root,
|
|
env={
|
|
**os.environ,
|
|
"PYTHONUTF8": "1",
|
|
"PYTHONIOENCODING": "utf-8",
|
|
},
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=timeout_s,
|
|
check=False,
|
|
)
|
|
output = "\n".join(
|
|
part for part in (completed.stdout, completed.stderr) if part
|
|
)
|
|
if evidence_path is not None:
|
|
evidence_path.parent.mkdir(parents=True, exist_ok=True)
|
|
evidence_path.write_text(output, encoding="utf-8")
|
|
return completed.returncode == 0, _tail(output), time.monotonic() - started
|
|
except subprocess.TimeoutExpired as exc:
|
|
output = "\n".join(
|
|
str(part) for part in (exc.stdout, exc.stderr) if part
|
|
)
|
|
if evidence_path is not None:
|
|
evidence_path.parent.mkdir(parents=True, exist_ok=True)
|
|
evidence_path.write_text(output, encoding="utf-8")
|
|
return False, f"timeout after {timeout_s}s\n{_tail(output)}", time.monotonic() - started
|
|
except OSError as exc:
|
|
return False, f"{type(exc).__name__}: {exc}", time.monotonic() - started
|
|
|
|
|
|
def audit_repository(
|
|
repo_root: Path,
|
|
*,
|
|
timeout_s: float = 900.0,
|
|
progress=lambda _message: None,
|
|
write_evidence: bool = True,
|
|
) -> CaseResult:
|
|
assertions: list[AssertionResult] = []
|
|
total_duration = 0.0
|
|
evidence_dir = repo_root / "evaluation" / "reports" / "evidence"
|
|
|
|
def evidence(name: str) -> Path | None:
|
|
return evidence_dir / f"{name}.txt" if write_evidence else None
|
|
|
|
progress("[INFO] engineering: compileall")
|
|
passed, detail, duration = _run(
|
|
("-m", "compileall", "-q", "core", "web", "tools", "rendering", "evaluation"),
|
|
repo_root=repo_root,
|
|
timeout_s=timeout_s,
|
|
evidence_path=evidence("compileall"),
|
|
)
|
|
total_duration += duration
|
|
assertions.append(
|
|
AssertionResult("compileall", passed, 5, "Python 全量编译", detail)
|
|
)
|
|
|
|
# coverage 已安装时让它承载同一轮单测；否则用 unittest 跑测试，并把覆盖率项明确
|
|
# 记为 unavailable。这样不会为了两个指标重复执行完整测试。
|
|
coverage_available = importlib.util.find_spec("coverage") is not None
|
|
test_args = (
|
|
("-m", "coverage", "run", "--source=core,web,tools,rendering", "-m",
|
|
"unittest", "discover", "-s", "tests", "-p", "test_*.py")
|
|
if coverage_available
|
|
else ("-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py")
|
|
)
|
|
progress("[INFO] engineering: unit tests")
|
|
tests_passed, tests_detail, duration = _run(
|
|
test_args,
|
|
repo_root=repo_root,
|
|
timeout_s=timeout_s,
|
|
evidence_path=evidence("unit-tests"),
|
|
)
|
|
total_duration += duration
|
|
assertions.append(
|
|
AssertionResult(
|
|
"unit_tests", tests_passed, 35, "自动化测试全量通过", tests_detail
|
|
)
|
|
)
|
|
|
|
if coverage_available and tests_passed:
|
|
progress("[INFO] engineering: coverage")
|
|
coverage_passed, coverage_detail, duration = _run(
|
|
("-m", "coverage", "report", "--fail-under=70"),
|
|
repo_root=repo_root,
|
|
timeout_s=timeout_s,
|
|
evidence_path=evidence("coverage"),
|
|
)
|
|
total_duration += duration
|
|
assertions.append(
|
|
AssertionResult(
|
|
"coverage",
|
|
coverage_passed,
|
|
15,
|
|
"语句覆盖率不低于 70%",
|
|
coverage_detail,
|
|
)
|
|
)
|
|
else:
|
|
reason = (
|
|
"测试失败，未计算覆盖率"
|
|
if coverage_available
|
|
else "unavailable: coverage 未安装"
|
|
)
|
|
assertions.append(
|
|
AssertionResult("coverage", False, 15, "语句覆盖率不低于 70%", reason)
|
|
)
|
|
|
|
for check in ENGINEERING_CHECKS:
|
|
progress(f"[INFO] engineering: {check.id}")
|
|
if importlib.util.find_spec(check.module) is None:
|
|
assertions.append(
|
|
AssertionResult(
|
|
check.id,
|
|
False,
|
|
check.weight,
|
|
check.description,
|
|
f"unavailable: {check.module} 未安装",
|
|
)
|
|
)
|
|
continue
|
|
passed, detail, duration = _run(
|
|
check.arguments,
|
|
repo_root=repo_root,
|
|
timeout_s=timeout_s,
|
|
evidence_path=evidence(check.id),
|
|
)
|
|
total_duration += duration
|
|
assertions.append(
|
|
AssertionResult(
|
|
check.id, passed, check.weight, check.description, detail
|
|
)
|
|
)
|
|
|
|
return _case_result(
|
|
case_id="engineering-audit",
|
|
name="仓库工程自动化审计",
|
|
dimension="engineering",
|
|
assertions=assertions,
|
|
duration_s=total_duration,
|
|
)
|
|
|
|
|
|
def audit_security_repository(
|
|
repo_root: Path,
|
|
*,
|
|
timeout_s: float = 900.0,
|
|
progress=lambda _message: None,
|
|
write_evidence: bool = True,
|
|
) -> CaseResult:
|
|
"""运行不接触生产数据的静态代码与依赖安全审计。"""
|
|
assertions: list[AssertionResult] = []
|
|
total_duration = 0.0
|
|
evidence_dir = repo_root / "evaluation" / "reports" / "evidence"
|
|
for check in SECURITY_CHECKS:
|
|
progress(f"[INFO] security: {check.id}")
|
|
if importlib.util.find_spec(check.module) is None:
|
|
assertions.append(
|
|
AssertionResult(
|
|
check.id,
|
|
False,
|
|
check.weight,
|
|
check.description,
|
|
f"unavailable: {check.module} 未安装",
|
|
)
|
|
)
|
|
continue
|
|
passed, detail, duration = _run(
|
|
check.arguments,
|
|
repo_root=repo_root,
|
|
timeout_s=timeout_s,
|
|
evidence_path=(
|
|
evidence_dir / f"{check.id}.txt" if write_evidence else None
|
|
),
|
|
)
|
|
total_duration += duration
|
|
assertions.append(
|
|
AssertionResult(
|
|
check.id, passed, check.weight, check.description, detail
|
|
)
|
|
)
|
|
return _case_result(
|
|
case_id="security-static-audit",
|
|
name="代码与依赖安全审计",
|
|
dimension="security",
|
|
assertions=assertions,
|
|
duration_s=total_duration,
|
|
)
|