77 lines
2.1 KiB
PowerShell
77 lines
2.1 KiB
PowerShell
param(
|
|
[ValidateSet("local", "production-smoke", "production-full")]
|
|
[string]$Mode = "local"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
|
$Python = Join-Path $RepoRoot ".venv\Scripts\python.exe"
|
|
$Reports = Join-Path $RepoRoot "evaluation\reports"
|
|
|
|
if (-not (Test-Path -LiteralPath $Python)) {
|
|
Write-Error "[ERR] Python venv not found: $Python"
|
|
}
|
|
if (
|
|
$Mode -ne "local" -and
|
|
[string]::IsNullOrWhiteSpace($env:ZCBOT_EVAL_TOKEN)
|
|
) {
|
|
Write-Error "[ERR] production evaluation requires ZCBOT_EVAL_TOKEN"
|
|
}
|
|
|
|
function Invoke-Evaluation {
|
|
param([string[]]$Arguments)
|
|
& $Python -m evaluation @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "[ERR] evaluation command failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
Push-Location $RepoRoot
|
|
try {
|
|
Write-Host "[INFO] Running local engineering audit"
|
|
Invoke-Evaluation @(
|
|
"audit",
|
|
"--output", $Reports
|
|
)
|
|
|
|
$Inputs = @(
|
|
"--report",
|
|
(Join-Path $Reports "engineering-audit.json")
|
|
)
|
|
|
|
if ($Mode -ne "local") {
|
|
$Dataset = if ($Mode -eq "production-full") {
|
|
"production_full_safe.json"
|
|
} else {
|
|
"production_smoke.json"
|
|
}
|
|
$OnlineOutput = Join-Path $Reports "latest-online"
|
|
Write-Host "[INFO] Running safe production evaluation"
|
|
Invoke-Evaluation @(
|
|
"run",
|
|
"--execute",
|
|
"--allow-remote",
|
|
"--config", (Join-Path $RepoRoot "evaluation\config.production-smoke.json"),
|
|
"--suite", (Join-Path $RepoRoot "evaluation\datasets\$Dataset"),
|
|
"--output", $OnlineOutput,
|
|
"--timeout-s", "180"
|
|
)
|
|
$Inputs += @("--report", (Join-Path $OnlineOutput "report.json"))
|
|
}
|
|
|
|
$CombinedOutput = if ($Mode -eq "production-full") {
|
|
Join-Path $Reports "full"
|
|
} else {
|
|
Join-Path $Reports "latest"
|
|
}
|
|
Write-Host "[INFO] Combining reports"
|
|
Invoke-Evaluation (@("combine") + $Inputs + @(
|
|
"--output",
|
|
$CombinedOutput
|
|
))
|
|
Write-Host "[OK] report=$CombinedOutput\report.md"
|
|
}
|
|
finally {
|
|
Pop-Location
|
|
}
|