64 lines
1.9 KiB
Bash
64 lines
1.9 KiB
Bash
#!/usr/bin/env sh
|
|
set -eu
|
|
|
|
MODE="${1:-local}"
|
|
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
|
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
|
|
REPORTS="$REPO_ROOT/evaluation/reports"
|
|
|
|
if [ -x "$REPO_ROOT/.venv/bin/python" ]; then
|
|
PYTHON="$REPO_ROOT/.venv/bin/python"
|
|
elif [ -x "$REPO_ROOT/.venv/Scripts/python.exe" ]; then
|
|
PYTHON="$REPO_ROOT/.venv/Scripts/python.exe"
|
|
else
|
|
echo "[ERR] Python venv not found under $REPO_ROOT/.venv" >&2
|
|
exit 2
|
|
fi
|
|
|
|
case "$MODE" in
|
|
local|production-smoke|production-full) ;;
|
|
*)
|
|
echo "[ERR] mode must be local, production-smoke or production-full" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
if [ "$MODE" != "local" ] && [ -z "${ZCBOT_EVAL_TOKEN:-}" ]; then
|
|
echo "[ERR] production evaluation requires ZCBOT_EVAL_TOKEN" >&2
|
|
exit 2
|
|
fi
|
|
|
|
cd "$REPO_ROOT"
|
|
echo "[INFO] Running local engineering audit"
|
|
"$PYTHON" -m evaluation audit --output "$REPORTS"
|
|
|
|
if [ "$MODE" != "local" ]; then
|
|
if [ "$MODE" = "production-full" ]; then
|
|
DATASET="production_full_safe.json"
|
|
else
|
|
DATASET="production_smoke.json"
|
|
fi
|
|
ONLINE_OUTPUT="$REPORTS/latest-online"
|
|
echo "[INFO] Running safe production evaluation"
|
|
"$PYTHON" -m evaluation run \
|
|
--execute \
|
|
--allow-remote \
|
|
--config "$REPO_ROOT/evaluation/config.production-smoke.json" \
|
|
--suite "$REPO_ROOT/evaluation/datasets/$DATASET" \
|
|
--output "$ONLINE_OUTPUT" \
|
|
--timeout-s 180
|
|
set -- \
|
|
--report "$REPORTS/engineering-audit.json" \
|
|
--report "$ONLINE_OUTPUT/report.json"
|
|
else
|
|
set -- --report "$REPORTS/engineering-audit.json"
|
|
fi
|
|
|
|
echo "[INFO] Combining reports"
|
|
if [ "$MODE" = "production-full" ]; then
|
|
COMBINED_OUTPUT="$REPORTS/full"
|
|
else
|
|
COMBINED_OUTPUT="$REPORTS/latest"
|
|
fi
|
|
"$PYTHON" -m evaluation combine "$@" --output "$COMBINED_OUTPUT"
|
|
echo "[OK] report=$COMBINED_OUTPUT/report.md"
|