398 lines
14 KiB
Python
398 lines
14 KiB
Python
"""Run fixed origin.analysis@v1 acceptance cases on a dedicated Windows node."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import locale
|
|
import os
|
|
import platform
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from importlib.metadata import PackageNotFoundError, version
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import NAMESPACE_URL, uuid5
|
|
|
|
ADAPTER_DIR = Path(__file__).resolve().parent
|
|
WORKER_PATH = ADAPTER_DIR / "worker.py"
|
|
_WORKER_SPEC = importlib.util.spec_from_file_location(
|
|
"zcbot_origin_analysis_acceptance_worker", WORKER_PATH
|
|
)
|
|
if _WORKER_SPEC is None or _WORKER_SPEC.loader is None:
|
|
raise RuntimeError("ORIGIN_ANALYSIS_WORKER_IMPORT_FAILED")
|
|
worker = importlib.util.module_from_spec(_WORKER_SPEC)
|
|
_WORKER_SPEC.loader.exec_module(worker)
|
|
|
|
REPORT_SCHEMA_VERSION = 1
|
|
ORIGIN_PROCESS_PATTERN = r"^origin(?:\d+)?(?:_?\d+)?(?:64)?\.exe$"
|
|
|
|
|
|
def _artifact_id(case_name: str) -> str:
|
|
return str(uuid5(NAMESPACE_URL, f"zcbot-origin-analysis-acceptance:{case_name}"))
|
|
|
|
|
|
def _request(case_name: str, operation: str, parameters: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": 1,
|
|
"inputs": [{"key": "sample", "artifact_id": _artifact_id(case_name)}],
|
|
"operation": {
|
|
"analysis": {
|
|
"type": operation,
|
|
"title": f"Origin analysis acceptance: {operation}",
|
|
"data": {"input": "sample", "x": "x", "y": "y"},
|
|
"parameters": parameters,
|
|
}
|
|
},
|
|
"outputs": (
|
|
[{"key": "result_workbook", "type": "table", "format": "xlsx"}]
|
|
if operation == "linear_fit"
|
|
else []
|
|
),
|
|
}
|
|
|
|
|
|
def _cases() -> dict[str, dict[str, Any]]:
|
|
data_check = _request(
|
|
"data_check",
|
|
"data_check",
|
|
{"fail_on": ["missing", "non_finite", "duplicate_x", "uneven_spacing"]},
|
|
)
|
|
return {
|
|
"data_check": {
|
|
"csv": "x,y\n0,1\n1,\n1,inf\n3,4\n",
|
|
"request": data_check,
|
|
"oracle": {
|
|
"passed": False,
|
|
"failed_checks": ["duplicate_x", "missing", "non_finite", "uneven_spacing"],
|
|
},
|
|
},
|
|
"normalize": {
|
|
"csv": "x,y\n0,0\n1,2\n2,4\n",
|
|
"request": _request(
|
|
"normalize", "normalize", {"method": "reference", "reference_x": 1.5}
|
|
),
|
|
"oracle": {"divisor": 3.0, "last_value": 4 / 3},
|
|
},
|
|
"smooth": {
|
|
"csv": "x,y\n" + "\n".join(f"{x},{x*x + 2*x + 3}" for x in range(7)) + "\n",
|
|
"request": _request(
|
|
"smooth",
|
|
"smooth",
|
|
{"method": "savitzky_golay", "window": 5, "polynomial_order": 2},
|
|
),
|
|
"oracle": {"rmse": 0.0},
|
|
},
|
|
"differentiate": {
|
|
"csv": "x,y\n" + "\n".join(f"{x},{x*x}" for x in range(5)) + "\n",
|
|
"request": _request("differentiate", "differentiate", {"order": 1}),
|
|
"oracle": {"derived": [0.0, 2.0, 4.0, 6.0, 8.0]},
|
|
},
|
|
"integrate": {
|
|
"csv": "x,y\n" + "\n".join(f"{x},{2*x}" for x in range(5)) + "\n",
|
|
"request": _request("integrate", "integrate", {"from": 1.0, "to": 3.0}),
|
|
"oracle": {"integral": 8.0, "full_integral": 16.0},
|
|
},
|
|
"linear_fit": {
|
|
"csv": "x,y\n" + "\n".join(f"{x},{2.5*x - 1.25}" for x in range(1, 7)) + "\n",
|
|
"request": _request(
|
|
"linear_fit",
|
|
"linear_fit",
|
|
{"include_intercept": True, "confidence_level": 0.95},
|
|
),
|
|
"oracle": {"slope": 2.5, "intercept": -1.25, "r_squared": 1.0},
|
|
},
|
|
}
|
|
|
|
|
|
def _canonical_request(request: dict[str, Any]) -> tuple[str, str]:
|
|
encoded = json.dumps(request, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
return encoded, hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _stage(root: Path, case_name: str, case: dict[str, Any]) -> Path:
|
|
job_dir = root / case_name
|
|
request_dir = job_dir / "request"
|
|
input_dir = job_dir / "input" / "sample"
|
|
request_dir.mkdir(parents=True)
|
|
input_dir.mkdir(parents=True)
|
|
(input_dir / "sample.csv").write_text(case["csv"], encoding="utf-8", newline="")
|
|
_, digest = _canonical_request(case["request"])
|
|
record = {
|
|
"job_id": str(uuid5(NAMESPACE_URL, f"zcbot-origin-analysis-job:{case_name}")),
|
|
"lease_id": str(uuid5(NAMESPACE_URL, f"zcbot-origin-analysis-lease:{case_name}")),
|
|
"request_digest": digest,
|
|
"request": case["request"],
|
|
}
|
|
worker._atomic_json(request_dir / "request.json", record)
|
|
return job_dir
|
|
|
|
|
|
def _read_result(path: Path) -> tuple[list[str], list[list[float | str | None]]]:
|
|
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
|
rows = list(csv.reader(handle))
|
|
converted: list[list[float | str | None]] = []
|
|
for row in rows[1:]:
|
|
values: list[float | str | None] = []
|
|
for value in row:
|
|
if value == "":
|
|
values.append(None)
|
|
continue
|
|
try:
|
|
values.append(float(value))
|
|
except ValueError:
|
|
values.append(value)
|
|
converted.append(values)
|
|
return rows[0], converted
|
|
|
|
|
|
def _assert_close(actual: float, expected: float, tolerance: float = 1e-9) -> None:
|
|
if abs(float(actual) - float(expected)) > tolerance:
|
|
raise RuntimeError(f"NUMERICAL_ORACLE_MISMATCH:{actual}:{expected}")
|
|
|
|
|
|
def _validate_oracle(
|
|
case_name: str,
|
|
diagnostics: dict[str, Any],
|
|
result_headers: list[str],
|
|
result_rows: list[list[Any]],
|
|
oracle: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if case_name == "data_check":
|
|
if diagnostics["passed"] is not oracle["passed"]:
|
|
raise RuntimeError("DATA_CHECK_PASS_STATE_MISMATCH")
|
|
if diagnostics["failed_checks"] != oracle["failed_checks"]:
|
|
raise RuntimeError("DATA_CHECK_CODES_MISMATCH")
|
|
elif case_name == "normalize":
|
|
_assert_close(diagnostics["divisor"], oracle["divisor"])
|
|
_assert_close(result_rows[-1][result_headers.index("normalized_y")], oracle["last_value"])
|
|
elif case_name == "smooth":
|
|
_assert_close(diagnostics["rmse"], oracle["rmse"], 1e-10)
|
|
elif case_name == "differentiate":
|
|
values = [row[result_headers.index("derivative_order_1")] for row in result_rows]
|
|
for actual, expected in zip(values, oracle["derived"], strict=True):
|
|
_assert_close(actual, expected)
|
|
elif case_name == "integrate":
|
|
_assert_close(diagnostics["integral"], oracle["integral"])
|
|
_assert_close(diagnostics["full_integral"], oracle["full_integral"])
|
|
elif case_name == "linear_fit":
|
|
for name in ("slope", "intercept", "r_squared"):
|
|
_assert_close(diagnostics[name], oracle[name])
|
|
return {"passed": True, "oracle": oracle}
|
|
|
|
|
|
def _validate_manifest(job_dir: Path, case: dict[str, Any]) -> dict[str, Any]:
|
|
terminal = json.loads((job_dir / "terminal.json").read_text(encoding="utf-8"))
|
|
if terminal.get("status") != "succeeded":
|
|
raise RuntimeError(f"WORKER_FAILED:{terminal.get('error')}")
|
|
expected = {"project", "result_table", "diagnostics", "analysis_spec", "provenance"}
|
|
if case["request"]["outputs"]:
|
|
expected.add("result_workbook")
|
|
manifest = terminal.get("artifact_manifest") or []
|
|
by_id = {item["artifact_id"]: item for item in manifest}
|
|
if set(by_id) != expected:
|
|
raise RuntimeError(f"ARTIFACT_MANIFEST_MISMATCH:{sorted(by_id)}")
|
|
output = job_dir / "output"
|
|
paths = {artifact_id: output / item["filename"] for artifact_id, item in by_id.items()}
|
|
for artifact_id, path in paths.items():
|
|
if not path.is_file() or path.stat().st_size != by_id[artifact_id]["size_bytes"]:
|
|
raise RuntimeError(f"ARTIFACT_INVALID:{artifact_id}")
|
|
if worker._file_sha256(path) != by_id[artifact_id]["sha256"]:
|
|
raise RuntimeError(f"ARTIFACT_DIGEST_MISMATCH:{artifact_id}")
|
|
diagnostics = json.loads(paths["diagnostics"].read_text(encoding="utf-8"))
|
|
headers, rows = _read_result(paths["result_table"])
|
|
oracle = _validate_oracle(case["request"]["operation"]["analysis"]["type"], diagnostics, headers, rows, case["oracle"])
|
|
return {
|
|
"artifacts": {
|
|
key: {"size_bytes": path.stat().st_size, "sha256": worker._file_sha256(path)}
|
|
for key, path in sorted(paths.items())
|
|
},
|
|
"oracle_validation": oracle,
|
|
}
|
|
|
|
|
|
def _reopen_project(job_dir: Path) -> dict[str, Any]:
|
|
import originpro as op
|
|
|
|
project = job_dir / "output" / "analysis.opju"
|
|
op.set_show(False)
|
|
try:
|
|
if not op.open(str(project), readonly=True):
|
|
raise RuntimeError("OPJU_REOPEN_FAILED")
|
|
books = list(op.pages("w"))
|
|
sheets = [sheet for book in books for sheet in book]
|
|
names = [
|
|
str(sheet.lname or getattr(book, "lname", "") or getattr(book, "name", "") or "")
|
|
for book in books
|
|
for sheet in book
|
|
]
|
|
for required in ("RawData", "Result", "Diagnostics"):
|
|
if required not in names:
|
|
raise RuntimeError(f"OPJU_SHEET_MISSING:{required}")
|
|
return {"worksheet_count": len(sheets), "worksheet_names": names}
|
|
finally:
|
|
if op.oext:
|
|
op.exit()
|
|
|
|
|
|
def _environment_fingerprint() -> dict[str, Any]:
|
|
try:
|
|
originpro_version = version("originpro")
|
|
except PackageNotFoundError:
|
|
originpro_version = "unknown"
|
|
return {
|
|
"adapter_version": worker.ADAPTER_VERSION,
|
|
"algorithm_version": worker.ALGORITHM_VERSION,
|
|
"originpro_version": originpro_version,
|
|
"numpy_version": worker.np.__version__,
|
|
"python_version": platform.python_version(),
|
|
"platform": platform.platform(),
|
|
"locale": locale.getlocale(),
|
|
"execution_mode": "hidden",
|
|
"configured_origin_executable": bool(os.environ.get("ZCBOT_ORIGIN_EXE")),
|
|
}
|
|
|
|
|
|
def _origin_processes() -> dict[int, str]:
|
|
completed = None
|
|
for attempt in range(3):
|
|
completed = subprocess.run(
|
|
["tasklist.exe", "/fo", "csv", "/nh"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
if completed.returncode == 0:
|
|
break
|
|
if attempt < 2:
|
|
time.sleep(1)
|
|
if completed is None or completed.returncode != 0:
|
|
detail = completed.stderr[-500:] if completed is not None else "not started"
|
|
raise RuntimeError(f"TASKLIST_FAILED:{detail}")
|
|
result: dict[int, str] = {}
|
|
for row in csv.reader(completed.stdout.splitlines()):
|
|
name = row[0].casefold() if row else ""
|
|
if len(row) >= 2 and re.fullmatch(ORIGIN_PROCESS_PATTERN, name):
|
|
result[int(row[1])] = row[0]
|
|
return result
|
|
|
|
|
|
def _wait_for_origin_release(baseline: dict[int, str], timeout_seconds: int) -> None:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
while True:
|
|
remaining = {
|
|
pid: name for pid, name in _origin_processes().items() if pid not in baseline
|
|
}
|
|
if not remaining:
|
|
return
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeError(f"ORIGIN_PROCESS_REMAINS:{remaining}")
|
|
time.sleep(2)
|
|
|
|
|
|
def _run_case(
|
|
root: Path,
|
|
case_name: str,
|
|
case: dict[str, Any],
|
|
baseline: dict[int, str],
|
|
release_wait: int,
|
|
) -> dict[str, Any]:
|
|
job_dir = _stage(root, case_name, case)
|
|
started = time.monotonic()
|
|
completed = subprocess.run(
|
|
[sys.executable, str(WORKER_PATH), str(job_dir)],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
check=False,
|
|
)
|
|
if completed.returncode != 0:
|
|
raise RuntimeError(f"WORKER_PROCESS_FAILED:{completed.stderr[-500:]}")
|
|
validation = _validate_manifest(job_dir, case)
|
|
reopened = _reopen_project(job_dir)
|
|
_wait_for_origin_release(baseline, release_wait)
|
|
_, digest = _canonical_request(case["request"])
|
|
return {
|
|
"case": case_name,
|
|
"elapsed_seconds": round(time.monotonic() - started, 3),
|
|
"request_digest": digest,
|
|
"validation": validation,
|
|
"reopen": reopened,
|
|
"origin_processes_released": True,
|
|
}
|
|
|
|
|
|
def run_suite(
|
|
root: Path,
|
|
selected: list[str] | None = None,
|
|
*,
|
|
release_wait: int = 60,
|
|
) -> dict[str, Any]:
|
|
cases = _cases()
|
|
names = selected or list(cases)
|
|
unknown = sorted(set(names) - cases.keys())
|
|
if unknown:
|
|
raise ValueError(f"UNKNOWN_ACCEPTANCE_CASES:{','.join(unknown)}")
|
|
root.mkdir(parents=True, exist_ok=False)
|
|
baseline = _origin_processes()
|
|
report = {
|
|
"schema_version": REPORT_SCHEMA_VERSION,
|
|
"started_at": datetime.now(timezone.utc).isoformat(),
|
|
"environment": _environment_fingerprint(),
|
|
"selected_cases": names,
|
|
"cases": [],
|
|
"passed": False,
|
|
}
|
|
report_path = root / "acceptance-report.json"
|
|
try:
|
|
for index, name in enumerate(names, start=1):
|
|
report["cases"].append(
|
|
_run_case(root, name, cases[name], baseline, release_wait)
|
|
)
|
|
worker._atomic_json(report_path, report)
|
|
print(f"[OK] Origin analysis acceptance {index}/{len(names)}: {name}")
|
|
report["passed"] = True
|
|
return report
|
|
except Exception as exc:
|
|
report["failure"] = {
|
|
"type": type(exc).__name__,
|
|
"detail": str(exc)[:1000],
|
|
"completed_cases": len(report["cases"]),
|
|
}
|
|
raise
|
|
finally:
|
|
report["completed_at"] = datetime.now(timezone.utc).isoformat()
|
|
worker._atomic_json(report_path, report)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Run fixed Origin analysis acceptance cases.")
|
|
parser.add_argument("--work-root", type=Path, required=True)
|
|
parser.add_argument("--release-wait", type=int, default=60)
|
|
parser.add_argument(
|
|
"--case", action="append", choices=tuple(_cases()), dest="cases"
|
|
)
|
|
args = parser.parse_args()
|
|
if sys.platform != "win32":
|
|
raise RuntimeError("Origin analysis acceptance requires Windows")
|
|
if args.release_wait < 1:
|
|
raise ValueError("release-wait must be positive")
|
|
root = args.work_root.resolve()
|
|
report = run_suite(root, args.cases, release_wait=args.release_wait)
|
|
print(f"[OK] Origin analysis acceptance passed. Report: {root / 'acceptance-report.json'}")
|
|
print(f"[INFO] Cases: {len(report['cases'])}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|