diff --git a/evaluation/audit.py b/evaluation/audit.py index 4fe84ac..ccf48e5 100644 --- a/evaluation/audit.py +++ b/evaluation/audit.py @@ -31,6 +31,7 @@ class AuditCheck: weight: float module: str arguments: tuple[str, ...] + direct_network: bool = False ENGINEERING_CHECKS = ( @@ -69,12 +70,21 @@ SECURITY_CHECKS = ( "pip_audit", ( "-m", "pip_audit", "--local", - "--cache-dir", "evaluation/.cache/pip-audit", "--progress-spinner", "off", ), + direct_network=True, ), ) +_PROXY_ENV_KEYS = ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", +) + def _case_result( *, @@ -119,23 +129,32 @@ def _tail(text: str, limit: int = 1200) -> str: return compact[-limit:] if compact else "" +def _subprocess_env(*, direct_network: bool) -> dict[str, str]: + env = { + **os.environ, + "PYTHONUTF8": "1", + "PYTHONIOENCODING": "utf-8", + } + if direct_network: + for key in _PROXY_ENV_KEYS: + env.pop(key, None) + return env + + def _run( arguments: tuple[str, ...], *, repo_root: Path, timeout_s: float, evidence_path: Path | None = None, + direct_network: bool = False, ) -> tuple[bool, str, float]: started = time.monotonic() try: completed = subprocess.run( [sys.executable, *arguments], cwd=repo_root, - env={ - **os.environ, - "PYTHONUTF8": "1", - "PYTHONIOENCODING": "utf-8", - }, + env=_subprocess_env(direct_network=direct_network), capture_output=True, text=True, encoding="utf-8", @@ -305,6 +324,7 @@ def audit_security_repository( evidence_path=( evidence_dir / f"{check.id}.txt" if write_evidence else None ), + direct_network=check.direct_network, ) total_duration += duration assertions.append( diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index a6381b6..5dee6c2 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -6,9 +6,14 @@ import unittest import zipfile from io import BytesIO from pathlib import Path +from unittest.mock import patch +from evaluation.audit import ( + _subprocess_env, + audit_repository, + audit_security_repository, +) from evaluation.client import EvalClientError, require_safe_base_url -from evaluation.audit import audit_repository, audit_security_repository from evaluation.combine import combine_reports from evaluation.inspect_bridge import export_inspect_jsonl from evaluation.models import AssertionSpec, EvalConfigError, RunObservation @@ -135,6 +140,33 @@ class ClientSafetyTests(unittest.TestCase): class ReportTests(unittest.TestCase): + def test_direct_audit_environment_ignores_proxy(self): + for key in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ): + with self.subTest(key=key), patch.dict( + "evaluation.audit.os.environ", + {key: "http://proxy.example"}, + clear=True, + ): + direct_env = _subprocess_env(direct_network=True) + inherited_env = _subprocess_env(direct_network=False) + + direct_keys = {item.casefold() for item in direct_env} + inherited_by_key = { + item.casefold(): value for item, value in inherited_env.items() + } + self.assertNotIn(key.casefold(), direct_keys) + self.assertEqual( + inherited_by_key[key.casefold()], "http://proxy.example" + ) + self.assertEqual(direct_env["PYTHONUTF8"], "1") + def test_combine_existing_dimension_reports(self): engineering = { "suite": {"name": "engineering", "version": "1"},