diff --git a/PROGRESS.md b/PROGRESS.md index d85cd14..7b1d030 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -23,6 +23,7 @@ ### 2026-07 +- **07-06 / 0.39.2 research 对接 paper_server API Key**:paper_server GET 接口收紧要 Key 后,`skills/research/paper.py` 全部请求带 `?api_key=`(env `PAPER_SERVER_API_KEY`),认证失败(paper_server 异常处理器压成 HTTP 400 + err_code)转成带指引的 RuntimeError;这把低价值只读 key 是唯一刻意放进 sandbox 的凭证——`run_python` env 过滤器加 allowlist 放行、docker 模式 `_sandbox_env()` 透传进容器(顺带修掉 `PAPER_SERVER_URL` 覆盖不进 docker 的缺口)。 - **07-06 / 0.39.1 文档精简**:PROGRESS(865→188 行)与 DESIGN(797→218 行)按各自边界契约压缩——PROGRESS 每条收成一行(做了啥+关键判断,细节查 git log),DESIGN 保留全部决策/取舍/硬协议清单/升级信号、砍实现复述(协议细节指向代码与 probe 脚本),顺手把 §2 架构树补齐实际模块。 - **07-06 / 0.39.0 skill 定向模型**:内置 skill 可在 frontmatter 声明 `model:`(ppt→GLM-5.2),对话中 `load_skill` 命中即 run 内热切(持久化 + emit `model_switch` + 前端分隔线,失败降级原模型);只信内置 skill、跳档位门控、不自动切回,"开关"=frontmatter 那一行(admin 总开关 + 预切方案经架构评审砍掉——第二事实源/第二执行点)。详 DESIGN §3.5;新增 `tests/test_skill_model_pinning.py`。 - **07-03 / 0.38.8**:列表状态灯(终态徽章+运行圆点)挪到文件夹行行首,底部数据行 space-between 均匀分布。 diff --git a/RUN.md b/RUN.md index 7c3f778..c974ade 100644 --- a/RUN.md +++ b/RUN.md @@ -31,6 +31,13 @@ # 离线分析(CIF / POSCAR + SpacegroupAnalyzer + XRDCalculator + CEMENT_PHASES)仍在 sandbox 跑。 # 申请 https://materialsproject.org/api(免费) MP_API_KEY=... + # research skill 的 paper_server 文献库:GET 接口要 API Key(paper_server Django admin + # 里建 ApiKey,helper 以 ?api_key= 查询参数带上)。research 在 sandbox 里直连 paper_server, + # 所以这把 key 会透传进 sandbox / 容器(唯一例外;低价值只读,可随时在 admin 撤销)。 + # 未设:search/fetch 即报 "paper_server auth failed (HTTP 400, not_authenticated)"。 + PAPER_SERVER_API_KEY=... + # 可选:覆盖 paper_server 地址(host 与 docker 模式均生效) + # PAPER_SERVER_URL=http://paper.xxhhcty.xyz:8080 # 本地 / 内网部署 LLM(`config/models/local.yaml`,DeepSeek-R1 满血 / QwQ-32B 原生 32K, # 共享同一台推理服务 http://182.54.21.126:9000/v1)。涉密任务用户显式选 `local.r1` / `local.qwq` # 代替默认 deepseek_v4.flash;未设 env 时这两条 variant 调用即抛 RuntimeError(其他模型不影响) @@ -718,6 +725,7 @@ sudo xfs_quota -x -c "limit -p bhard=10g zcbot_" /opt | `ZCBOT_DB_URL is not set` | `.env` 没写 / litellm 链路没触发。直跑脚本时 `import litellm` 或 `export ZCBOT_DB_URL=...` | | `ModuleNotFoundError: litellm` | 用了全局 `python`,改 `.venv/Scripts/python.exe ...` | | Windows 控制台 emoji 崩 | Python stdout 是 GBK。用 `[OK]` / `[ng]` 等 ASCII 标签(见 memory) | +| research 报 `paper_server auth failed (HTTP 400, not_authenticated)` | paper_server GET 接口已要求 API Key。`.env` 配 `PAPER_SERVER_API_KEY=`(paper_server Django admin 里建 ApiKey);docker 模式会自动透传进容器,改完重启 web 进程生效 | | `db upgrade` 报 `column already exists` | DB 已被改过,`db current` 确认 revision,必要时手 ALTER 或 `db downgrade base` 重来 | | Resume 找不到 task | dev SPA 左侧 task 列表看 task_id 是否在;或 `curl /v1/tasks` 拉 | | task 删了文件还在 | 现在 `DELETE /v1/tasks/{id}` 是**软删**,本就不动任何磁盘文件(留作语料 + 可恢复);要清磁盘走 `POST /v1/files/delete`。彻底物理删 task(及 messages)留给将来的管理员清理工具;当前如需手动:`psql> DELETE FROM tasks WHERE task_id=...`(messages/usage_events CASCADE) | diff --git a/core/__init__.py b/core/__init__.py index 4570096..a8f1092 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,3 +1,3 @@ # zcbot 版本号单一事实源:web/app.py 的 FastAPI version、/healthz 返回、前端展示都引这里。 # 改版本只动这一行。 -__version__ = "0.39.1" +__version__ = "0.39.2" diff --git a/core/executor_docker.py b/core/executor_docker.py index 6e9af16..4d3862a 100644 --- a/core/executor_docker.py +++ b/core/executor_docker.py @@ -86,6 +86,24 @@ _CONTAINER_ENV = { "HOME": "/tmp", } +# research skill 在容器里直连 paper_server,这两个 env 按需透传(host .env → docker exec -e)。 +# PAPER_SERVER_API_KEY 是唯一刻意放进 sandbox 的凭证(低价值只读文献库 key,可随时在 +# paper_server admin 撤销),与「Bocha/ARK 等高价值 key 不入容器」的规矩不冲突; +# PAPER_SERVER_URL 顺带透传,修掉 host 覆盖 URL 时 docker 模式仍用硬编码默认值的缺口。 +_PASSTHROUGH_ENV = ("PAPER_SERVER_URL", "PAPER_SERVER_API_KEY") + + +def _sandbox_env(extra: Optional[Dict[str, str]] = None) -> Dict[str, str]: + """shell / run_python 的容器 env:静态 _CONTAINER_ENV + host 透传项 + 调用点补充。""" + env = dict(_CONTAINER_ENV) + for k in _PASSTHROUGH_ENV: + v = os.environ.get(k, "").strip() + if v: + env[k] = v + if extra: + env.update(extra) + return env + # host 侧 tmp 脚本目录(user_root 内 dotfile,被 /v1/files API 隐藏) TMP_SUBDIR = ".zcbot_tmp" @@ -165,7 +183,7 @@ class DockerExecutor(Executor): # PGID kill 协议(§7.5 Step 3b)未来要做时换 `setsid --wait` 或 wrapper, # 不能裸 setsid。 argv = self._docker_exec_argv( - container, extra_env=_CONTAINER_ENV + container, extra_env=_sandbox_env() ) + ["bash", "-c", cmd] result = self._run_subprocess(argv, timeout=timeout, ctx=ctx) self.pool.mark_active(self.user_id) @@ -187,7 +205,7 @@ class DockerExecutor(Executor): container_script = self._container_script_path(script_path) argv = self._docker_exec_argv( container, - extra_env={**_CONTAINER_ENV, "PYTHONIOENCODING": "utf-8"}, + extra_env=_sandbox_env({"PYTHONIOENCODING": "utf-8"}), ) + ["python", container_script] result = self._run_subprocess(argv, timeout=timeout, ctx=ctx) self.pool.mark_active(self.user_id) @@ -213,7 +231,7 @@ class DockerExecutor(Executor): container = self.pool.ensure(self.user_id) argv = self._docker_exec_argv( container, - extra_env={**_CONTAINER_ENV, "PYTHONIOENCODING": "utf-8"}, + extra_env=_sandbox_env({"PYTHONIOENCODING": "utf-8"}), ) + ["python", container_script] # 删 setsid 同上(_exec_shell 注释) result = self._run_subprocess(argv, timeout=timeout, ctx=ctx) self.pool.mark_active(self.user_id) diff --git a/scripts/smoke_paper_skill.py b/scripts/smoke_paper_skill.py index 72b6d0a..856874c 100644 --- a/scripts/smoke_paper_skill.py +++ b/scripts/smoke_paper_skill.py @@ -23,7 +23,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) -# 读 .env 注入(PAPER_SERVER_URL 可在这里覆盖,默认 http://paper.xxhhcty.xyz:8080) +# 读 .env 注入(PAPER_SERVER_API_KEY 必配,否则 401;PAPER_SERVER_URL 可选覆盖) env_file = ROOT / ".env" if env_file.exists(): for line in env_file.read_text(encoding="utf-8").splitlines(): @@ -70,6 +70,7 @@ def step0_trgm_speed() -> None: def step1_search() -> list[dict]: _hr("step 1: search(keyword='cement', has_pdf=True, limit=5)") print(f"PAPER_SERVER_URL = {_BASE_URL}") + print(f"PAPER_SERVER_API_KEY = {'set' if os.environ.get('PAPER_SERVER_API_KEY') else 'NOT SET (expect 401)'}") print("note: 带 has_pdf=True 走 has_fulltext_pdf 索引先过滤,纯 keyword 大词会 ILIKE 全表扫导致 30s timeout (paper_server 既有问题)") t0 = time.time() try: diff --git a/skills/research/SKILL.md b/skills/research/SKILL.md index 4893381..3dc53f0 100644 --- a/skills/research/SKILL.md +++ b/skills/research/SKILL.md @@ -27,7 +27,7 @@ from skills.research.paper import search, get_paper, fetch_pdf, fetch_xml (import 路径由 `run_python` 注入的 `PYTHONPATH` 提供,直接写就行,不必折腾 `sys.path`) -> **本 skill 不持 secret**。`PAPER_SERVER_URL`(可选,默认 `http://paper.xxhhcty.xyz:8080`)是 URL 不是 key,sandbox 的 `_SENSITIVE_PATTERNS` env 过滤器不碰它 → host / docker 任何模式都能用。**别因 documents / pymatgen 报"key 没配"就连带放弃 research**,这俩不可用时它就是降级首选。 +> **本 skill 持一把低价值 key**。paper_server 的 GET 接口要 `PAPER_SERVER_API_KEY`(helper 自动以 `?api_key=` 查询参数带上,你不用管);它是唯一刻意放进 sandbox 的凭证 —— `run_python` 的 env 过滤器对它放行,docker 模式由宿主透传进容器 → host / docker 任何模式都能用。`PAPER_SERVER_URL` 可选覆盖(默认 `http://paper.xxhhcty.xyz:8080`)。**别因 documents / pymatgen 报"key 没配"就连带放弃 research**,这俩不可用时它就是降级首选。 ## 关键:keyword 优先用英文 @@ -118,6 +118,7 @@ rel = fetch_pdf("10.1016/j.cemconres.2020.106156", working_dir="") # ## 错误处理 - 网络超时 / paper_server 不可达:`httpx.ConnectError` / `httpx.TimeoutException` —— 告诉用户"paper_server 暂时连不上",不要重试堆栈刷屏 +- `paper_server auth failed (HTTP 400/401/403, not_authenticated/...)`:`PAPER_SERVER_API_KEY` 未配置或无效 —— 告诉用户找管理员在 `.env` 配 key,换 keyword 重试没用 - `doi 未命中` / `doi 命中多条`:`get_paper` / `fetch_pdf` / `fetch_xml` 内部 `_resolve_to_id` 抛 `ValueError` —— DOI 拼写错或库里没收录,改 keyword 重搜 - `has_fulltext_pdf=False` / `has_fulltext_xml=False`:`fetch_pdf` / `fetch_xml` 抛 `RuntimeError` —— 服务器还没下到对应格式;若另一格式存在则换用,都没就只能用 abstract - `xml_url` 空(publication_date 缺失,paper 落到 unknown 目录):`fetch_xml` 抛 `RuntimeError(xml_url unavailable...)` —— 改试 `fetch_pdf` diff --git a/skills/research/paper.py b/skills/research/paper.py index 80affae..48bd812 100644 --- a/skills/research/paper.py +++ b/skills/research/paper.py @@ -10,6 +10,18 @@ import httpx _BASE_URL = os.environ.get("PAPER_SERVER_URL", "http://paper.xxhhcty.xyz:8080").rstrip("/") _API = f"{_BASE_URL}/api/resm/paper" _TIMEOUT = 30.0 +# paper_server 的 GET 接口要 API Key(走 ?api_key= 查询参数,下载直链也好带)。 +# 这是唯一刻意放进 sandbox 的凭证:run_python env 过滤器对它放行、docker 模式宿主透传; +# 低价值只读 key,可随时在 paper_server Django admin 撤销。 +_API_KEY = os.environ.get("PAPER_SERVER_API_KEY", "").strip() + + +def _with_key(params: Optional[dict] = None) -> dict: + """给请求 params 并上 api_key(未配置时原样返回,由服务端报 401/403)。""" + merged = dict(params or {}) + if _API_KEY: + merged["api_key"] = _API_KEY + return merged _LIST_FIELDS = ( "id", @@ -31,6 +43,29 @@ _LIST_FIELDS = ( ) +_AUTH_ERR_CODES = {"not_authenticated", "authentication_failed", "permission_denied"} + + +def _raise_with_auth_hint(r: httpx.Response) -> None: + """认证类失败转成带指引的报错(比裸 HTTPStatusError 可读),其余走 raise_for_status。 + + paper_server 的 custom_exception_hander 把非 401/404 的错误统一压成 400, + 所以不能只看 status code,要看 body 里的 err_code。 + """ + if r.status_code in (400, 401, 403): + try: + r.read() # stream 模式下 body 未读,json() 会抛 ResponseNotRead;非 stream 幂等 + err_code = r.json().get("err_code", "") + except Exception: + err_code = "" + if r.status_code in (401, 403) or err_code in _AUTH_ERR_CODES: + raise RuntimeError( + f"paper_server auth failed (HTTP {r.status_code}, {err_code or 'no err_code'}): " + "PAPER_SERVER_API_KEY not set or invalid" + ) + r.raise_for_status() + + def _safe_doi(doi: str) -> str: return doi.replace("/", "_") @@ -44,8 +79,8 @@ def _resolve_to_id(id_or_doi: str) -> str: s = id_or_doi.strip() if not _is_doi(s): return s - r = httpx.get(_API + "/", params={"doi": s}, timeout=_TIMEOUT) - r.raise_for_status() + r = httpx.get(_API + "/", params=_with_key({"doi": s}), timeout=_TIMEOUT) + _raise_with_auth_hint(r) data = r.json() results = data.get("results") if isinstance(data, dict) and "results" in data else data if not results: @@ -105,8 +140,8 @@ def search( params["is_oa"] = "true" elif is_oa is False: params["is_oa"] = "false" - r = httpx.get(_API + "/", params=params, timeout=_TIMEOUT) - r.raise_for_status() + r = httpx.get(_API + "/", params=_with_key(params), timeout=_TIMEOUT) + _raise_with_auth_hint(r) data = r.json() results = data.get("results") if isinstance(data, dict) and "results" in data else data return [{k: p.get(k) for k in _LIST_FIELDS} for p in results[:limit]] @@ -118,15 +153,15 @@ def get_paper(id_or_doi: str) -> dict: list 端点已带 abstract,正常工作流不需要调本函数;仅在用户给单个 id/DOI 想拿全字段时用。 """ pid = _resolve_to_id(id_or_doi) - r = httpx.get(f"{_API}/{pid}/", timeout=_TIMEOUT) - r.raise_for_status() + r = httpx.get(f"{_API}/{pid}/", params=_with_key(), timeout=_TIMEOUT) + _raise_with_auth_hint(r) return r.json() def _stream_to(url: str, dest: Path) -> None: dest.parent.mkdir(parents=True, exist_ok=True) - with httpx.stream("GET", url, timeout=60.0) as resp: - resp.raise_for_status() + with httpx.stream("GET", url, params=_with_key(), timeout=60.0) as resp: + _raise_with_auth_hint(resp) with open(dest, "wb") as f: for chunk in resp.iter_bytes(chunk_size=64 * 1024): f.write(chunk) diff --git a/tools/run_python.py b/tools/run_python.py index cddbfdb..32c3e4c 100644 --- a/tools/run_python.py +++ b/tools/run_python.py @@ -17,6 +17,9 @@ from pathlib import Path from .base import Tool, compact_tool_output _SENSITIVE_PATTERNS = ("API_KEY", "TOKEN", "SECRET", "PASSWORD", "PRIVATE_KEY") +# 刻意放行的例外:research skill 在 sandbox 里直连 paper_server,这把只读文献库 key +# 的消费方就是沙盒代码本身;低价值、可随时在 paper_server admin 撤销。 +_ENV_ALLOWLIST = frozenset({"PAPER_SERVER_API_KEY"}) class RunPythonTool(Tool): @@ -75,7 +78,7 @@ class RunPythonTool(Tool): env = os.environ.copy() for k in list(env): u = k.upper() - if any(p in u for p in _SENSITIVE_PATTERNS): + if u not in _ENV_ALLOWLIST and any(p in u for p in _SENSITIVE_PATTERNS): del env[k] env["PYTHONIOENCODING"] = "utf-8" env["PYTHONPATH"] = str(self.base_dir) + os.pathsep + env.get("PYTHONPATH", "")