"""paper_server 客户端 helper。base_url 默认硬编码,env 可覆盖。""" from __future__ import annotations import os from pathlib import Path from typing import Any, Optional import httpx _BASE_URL = os.environ.get("PAPER_SERVER_URL", "http://paper.xxhhcty.xyz:8080").rstrip("/") _API = f"{_BASE_URL}/api/resm/paper" _TIMEOUT = 30.0 # paper_server 的 GET 接口要 API Key(走 ?api_key= 查询参数,下载直链也好带)。 # 这是唯一刻意放进 sandbox 的凭证:run_python env 过滤器对它放行、docker 模式宿主透传; # 低价值只读 key,可随时在 paper_server Django admin 撤销。 _API_KEY = os.environ.get("PAPER_SERVER_API_KEY", "").strip() def _with_key(params: Optional[dict] = None) -> dict: """给请求 params 并上 api_key(未配置时原样返回,由服务端报 401/403)。""" merged = dict(params or {}) if _API_KEY: merged["api_key"] = _API_KEY return merged _LIST_FIELDS = ( "id", "doi", "title", "first_author", "first_author_institution", "publication_year", "publication_date", "publication_name", "has_fulltext_pdf", "has_fulltext_xml", "has_abstract", "is_oa", "type", "abstract", "pdf_url", "xml_url", ) _AUTH_ERR_CODES = {"not_authenticated", "authentication_failed", "permission_denied"} def _raise_with_auth_hint(r: httpx.Response) -> None: """认证类失败转成带指引的报错(比裸 HTTPStatusError 可读),其余走 raise_for_status。 paper_server 的 custom_exception_hander 把非 401/404 的错误统一压成 400, 所以不能只看 status code,要看 body 里的 err_code。 """ if r.status_code in (400, 401, 403): try: r.read() # stream 模式下 body 未读,json() 会抛 ResponseNotRead;非 stream 幂等 err_code = r.json().get("err_code", "") except Exception: err_code = "" if r.status_code in (401, 403) or err_code in _AUTH_ERR_CODES: raise RuntimeError( f"paper_server auth failed (HTTP {r.status_code}, {err_code or 'no err_code'}): " "PAPER_SERVER_API_KEY not set or invalid" ) r.raise_for_status() def _safe_doi(doi: str) -> str: return doi.replace("/", "_") def _is_doi(s: str) -> bool: return "/" in s and s.lstrip().startswith("10.") def _resolve_to_id(id_or_doi: str) -> str: """传 id 直接返,传 doi → 调 list 接口取 id。命中 0 / 多条都抛。""" s = id_or_doi.strip() if not _is_doi(s): return s r = httpx.get(_API + "/", params=_with_key({"doi": s}), timeout=_TIMEOUT) _raise_with_auth_hint(r) data = r.json() results = data.get("results") if isinstance(data, dict) and "results" in data else data if not results: raise ValueError(f"doi 未命中: {s}") if len(results) > 1: raise ValueError(f"doi 命中多条({len(results)}): {s}") return results[0]["id"] def search( keyword: str = "", year: Optional[int] = None, year_gte: Optional[int] = None, year_lte: Optional[int] = None, doi: str = "", first_author: str = "", publication_name: str = "", has_pdf: Optional[bool] = None, is_oa: Optional[bool] = None, limit: int = 10, ) -> list[dict]: """搜文献,返回精简列表(每条含 abstract 字段,有就是文本,没就是空串)。 keyword: paper_server SearchFilter,模糊匹配 title / first_author / first_author_institution 库里主语料是英文,**优先英文关键词**(用户中文输入要先转专业英文术语) year: 精确年份 year_gte/year_lte: 年份范围(做"近 N 年文献"用) doi: 精确 DOI(命中 0/1 条) first_author: 精确作者名 publication_name: 精确期刊名 has_pdf: True 仅返 PDF 已下好的;False 仅返没 PDF 的;None 都返 is_oa: True 仅返 OA 的;False 仅返非 OA;None 都返 limit: 默认 10,上限 50 """ if limit > 50: limit = 50 params: dict[str, Any] = {"page_size": limit} if keyword: params["search"] = keyword if year is not None: params["publication_year"] = year if year_gte is not None: params["publication_year_gte"] = year_gte if year_lte is not None: params["publication_year_lte"] = year_lte if doi: params["doi"] = doi if first_author: params["first_author"] = first_author if publication_name: params["publication_name"] = publication_name if has_pdf is True: params["has_fulltext_pdf"] = "true" elif has_pdf is False: params["has_fulltext_pdf"] = "false" if is_oa is True: params["is_oa"] = "true" elif is_oa is False: params["is_oa"] = "false" r = httpx.get(_API + "/", params=_with_key(params), timeout=_TIMEOUT) _raise_with_auth_hint(r) data = r.json() results = data.get("results") if isinstance(data, dict) and "results" in data else data return [{k: p.get(k) for k in _LIST_FIELDS} for p in results[:limit]] def get_paper(id_or_doi: str) -> dict: """取单条完整 metadata + abstract。 list 端点已带 abstract,正常工作流不需要调本函数;仅在用户给单个 id/DOI 想拿全字段时用。 """ pid = _resolve_to_id(id_or_doi) r = httpx.get(f"{_API}/{pid}/", params=_with_key(), timeout=_TIMEOUT) _raise_with_auth_hint(r) return r.json() def _stream_to(url: str, dest: Path) -> None: dest.parent.mkdir(parents=True, exist_ok=True) with httpx.stream("GET", url, params=_with_key(), timeout=60.0) as resp: _raise_with_auth_hint(resp) with open(dest, "wb") as f: for chunk in resp.iter_bytes(chunk_size=64 * 1024): f.write(chunk) def fetch_pdf(id_or_doi: str, working_dir: str) -> str: """下载 PDF 到 /papers/.pdf,返回相对路径 'papers/.pdf'。 走 paper_server media 静态直链(从 list/retrieve 返回的 pdf_url 字段),跟 fetch_xml 同范式。 paper.has_fulltext_pdf=False / pdf_url 空(publication_date 缺失时)→ 抛 RuntimeError。 已存在跳过下载直接复用。 """ paper = get_paper(id_or_doi) if not paper.get("has_fulltext_pdf"): reason = paper.get("fail_reason") or "no PDF on server" raise RuntimeError(f"paper has no PDF: id={paper.get('id')} reason={reason}") pdf_url = paper.get("pdf_url") or "" if not pdf_url: raise RuntimeError( f"paper pdf_url unavailable (likely missing publication_date): id={paper.get('id')}" ) safe = _safe_doi(paper["doi"]) rel = f"papers/{safe}.pdf" dest = Path(working_dir) / rel if dest.exists() and dest.stat().st_size > 0: return rel _stream_to(pdf_url, dest) return rel def fetch_xml(id_or_doi: str, working_dir: str) -> str: """下载 XML 到 /papers/.xml,返回相对路径 'papers/.xml'。 XML 走 paper_server media 静态直链(由 list/retrieve 返回的 xml_url 字段提供); paper_pdf_view 只覆盖 PDF,XML 没对应 API。 paper.has_fulltext_xml=False / xml_url 空 → 抛 RuntimeError。 已存在跳过下载直接复用。 """ paper = get_paper(id_or_doi) if not paper.get("has_fulltext_xml"): raise RuntimeError(f"paper has no XML: id={paper.get('id')}") xml_url = paper.get("xml_url") or "" if not xml_url: # publication_date 缺失(unknown 目录)→ paper_server 没暴露这层 media URL raise RuntimeError( f"paper xml_url unavailable (likely missing publication_date): id={paper.get('id')}" ) safe = _safe_doi(paper["doi"]) rel = f"papers/{safe}.xml" dest = Path(working_dir) / rel if dest.exists() and dest.stat().st_size > 0: return rel _stream_to(xml_url, dest) return rel