perf(resm): scihub改用curl-cffi轻量解析, 移除oa_url内联Playwright回退

download_pdf 链两处都不再在 worker 里起浏览器(此前是并发扩容的内存瓶颈):

- Sci-Hub: 新增 d_scihub.download_by_doi_http, 用 curl-cffi 取页面解析内嵌
  PDF 地址直接下载并校验魔数, 覆盖各域名 embed#pdf/iframe#pdf/location.href
  结构; 缓存存活域名做秒级故障切换; 撞 Cloudflare/DDoS 挑战返回
  scihub_need_play 交独立脚本兜底。浏览器版 download_paper_by_doi 保留。
- oa_url: 移除策略3内联 Playwright(还会开有头窗口), curl-cffi 失败即打
  oa_url_need_play, 交 scripts/get_pdf_by_playwright.py 兜底。
  run_async 保留(该脚本从 tasks 导入)。

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/resm/d_scihub.py

@@ -1,8 +1,11 @@
 import argparse
 import asyncio
 import logging
+import os
+import re
 from pathlib import Path
 from typing import Optional
+from urllib.parse import urljoin
 from playwright.async_api import async_playwright, Page, Browser
 
 # 尝试导入 playwright-stealth
@@ -289,6 +292,146 @@ _SCIHUB_DOMAINS = [
     "sci-hub.se",
 ]
 
+# bot 挑战页标题/正文关键词: 命中则需浏览器兜底
+_CHALLENGE_MARKERS = ("just a moment", "cloudflare", "checking your browser",
+                      "ddos-guard", "ddos guard", "attention required")
+
+# Sci-Hub 缓存当前存活域名, 避免每篇都从头探测已挂的域名
+_ALIVE_DOMAIN_KEY = "scihub_alive_domain"
+_ALIVE_DOMAIN_TTL = 600  # 秒
+
+
+def _normalize_pdf_url(raw: str, base_url: str) -> str:
+    """把 Sci-Hub 页面里抓到的 PDF 地址归一成可直接请求的绝对 URL。
+    处理: JSON 转义 \\/ 、#fragment、协议相对 // 、站内相对路径。"""
+    raw = raw.strip().replace("\\/", "/").split("#")[0]
+    if raw.startswith("//"):
+        return "https:" + raw
+    if raw.startswith("http"):
+        return raw
+    return urljoin(base_url, raw)
+
+
+def _extract_scihub_pdf_url(html: str, base_url: str) -> Optional[str]:
+    """从 Sci-Hub 页面 HTML 提取内嵌 PDF 地址, 覆盖各域名/版本的常见结构;
+    找不到返回 None。lxml 优先(结构化), 正则兜底(应对畸形 HTML/JS 赋值)。"""
+    try:
+        from lxml import etree
+        root = etree.HTML(html)
+        if root is not None:
+            for xp in ('//embed[@id="pdf"]/@src',
+                       '//iframe[@id="pdf"]/@src',
+                       '//div[@id="article"]//embed/@src',
+                       '//div[@id="article"]//iframe/@src',
+                       '//embed[@type="application/pdf"]/@src'):
+                hit = root.xpath(xp)
+                if hit and hit[0].strip():
+                    return _normalize_pdf_url(hit[0], base_url)
+    except Exception:
+        pass
+    # 正则兜底: 按钮 location.href='...pdf...' / 任意 src|href 指向 .pdf
+    for pat in (r"location\.href\s*=\s*['\"]([^'\"]+?\.pdf[^'\"]*)['\"]",
+                r"(?:src|href)\s*=\s*['\"]([^'\"]*?/downloads/[^'\"]+?\.pdf[^'\"]*)['\"]",
+                r"(?:src|href)\s*=\s*['\"]([^'\"]*?\.pdf(?:\?[^'\"]*)?)['\"]"):
+        m = re.search(pat, html, re.I)
+        if m and m.group(1).strip():
+            return _normalize_pdf_url(m.group(1), base_url)
+    return None
+
+
+def _save_pdf_bytes(content: bytes, output: str) -> bool:
+    """校验 PDF 魔数 + 最小体积后落盘; 不合格返回 False。"""
+    if not content or content[:4] != b"%PDF" or len(content) < 10240:
+        return False
+    os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True)
+    with open(output, "wb") as f:
+        f.write(content)
+    return True
+
+
+def download_by_doi_http(doi: str, output: str, timeout: int = 30) -> tuple[bool, str]:
+    """curl-cffi 轻量下载(worker 内首选, 不起浏览器): 逐域名取 Sci-Hub 页面,
+    解析内嵌 PDF 地址后直接下载并校验魔数。绝大多数无挑战的情况几秒内完成。
+
+    返回 (ok, err)。err 前缀:
+      - scihub_error_empty_doi:      DOI 为空
+      - scihub_curl_cffi_not_installed
+      - scihub_need_play:            命中 bot 挑战, 需浏览器兜底(交独立脚本)
+      - scihub_error_pdf_not_found:  页面正常但无 PDF / 所有域名 HTTP 均失败
+    """
+    doi = (doi or "").strip()
+    if not doi:
+        return False, "scihub_error_empty_doi"
+    try:
+        import curl_cffi.requests as cf
+    except ImportError:
+        return False, "scihub_curl_cffi_not_installed"
+
+    try:
+        from django.core.cache import cache
+    except Exception:
+        cache = None
+
+    # 存活域名优先, 其余按默认顺序补上(去重)
+    domains = list(_SCIHUB_DOMAINS)
+    if cache is not None:
+        alive = cache.get(_ALIVE_DOMAIN_KEY)
+        if alive in domains:
+            domains = [alive] + [d for d in domains if d != alive]
+
+    challenged = False
+    for domain in domains:
+        page_url = f"https://{domain}/{doi}"
+        try:
+            resp = cf.get(page_url, impersonate="chrome131",
+                          headers={"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"},
+                          timeout=timeout, allow_redirects=True)
+        except Exception as e:
+            logger.info(f"{domain} 请求失败: {e}")
+            continue
+
+        ct = resp.headers.get("content-type", "")
+        # 有些域名直接把 PDF 返回在 DOI 页
+        if resp.content[:4] == b"%PDF" or "application/pdf" in ct:
+            if _save_pdf_bytes(resp.content, output):
+                if cache is not None:
+                    cache.set(_ALIVE_DOMAIN_KEY, domain, timeout=_ALIVE_DOMAIN_TTL)
+                logger.info(f"✓ HTTP 直取 PDF 成功({domain}): {output}")
+                return True, ""
+
+        if resp.status_code != 200:
+            if resp.status_code in (403, 503):  # 多为挑战/封禁
+                challenged = True
+            continue
+
+        try:
+            text = resp.text
+        except Exception:
+            continue
+        if any(k in text[:3000].lower() for k in _CHALLENGE_MARKERS):
+            challenged = True
+            continue
+
+        pdf_url = _extract_scihub_pdf_url(text, page_url)
+        if not pdf_url:
+            continue
+        try:
+            pr = cf.get(pdf_url, impersonate="chrome131",
+                        headers={"Referer": page_url, "Accept": "application/pdf,*/*"},
+                        timeout=timeout, allow_redirects=True)
+        except Exception as e:
+            logger.info(f"{domain} PDF 下载失败: {e}")
+            continue
+        if pr.status_code == 200 and _save_pdf_bytes(pr.content, output):
+            if cache is not None:
+                cache.set(_ALIVE_DOMAIN_KEY, domain, timeout=_ALIVE_DOMAIN_TTL)
+            logger.info(f"✓ HTTP 解析下载成功({domain}): {output}")
+            return True, ""
+
+    if challenged:
+        return False, "scihub_need_play: bot 挑战, 需浏览器兜底"
+    return False, "scihub_error_pdf_not_found: 所有域名 HTTP 均未获取 PDF"
+
 
 def download_paper_by_doi(doi: str, output: Optional[str] = None, headless: bool = True) -> tuple[bool, str]:
     """

apps/resm/tasks.py

@@ -12,7 +12,6 @@ from celery import current_app
 from datetime import datetime, timedelta
 import random
 from .pdf_utils import _is_elsevier_preview_pdf
-from .d_oaurl import download_from_url_playwright
 from uuid import uuid4
 import asyncio
 import sys
@@ -833,7 +832,7 @@ def download_pdf(paper_id):
 
 
 def save_pdf_from_oa_url(paper: Paper):
-    from .d_oaurl import download_pdf_with_curl_cffi, download_from_url_playwright
+    from .d_oaurl import download_pdf_with_curl_cffi
 
     # 策略1: 直接请求
     try:
@@ -866,16 +865,10 @@ def save_pdf_from_oa_url(paper: Paper):
         paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "update_time"])
         return "success"
 
-    # 策略3: Playwright（最终回退）
+    # 内联 Playwright 回退已移除(在 worker 里起浏览器太重, 且会开有头窗口)。
-    is_ok, err_msg = run_async(download_from_url_playwright(paper.oa_url, paper_path))
+    # 打 oa_url_need_play 标记, 交由独立脚本 scripts/get_pdf_by_playwright.py 兜底。
-    if is_ok:
+    paper.save_fail_reason(f"oa_url_need_play: {err_msg}")
-        paper.has_fulltext = True
+    return f"oa_url_need_play: {err_msg}"
-        paper.has_fulltext_pdf = True
-        paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "update_time"])
-        return "success"
-
-    paper.save_fail_reason(f"oa_url_all_methods_failed: {err_msg}")
-    return f"oa_url_all_methods_failed: {err_msg}"
 
 def save_pdf_from_openalex(paper:Paper):
     if cache.get("openalex_api_exceed"):
@@ -937,8 +930,10 @@ def save_pdf_from_elsevier(paper:Paper):
         return f"elsevier_status_error: {res.status_code} {res.text}"
 
 def save_pdf_from_scihub(paper:Paper):
-    from .d_scihub import download_paper_by_doi
+    # worker 内只走 curl-cffi 轻量解析, 不起浏览器; 撞挑战页返回 scihub_need_play,
-    is_ok, err_msg = download_paper_by_doi(paper.doi, paper.init_paper_path("pdf"))
+    # 交由独立脚本用浏览器兜底(同 oa_url need_play 的分工)。
+    from .d_scihub import download_by_doi_http
+    is_ok, err_msg = download_by_doi_http(paper.doi, paper.init_paper_path("pdf"))
     if is_ok:
         paper.has_fulltext = True
         paper.has_fulltext_pdf = True