perf(resm): scihub改用curl-cffi轻量解析, 移除oa_url内联Playwright回退
download_pdf 链两处都不再在 worker 里起浏览器(此前是并发扩容的内存瓶颈): - Sci-Hub: 新增 d_scihub.download_by_doi_http, 用 curl-cffi 取页面解析内嵌 PDF 地址直接下载并校验魔数, 覆盖各域名 embed#pdf/iframe#pdf/location.href 结构; 缓存存活域名做秒级故障切换; 撞 Cloudflare/DDoS 挑战返回 scihub_need_play 交独立脚本兜底。浏览器版 download_paper_by_doi 保留。 - oa_url: 移除策略3内联 Playwright(还会开有头窗口), curl-cffi 失败即打 oa_url_need_play, 交 scripts/get_pdf_by_playwright.py 兜底。 run_async 保留(该脚本从 tasks 导入)。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
adf34b4bde
commit
3ab18f3ed9
|
|
@ -1,8 +1,11 @@
|
|||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import urljoin
|
||||
from playwright.async_api import async_playwright, Page, Browser
|
||||
|
||||
# 尝试导入 playwright-stealth
|
||||
|
|
@ -289,6 +292,146 @@ _SCIHUB_DOMAINS = [
|
|||
"sci-hub.se",
|
||||
]
|
||||
|
||||
# bot 挑战页标题/正文关键词: 命中则需浏览器兜底
|
||||
_CHALLENGE_MARKERS = ("just a moment", "cloudflare", "checking your browser",
|
||||
"ddos-guard", "ddos guard", "attention required")
|
||||
|
||||
# Sci-Hub 缓存当前存活域名, 避免每篇都从头探测已挂的域名
|
||||
_ALIVE_DOMAIN_KEY = "scihub_alive_domain"
|
||||
_ALIVE_DOMAIN_TTL = 600 # 秒
|
||||
|
||||
|
||||
def _normalize_pdf_url(raw: str, base_url: str) -> str:
|
||||
"""把 Sci-Hub 页面里抓到的 PDF 地址归一成可直接请求的绝对 URL。
|
||||
处理: JSON 转义 \\/ 、#fragment、协议相对 // 、站内相对路径。"""
|
||||
raw = raw.strip().replace("\\/", "/").split("#")[0]
|
||||
if raw.startswith("//"):
|
||||
return "https:" + raw
|
||||
if raw.startswith("http"):
|
||||
return raw
|
||||
return urljoin(base_url, raw)
|
||||
|
||||
|
||||
def _extract_scihub_pdf_url(html: str, base_url: str) -> Optional[str]:
|
||||
"""从 Sci-Hub 页面 HTML 提取内嵌 PDF 地址, 覆盖各域名/版本的常见结构;
|
||||
找不到返回 None。lxml 优先(结构化), 正则兜底(应对畸形 HTML/JS 赋值)。"""
|
||||
try:
|
||||
from lxml import etree
|
||||
root = etree.HTML(html)
|
||||
if root is not None:
|
||||
for xp in ('//embed[@id="pdf"]/@src',
|
||||
'//iframe[@id="pdf"]/@src',
|
||||
'//div[@id="article"]//embed/@src',
|
||||
'//div[@id="article"]//iframe/@src',
|
||||
'//embed[@type="application/pdf"]/@src'):
|
||||
hit = root.xpath(xp)
|
||||
if hit and hit[0].strip():
|
||||
return _normalize_pdf_url(hit[0], base_url)
|
||||
except Exception:
|
||||
pass
|
||||
# 正则兜底: 按钮 location.href='...pdf...' / 任意 src|href 指向 .pdf
|
||||
for pat in (r"location\.href\s*=\s*['\"]([^'\"]+?\.pdf[^'\"]*)['\"]",
|
||||
r"(?:src|href)\s*=\s*['\"]([^'\"]*?/downloads/[^'\"]+?\.pdf[^'\"]*)['\"]",
|
||||
r"(?:src|href)\s*=\s*['\"]([^'\"]*?\.pdf(?:\?[^'\"]*)?)['\"]"):
|
||||
m = re.search(pat, html, re.I)
|
||||
if m and m.group(1).strip():
|
||||
return _normalize_pdf_url(m.group(1), base_url)
|
||||
return None
|
||||
|
||||
|
||||
def _save_pdf_bytes(content: bytes, output: str) -> bool:
|
||||
"""校验 PDF 魔数 + 最小体积后落盘; 不合格返回 False。"""
|
||||
if not content or content[:4] != b"%PDF" or len(content) < 10240:
|
||||
return False
|
||||
os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True)
|
||||
with open(output, "wb") as f:
|
||||
f.write(content)
|
||||
return True
|
||||
|
||||
|
||||
def download_by_doi_http(doi: str, output: str, timeout: int = 30) -> tuple[bool, str]:
|
||||
"""curl-cffi 轻量下载(worker 内首选, 不起浏览器): 逐域名取 Sci-Hub 页面,
|
||||
解析内嵌 PDF 地址后直接下载并校验魔数。绝大多数无挑战的情况几秒内完成。
|
||||
|
||||
返回 (ok, err)。err 前缀:
|
||||
- scihub_error_empty_doi: DOI 为空
|
||||
- scihub_curl_cffi_not_installed
|
||||
- scihub_need_play: 命中 bot 挑战, 需浏览器兜底(交独立脚本)
|
||||
- scihub_error_pdf_not_found: 页面正常但无 PDF / 所有域名 HTTP 均失败
|
||||
"""
|
||||
doi = (doi or "").strip()
|
||||
if not doi:
|
||||
return False, "scihub_error_empty_doi"
|
||||
try:
|
||||
import curl_cffi.requests as cf
|
||||
except ImportError:
|
||||
return False, "scihub_curl_cffi_not_installed"
|
||||
|
||||
try:
|
||||
from django.core.cache import cache
|
||||
except Exception:
|
||||
cache = None
|
||||
|
||||
# 存活域名优先, 其余按默认顺序补上(去重)
|
||||
domains = list(_SCIHUB_DOMAINS)
|
||||
if cache is not None:
|
||||
alive = cache.get(_ALIVE_DOMAIN_KEY)
|
||||
if alive in domains:
|
||||
domains = [alive] + [d for d in domains if d != alive]
|
||||
|
||||
challenged = False
|
||||
for domain in domains:
|
||||
page_url = f"https://{domain}/{doi}"
|
||||
try:
|
||||
resp = cf.get(page_url, impersonate="chrome131",
|
||||
headers={"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"},
|
||||
timeout=timeout, allow_redirects=True)
|
||||
except Exception as e:
|
||||
logger.info(f"{domain} 请求失败: {e}")
|
||||
continue
|
||||
|
||||
ct = resp.headers.get("content-type", "")
|
||||
# 有些域名直接把 PDF 返回在 DOI 页
|
||||
if resp.content[:4] == b"%PDF" or "application/pdf" in ct:
|
||||
if _save_pdf_bytes(resp.content, output):
|
||||
if cache is not None:
|
||||
cache.set(_ALIVE_DOMAIN_KEY, domain, timeout=_ALIVE_DOMAIN_TTL)
|
||||
logger.info(f"✓ HTTP 直取 PDF 成功({domain}): {output}")
|
||||
return True, ""
|
||||
|
||||
if resp.status_code != 200:
|
||||
if resp.status_code in (403, 503): # 多为挑战/封禁
|
||||
challenged = True
|
||||
continue
|
||||
|
||||
try:
|
||||
text = resp.text
|
||||
except Exception:
|
||||
continue
|
||||
if any(k in text[:3000].lower() for k in _CHALLENGE_MARKERS):
|
||||
challenged = True
|
||||
continue
|
||||
|
||||
pdf_url = _extract_scihub_pdf_url(text, page_url)
|
||||
if not pdf_url:
|
||||
continue
|
||||
try:
|
||||
pr = cf.get(pdf_url, impersonate="chrome131",
|
||||
headers={"Referer": page_url, "Accept": "application/pdf,*/*"},
|
||||
timeout=timeout, allow_redirects=True)
|
||||
except Exception as e:
|
||||
logger.info(f"{domain} PDF 下载失败: {e}")
|
||||
continue
|
||||
if pr.status_code == 200 and _save_pdf_bytes(pr.content, output):
|
||||
if cache is not None:
|
||||
cache.set(_ALIVE_DOMAIN_KEY, domain, timeout=_ALIVE_DOMAIN_TTL)
|
||||
logger.info(f"✓ HTTP 解析下载成功({domain}): {output}")
|
||||
return True, ""
|
||||
|
||||
if challenged:
|
||||
return False, "scihub_need_play: bot 挑战, 需浏览器兜底"
|
||||
return False, "scihub_error_pdf_not_found: 所有域名 HTTP 均未获取 PDF"
|
||||
|
||||
|
||||
def download_paper_by_doi(doi: str, output: Optional[str] = None, headless: bool = True) -> tuple[bool, str]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ from celery import current_app
|
|||
from datetime import datetime, timedelta
|
||||
import random
|
||||
from .pdf_utils import _is_elsevier_preview_pdf
|
||||
from .d_oaurl import download_from_url_playwright
|
||||
from uuid import uuid4
|
||||
import asyncio
|
||||
import sys
|
||||
|
|
@ -833,7 +832,7 @@ def download_pdf(paper_id):
|
|||
|
||||
|
||||
def save_pdf_from_oa_url(paper: Paper):
|
||||
from .d_oaurl import download_pdf_with_curl_cffi, download_from_url_playwright
|
||||
from .d_oaurl import download_pdf_with_curl_cffi
|
||||
|
||||
# 策略1: 直接请求
|
||||
try:
|
||||
|
|
@ -866,16 +865,10 @@ def save_pdf_from_oa_url(paper: Paper):
|
|||
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "update_time"])
|
||||
return "success"
|
||||
|
||||
# 策略3: Playwright（最终回退）
|
||||
is_ok, err_msg = run_async(download_from_url_playwright(paper.oa_url, paper_path))
|
||||
if is_ok:
|
||||
paper.has_fulltext = True
|
||||
paper.has_fulltext_pdf = True
|
||||
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "update_time"])
|
||||
return "success"
|
||||
|
||||
paper.save_fail_reason(f"oa_url_all_methods_failed: {err_msg}")
|
||||
return f"oa_url_all_methods_failed: {err_msg}"
|
||||
# 内联 Playwright 回退已移除(在 worker 里起浏览器太重, 且会开有头窗口)。
|
||||
# 打 oa_url_need_play 标记, 交由独立脚本 scripts/get_pdf_by_playwright.py 兜底。
|
||||
paper.save_fail_reason(f"oa_url_need_play: {err_msg}")
|
||||
return f"oa_url_need_play: {err_msg}"
|
||||
|
||||
def save_pdf_from_openalex(paper:Paper):
|
||||
if cache.get("openalex_api_exceed"):
|
||||
|
|
@ -937,8 +930,10 @@ def save_pdf_from_elsevier(paper:Paper):
|
|||
return f"elsevier_status_error: {res.status_code} {res.text}"
|
||||
|
||||
def save_pdf_from_scihub(paper:Paper):
|
||||
from .d_scihub import download_paper_by_doi
|
||||
is_ok, err_msg = download_paper_by_doi(paper.doi, paper.init_paper_path("pdf"))
|
||||
# worker 内只走 curl-cffi 轻量解析, 不起浏览器; 撞挑战页返回 scihub_need_play,
|
||||
# 交由独立脚本用浏览器兜底(同 oa_url need_play 的分工)。
|
||||
from .d_scihub import download_by_doi_http
|
||||
is_ok, err_msg = download_by_doi_http(paper.doi, paper.init_paper_path("pdf"))
|
||||
if is_ok:
|
||||
paper.has_fulltext = True
|
||||
paper.has_fulltext_pdf = True
|
||||
|
|
|
|||
Loading…
Reference in New Issue