diff --git a/apps/resm/d_scihub.py b/apps/resm/d_scihub.py index ef1fffa..d2be843 100644 --- a/apps/resm/d_scihub.py +++ b/apps/resm/d_scihub.py @@ -1,8 +1,11 @@ import argparse import asyncio import logging +import os +import re from pathlib import Path from typing import Optional +from urllib.parse import urljoin from playwright.async_api import async_playwright, Page, Browser # 尝试导入 playwright-stealth @@ -289,6 +292,146 @@ _SCIHUB_DOMAINS = [ "sci-hub.se", ] +# bot 挑战页标题/正文关键词: 命中则需浏览器兜底 +_CHALLENGE_MARKERS = ("just a moment", "cloudflare", "checking your browser", + "ddos-guard", "ddos guard", "attention required") + +# Sci-Hub 缓存当前存活域名, 避免每篇都从头探测已挂的域名 +_ALIVE_DOMAIN_KEY = "scihub_alive_domain" +_ALIVE_DOMAIN_TTL = 600 # 秒 + + +def _normalize_pdf_url(raw: str, base_url: str) -> str: + """把 Sci-Hub 页面里抓到的 PDF 地址归一成可直接请求的绝对 URL。 + 处理: JSON 转义 \\/ 、#fragment、协议相对 // 、站内相对路径。""" + raw = raw.strip().replace("\\/", "/").split("#")[0] + if raw.startswith("//"): + return "https:" + raw + if raw.startswith("http"): + return raw + return urljoin(base_url, raw) + + +def _extract_scihub_pdf_url(html: str, base_url: str) -> Optional[str]: + """从 Sci-Hub 页面 HTML 提取内嵌 PDF 地址, 覆盖各域名/版本的常见结构; + 找不到返回 None。lxml 优先(结构化), 正则兜底(应对畸形 HTML/JS 赋值)。""" + try: + from lxml import etree + root = etree.HTML(html) + if root is not None: + for xp in ('//embed[@id="pdf"]/@src', + '//iframe[@id="pdf"]/@src', + '//div[@id="article"]//embed/@src', + '//div[@id="article"]//iframe/@src', + '//embed[@type="application/pdf"]/@src'): + hit = root.xpath(xp) + if hit and hit[0].strip(): + return _normalize_pdf_url(hit[0], base_url) + except Exception: + pass + # 正则兜底: 按钮 location.href='...pdf...' / 任意 src|href 指向 .pdf + for pat in (r"location\.href\s*=\s*['\"]([^'\"]+?\.pdf[^'\"]*)['\"]", + r"(?:src|href)\s*=\s*['\"]([^'\"]*?/downloads/[^'\"]+?\.pdf[^'\"]*)['\"]", + r"(?:src|href)\s*=\s*['\"]([^'\"]*?\.pdf(?:\?[^'\"]*)?)['\"]"): + m = re.search(pat, html, re.I) + if m and m.group(1).strip(): + return _normalize_pdf_url(m.group(1), base_url) + return None + + +def _save_pdf_bytes(content: bytes, output: str) -> bool: + """校验 PDF 魔数 + 最小体积后落盘; 不合格返回 False。""" + if not content or content[:4] != b"%PDF" or len(content) < 10240: + return False + os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True) + with open(output, "wb") as f: + f.write(content) + return True + + +def download_by_doi_http(doi: str, output: str, timeout: int = 30) -> tuple[bool, str]: + """curl-cffi 轻量下载(worker 内首选, 不起浏览器): 逐域名取 Sci-Hub 页面, + 解析内嵌 PDF 地址后直接下载并校验魔数。绝大多数无挑战的情况几秒内完成。 + + 返回 (ok, err)。err 前缀: + - scihub_error_empty_doi: DOI 为空 + - scihub_curl_cffi_not_installed + - scihub_need_play: 命中 bot 挑战, 需浏览器兜底(交独立脚本) + - scihub_error_pdf_not_found: 页面正常但无 PDF / 所有域名 HTTP 均失败 + """ + doi = (doi or "").strip() + if not doi: + return False, "scihub_error_empty_doi" + try: + import curl_cffi.requests as cf + except ImportError: + return False, "scihub_curl_cffi_not_installed" + + try: + from django.core.cache import cache + except Exception: + cache = None + + # 存活域名优先, 其余按默认顺序补上(去重) + domains = list(_SCIHUB_DOMAINS) + if cache is not None: + alive = cache.get(_ALIVE_DOMAIN_KEY) + if alive in domains: + domains = [alive] + [d for d in domains if d != alive] + + challenged = False + for domain in domains: + page_url = f"https://{domain}/{doi}" + try: + resp = cf.get(page_url, impersonate="chrome131", + headers={"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"}, + timeout=timeout, allow_redirects=True) + except Exception as e: + logger.info(f"{domain} 请求失败: {e}") + continue + + ct = resp.headers.get("content-type", "") + # 有些域名直接把 PDF 返回在 DOI 页 + if resp.content[:4] == b"%PDF" or "application/pdf" in ct: + if _save_pdf_bytes(resp.content, output): + if cache is not None: + cache.set(_ALIVE_DOMAIN_KEY, domain, timeout=_ALIVE_DOMAIN_TTL) + logger.info(f"✓ HTTP 直取 PDF 成功({domain}): {output}") + return True, "" + + if resp.status_code != 200: + if resp.status_code in (403, 503): # 多为挑战/封禁 + challenged = True + continue + + try: + text = resp.text + except Exception: + continue + if any(k in text[:3000].lower() for k in _CHALLENGE_MARKERS): + challenged = True + continue + + pdf_url = _extract_scihub_pdf_url(text, page_url) + if not pdf_url: + continue + try: + pr = cf.get(pdf_url, impersonate="chrome131", + headers={"Referer": page_url, "Accept": "application/pdf,*/*"}, + timeout=timeout, allow_redirects=True) + except Exception as e: + logger.info(f"{domain} PDF 下载失败: {e}") + continue + if pr.status_code == 200 and _save_pdf_bytes(pr.content, output): + if cache is not None: + cache.set(_ALIVE_DOMAIN_KEY, domain, timeout=_ALIVE_DOMAIN_TTL) + logger.info(f"✓ HTTP 解析下载成功({domain}): {output}") + return True, "" + + if challenged: + return False, "scihub_need_play: bot 挑战, 需浏览器兜底" + return False, "scihub_error_pdf_not_found: 所有域名 HTTP 均未获取 PDF" + def download_paper_by_doi(doi: str, output: Optional[str] = None, headless: bool = True) -> tuple[bool, str]: """ diff --git a/apps/resm/tasks.py b/apps/resm/tasks.py index bbff0e3..77c7582 100644 --- a/apps/resm/tasks.py +++ b/apps/resm/tasks.py @@ -12,7 +12,6 @@ from celery import current_app from datetime import datetime, timedelta import random from .pdf_utils import _is_elsevier_preview_pdf -from .d_oaurl import download_from_url_playwright from uuid import uuid4 import asyncio import sys @@ -833,7 +832,7 @@ def download_pdf(paper_id): def save_pdf_from_oa_url(paper: Paper): - from .d_oaurl import download_pdf_with_curl_cffi, download_from_url_playwright + from .d_oaurl import download_pdf_with_curl_cffi # 策略1: 直接请求 try: @@ -866,16 +865,10 @@ def save_pdf_from_oa_url(paper: Paper): paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "update_time"]) return "success" - # 策略3: Playwright（最终回退） - is_ok, err_msg = run_async(download_from_url_playwright(paper.oa_url, paper_path)) - if is_ok: - paper.has_fulltext = True - paper.has_fulltext_pdf = True - paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "update_time"]) - return "success" - - paper.save_fail_reason(f"oa_url_all_methods_failed: {err_msg}") - return f"oa_url_all_methods_failed: {err_msg}" + # 内联 Playwright 回退已移除(在 worker 里起浏览器太重, 且会开有头窗口)。 + # 打 oa_url_need_play 标记, 交由独立脚本 scripts/get_pdf_by_playwright.py 兜底。 + paper.save_fail_reason(f"oa_url_need_play: {err_msg}") + return f"oa_url_need_play: {err_msg}" def save_pdf_from_openalex(paper:Paper): if cache.get("openalex_api_exceed"): @@ -937,8 +930,10 @@ def save_pdf_from_elsevier(paper:Paper): return f"elsevier_status_error: {res.status_code} {res.text}" def save_pdf_from_scihub(paper:Paper): - from .d_scihub import download_paper_by_doi - is_ok, err_msg = download_paper_by_doi(paper.doi, paper.init_paper_path("pdf")) + # worker 内只走 curl-cffi 轻量解析, 不起浏览器; 撞挑战页返回 scihub_need_play, + # 交由独立脚本用浏览器兜底(同 oa_url need_play 的分工)。 + from .d_scihub import download_by_doi_http + is_ok, err_msg = download_by_doi_http(paper.doi, paper.init_paper_path("pdf")) if is_ok: paper.has_fulltext = True paper.has_fulltext_pdf = True