feat: 增加download_pdf

apps/resm/models.py

@@ -74,7 +74,7 @@ class Paper(BaseModel):
         if self.fail_reason:
             self.fail_reason += f";{reason}"
         else:
-            self.fail_reason = reason
+            self.fail_reason = f";{reason}"
         self.save(update_fields=["fail_reason", "update_time"])
     
 

apps/resm/tasks.py

@@ -10,12 +10,14 @@ import requests
 from lxml import etree
 from celery import current_app
 from datetime import datetime
+import random
 
 config.email = "caoqianming@foxmail.com"
 config.max_retries = 0
 config.retry_backoff_factor = 0.1
 config.retry_http_codes = [429, 500, 503]
-config.api_key = "4KJZdkCFA0uFb6IsYKc8cd"
+OPENALEX_KEY = "4KJZdkCFA0uFb6IsYKc8cd"
+config.api_key = OPENALEX_KEY
 
 @shared_task(base=CustomTask)
 def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None):
@@ -92,6 +94,24 @@ def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:s
 
 
 ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf'
+
+# 常用的 User-Agent 列表
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15",
+]
+
+def get_random_headers():
+    """获取随机的请求头"""
+    return {
+        "User-Agent": random.choice(USER_AGENTS),
+        "Accept": "application/pdf, */*",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "Referer": "https://www.google.com/",
+    }
+
 def show_task_run(def_name: str):
     return cache.get(def_name, True)
 
@@ -214,7 +234,13 @@ def get_pdf_from_elsevier(number_of_task=100):
             except requests.RequestException:
                 err_msg = "elsevier_request_error"
                 break
-            if res.status_code == 200 and res.headers["content-type"] == "application/pdf":
+            if res.status_code == 200:
+                # 检查是否是PDF文件：检查魔数 %PDF 或 content-type
+                is_pdf = (
+                    res.content.startswith(b'%PDF') or 
+                    res.headers.get("content-type", "").startswith("application/pdf")
+                )
+                if is_pdf and len(res.content) > 1024:  # 至少1KB
                     paper.save_file_pdf(res.content)
                     paper.has_fulltext_pdf = True
                     paper.save(update_fields=["has_fulltext_pdf", "update_time"])
@@ -238,29 +264,100 @@ def get_pdf_from_oa_url(number_of_task=100):
     qs = Paper.objects.filter(is_oa=True, has_fulltext=False).exclude(
         fail_reason__contains="oa_url_request_error"
     ).exclude(fail_reason__contains="oa_url_not_pdf")
-    err_msg = ""
+    
-    for paper in qs[:number_of_task]:
+    if not qs.exists():
+        return "done"
+
+    qs0 = qs.order_by("?")
+    
+    # 分批发送任务，控制并发数量，避免过多请求
+    task_count = 0
+    for paper in qs0[:number_of_task]:
+        if not show_task_run(def_name):
+            break
         if paper.oa_url:
-            try:
+            # 使用 countdown 错开请求时间，避免过多并发
-                res = requests.get(paper.oa_url, timeout=(3, 15))
+            countdown = task_count * 2  # 每个任务间隔2秒
-            except requests.RequestException:
+            current_app.send_task(
-                paper.save_fail_reason("oa_url_request_error")
+                "apps.resm.tasks.download_pdf",
-                continue
+                kwargs={
-            if res.status_code == 200 and res.headers["content-type"] == "application/pdf":
+                    "paper_id": paper.id,
-                paper.save_file_pdf(res.content)
+                },
-                paper.has_fulltext = True
+                countdown=countdown,
-                paper.has_fulltext_pdf = True
+            )
-                paper.fetch_status = "fulltext_ready"
+            task_count += 1
-                paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
+
-            else:
-                paper.save_fail_reason("oa_url_not_pdf")
     qs_count = qs.count()
-    if show_task_run(def_name) and qs_count > 0:
+    if show_task_run(def_name) and qs_count > number_of_task:
         current_app.send_task(
             "apps.resm.tasks.get_pdf_from_oa_url",
             kwargs={
                 "number_of_task": number_of_task,
             },
-            countdown=5,
+            countdown=60,  # 等待当前批次完成后再继续
         )
-    return f'{def_name}, {err_msg}, remaining {qs_count} papers'
+    return f'{def_name}, sent {task_count} tasks, remaining {qs_count} papers'
+
+
+@shared_task(base=CustomTask)
+def download_pdf(self, paper_id):
+    """
+    下载单个论文的PDF
+    """
+    try:
+        paper = Paper.objects.get(id=paper_id)
+    except Paper.DoesNotExist:
+        return f"Paper {paper_id} not found"
+    
+    # 检查缓存，避免短时间内重复下载同一个paper
+    cache_key = f"download_pdf_{paper_id}"
+    if cache.get(cache_key):
+        return "already_processing"
+
+    # 设置处理中标记，防止并发重复处理
+    cache.set(cache_key, True, timeout=3600) 
+    
+    try:
+        headers = get_random_headers()
+        res = requests.get(paper.oa_url, headers=headers, timeout=(3, 15))
+    except requests.RequestException as e:
+        paper.save_fail_reason("oa_url_request_error")
+        return f"request_error_final: {str(e)}"
+    
+    if res.status_code == 200:
+        # 检查是否是PDF文件：检查魔数 %PDF 或 content-type
+        is_pdf = (
+            res.content.startswith(b'%PDF') or 
+            res.headers.get("content-type", "").startswith("application/pdf") or
+            res.headers.get("content-type", "") == "application/octet-stream"
+        )
+        if is_pdf and len(res.content) > 1024:  # 至少1KB
+            paper.save_file_pdf(res.content)
+            paper.has_fulltext = True
+            paper.has_fulltext_pdf = True
+            paper.fetch_status = "fulltext_ready"
+            paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
+            return "success"
+        else:
+            paper.save_fail_reason("oa_url_not_pdf")
+            return "not_pdf"
+    else:
+        # 尝试openalex下载
+        try:
+            res = requests.get(url=f"https://content.openalex.org/works/{paper.openalex_id}.pdf",
+            params={
+                "api_key": OPENALEX_KEY
+            })
+        except requests.RequestException as e:
+            paper.save_fail_reason("oa_url_not_pdf;openalex_pdf_error")
+            return f"openalex_pdf_error: {str(e)}"
+        if res.status_code == 200:
+            paper.save_file_pdf(res.content)
+            paper.has_fulltext = True
+            paper.has_fulltext_pdf = True
+            paper.fetch_status = "fulltext_ready"
+            paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
+            return "success"
+        else:
+            paper.save_fail_reason("oa_url_not_pdf;openalex_pdf_error")
+            return f"openalex_pdf_error: {res.status_code}"