From 7183dfd590661dd2ad66ea1ae82115b0a400972d Mon Sep 17 00:00:00 2001 From: caoqianming Date: Wed, 28 Jan 2026 15:01:49 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0download=5Fpdf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/resm/models.py | 2 +- apps/resm/services.py | 0 apps/resm/tasks.py | 143 +++++++++++++++++++++++++++++++++++------- 3 files changed, 121 insertions(+), 24 deletions(-) create mode 100644 apps/resm/services.py diff --git a/apps/resm/models.py b/apps/resm/models.py index 666617e..49ceca2 100644 --- a/apps/resm/models.py +++ b/apps/resm/models.py @@ -74,7 +74,7 @@ class Paper(BaseModel): if self.fail_reason: self.fail_reason += f";{reason}" else: - self.fail_reason = reason + self.fail_reason = f";{reason}" self.save(update_fields=["fail_reason", "update_time"]) diff --git a/apps/resm/services.py b/apps/resm/services.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/resm/tasks.py b/apps/resm/tasks.py index a46b1c1..599bd7f 100644 --- a/apps/resm/tasks.py +++ b/apps/resm/tasks.py @@ -10,12 +10,14 @@ import requests from lxml import etree from celery import current_app from datetime import datetime +import random config.email = "caoqianming@foxmail.com" config.max_retries = 0 config.retry_backoff_factor = 0.1 config.retry_http_codes = [429, 500, 503] -config.api_key = "4KJZdkCFA0uFb6IsYKc8cd" +OPENALEX_KEY = "4KJZdkCFA0uFb6IsYKc8cd" +config.api_key = OPENALEX_KEY @shared_task(base=CustomTask) def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None): @@ -92,6 +94,24 @@ def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:s ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf' + +# 常用的 User-Agent 列表 +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15", +] + +def get_random_headers(): + """获取随机的请求头""" + return { + "User-Agent": random.choice(USER_AGENTS), + "Accept": "application/pdf, */*", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Referer": "https://www.google.com/", + } + def show_task_run(def_name: str): return cache.get(def_name, True) @@ -214,10 +234,16 @@ def get_pdf_from_elsevier(number_of_task=100): except requests.RequestException: err_msg = "elsevier_request_error" break - if res.status_code == 200 and res.headers["content-type"] == "application/pdf": - paper.save_file_pdf(res.content) - paper.has_fulltext_pdf = True - paper.save(update_fields=["has_fulltext_pdf", "update_time"]) + if res.status_code == 200: + # 检查是否是PDF文件：检查魔数 %PDF 或 content-type + is_pdf = ( + res.content.startswith(b'%PDF') or + res.headers.get("content-type", "").startswith("application/pdf") + ) + if is_pdf and len(res.content) > 1024: # 至少1KB + paper.save_file_pdf(res.content) + paper.has_fulltext_pdf = True + paper.save(update_fields=["has_fulltext_pdf", "update_time"]) qs_count = qs.count() if show_task_run(def_name) and qs_count > 0: current_app.send_task( @@ -238,29 +264,100 @@ def get_pdf_from_oa_url(number_of_task=100): qs = Paper.objects.filter(is_oa=True, has_fulltext=False).exclude( fail_reason__contains="oa_url_request_error" ).exclude(fail_reason__contains="oa_url_not_pdf") - err_msg = "" - for paper in qs[:number_of_task]: + + if not qs.exists(): + return "done" + + qs0 = qs.order_by("?") + + # 分批发送任务，控制并发数量，避免过多请求 + task_count = 0 + for paper in qs0[:number_of_task]: + if not show_task_run(def_name): + break if paper.oa_url: - try: - res = requests.get(paper.oa_url, timeout=(3, 15)) - except requests.RequestException: - paper.save_fail_reason("oa_url_request_error") - continue - if res.status_code == 200 and res.headers["content-type"] == "application/pdf": - paper.save_file_pdf(res.content) - paper.has_fulltext = True - paper.has_fulltext_pdf = True - paper.fetch_status = "fulltext_ready" - paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"]) - else: - paper.save_fail_reason("oa_url_not_pdf") + # 使用 countdown 错开请求时间，避免过多并发 + countdown = task_count * 2 # 每个任务间隔2秒 + current_app.send_task( + "apps.resm.tasks.download_pdf", + kwargs={ + "paper_id": paper.id, + }, + countdown=countdown, + ) + task_count += 1 + qs_count = qs.count() - if show_task_run(def_name) and qs_count > 0: + if show_task_run(def_name) and qs_count > number_of_task: current_app.send_task( "apps.resm.tasks.get_pdf_from_oa_url", kwargs={ "number_of_task": number_of_task, }, - countdown=5, + countdown=60, # 等待当前批次完成后再继续 ) - return f'{def_name}, {err_msg}, remaining {qs_count} papers' \ No newline at end of file + return f'{def_name}, sent {task_count} tasks, remaining {qs_count} papers' + + +@shared_task(base=CustomTask) +def download_pdf(self, paper_id): + """ + 下载单个论文的PDF + """ + try: + paper = Paper.objects.get(id=paper_id) + except Paper.DoesNotExist: + return f"Paper {paper_id} not found" + + # 检查缓存，避免短时间内重复下载同一个paper + cache_key = f"download_pdf_{paper_id}" + if cache.get(cache_key): + return "already_processing" + + # 设置处理中标记，防止并发重复处理 + cache.set(cache_key, True, timeout=3600) + + try: + headers = get_random_headers() + res = requests.get(paper.oa_url, headers=headers, timeout=(3, 15)) + except requests.RequestException as e: + paper.save_fail_reason("oa_url_request_error") + return f"request_error_final: {str(e)}" + + if res.status_code == 200: + # 检查是否是PDF文件：检查魔数 %PDF 或 content-type + is_pdf = ( + res.content.startswith(b'%PDF') or + res.headers.get("content-type", "").startswith("application/pdf") or + res.headers.get("content-type", "") == "application/octet-stream" + ) + if is_pdf and len(res.content) > 1024: # 至少1KB + paper.save_file_pdf(res.content) + paper.has_fulltext = True + paper.has_fulltext_pdf = True + paper.fetch_status = "fulltext_ready" + paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"]) + return "success" + else: + paper.save_fail_reason("oa_url_not_pdf") + return "not_pdf" + else: + # 尝试openalex下载 + try: + res = requests.get(url=f"https://content.openalex.org/works/{paper.openalex_id}.pdf", + params={ + "api_key": OPENALEX_KEY + }) + except requests.RequestException as e: + paper.save_fail_reason("oa_url_not_pdf;openalex_pdf_error") + return f"openalex_pdf_error: {str(e)}" + if res.status_code == 200: + paper.save_file_pdf(res.content) + paper.has_fulltext = True + paper.has_fulltext_pdf = True + paper.fetch_status = "fulltext_ready" + paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"]) + return "success" + else: + paper.save_fail_reason("oa_url_not_pdf;openalex_pdf_error") + return f"openalex_pdf_error: {res.status_code}" \ No newline at end of file