feat: 增加download_pdf
This commit is contained in:
parent
044e5cec18
commit
7183dfd590
|
|
@ -74,7 +74,7 @@ class Paper(BaseModel):
|
|||
if self.fail_reason:
|
||||
self.fail_reason += f";{reason}"
|
||||
else:
|
||||
self.fail_reason = reason
|
||||
self.fail_reason = f";{reason}"
|
||||
self.save(update_fields=["fail_reason", "update_time"])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,12 +10,14 @@ import requests
|
|||
from lxml import etree
|
||||
from celery import current_app
|
||||
from datetime import datetime
|
||||
import random
|
||||
|
||||
config.email = "caoqianming@foxmail.com"
|
||||
config.max_retries = 0
|
||||
config.retry_backoff_factor = 0.1
|
||||
config.retry_http_codes = [429, 500, 503]
|
||||
config.api_key = "4KJZdkCFA0uFb6IsYKc8cd"
|
||||
OPENALEX_KEY = "4KJZdkCFA0uFb6IsYKc8cd"
|
||||
config.api_key = OPENALEX_KEY
|
||||
|
||||
@shared_task(base=CustomTask)
|
||||
def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None):
|
||||
|
|
@ -92,6 +94,24 @@ def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:s
|
|||
|
||||
|
||||
ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf'
|
||||
|
||||
# 常用的 User-Agent 列表
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15",
|
||||
]
|
||||
|
||||
def get_random_headers():
|
||||
"""获取随机的请求头"""
|
||||
return {
|
||||
"User-Agent": random.choice(USER_AGENTS),
|
||||
"Accept": "application/pdf, */*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Referer": "https://www.google.com/",
|
||||
}
|
||||
|
||||
def show_task_run(def_name: str):
|
||||
return cache.get(def_name, True)
|
||||
|
||||
|
|
@ -214,7 +234,13 @@ def get_pdf_from_elsevier(number_of_task=100):
|
|||
except requests.RequestException:
|
||||
err_msg = "elsevier_request_error"
|
||||
break
|
||||
if res.status_code == 200 and res.headers["content-type"] == "application/pdf":
|
||||
if res.status_code == 200:
|
||||
# 检查是否是PDF文件：检查魔数 %PDF 或 content-type
|
||||
is_pdf = (
|
||||
res.content.startswith(b'%PDF') or
|
||||
res.headers.get("content-type", "").startswith("application/pdf")
|
||||
)
|
||||
if is_pdf and len(res.content) > 1024: # 至少1KB
|
||||
paper.save_file_pdf(res.content)
|
||||
paper.has_fulltext_pdf = True
|
||||
paper.save(update_fields=["has_fulltext_pdf", "update_time"])
|
||||
|
|
@ -238,29 +264,100 @@ def get_pdf_from_oa_url(number_of_task=100):
|
|||
qs = Paper.objects.filter(is_oa=True, has_fulltext=False).exclude(
|
||||
fail_reason__contains="oa_url_request_error"
|
||||
).exclude(fail_reason__contains="oa_url_not_pdf")
|
||||
err_msg = ""
|
||||
for paper in qs[:number_of_task]:
|
||||
|
||||
if not qs.exists():
|
||||
return "done"
|
||||
|
||||
qs0 = qs.order_by("?")
|
||||
|
||||
# 分批发送任务，控制并发数量，避免过多请求
|
||||
task_count = 0
|
||||
for paper in qs0[:number_of_task]:
|
||||
if not show_task_run(def_name):
|
||||
break
|
||||
if paper.oa_url:
|
||||
try:
|
||||
res = requests.get(paper.oa_url, timeout=(3, 15))
|
||||
except requests.RequestException:
|
||||
paper.save_fail_reason("oa_url_request_error")
|
||||
continue
|
||||
if res.status_code == 200 and res.headers["content-type"] == "application/pdf":
|
||||
paper.save_file_pdf(res.content)
|
||||
paper.has_fulltext = True
|
||||
paper.has_fulltext_pdf = True
|
||||
paper.fetch_status = "fulltext_ready"
|
||||
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
|
||||
else:
|
||||
paper.save_fail_reason("oa_url_not_pdf")
|
||||
# 使用 countdown 错开请求时间，避免过多并发
|
||||
countdown = task_count * 2 # 每个任务间隔2秒
|
||||
current_app.send_task(
|
||||
"apps.resm.tasks.download_pdf",
|
||||
kwargs={
|
||||
"paper_id": paper.id,
|
||||
},
|
||||
countdown=countdown,
|
||||
)
|
||||
task_count += 1
|
||||
|
||||
qs_count = qs.count()
|
||||
if show_task_run(def_name) and qs_count > 0:
|
||||
if show_task_run(def_name) and qs_count > number_of_task:
|
||||
current_app.send_task(
|
||||
"apps.resm.tasks.get_pdf_from_oa_url",
|
||||
kwargs={
|
||||
"number_of_task": number_of_task,
|
||||
},
|
||||
countdown=5,
|
||||
countdown=60, # 等待当前批次完成后再继续
|
||||
)
|
||||
return f'{def_name}, {err_msg}, remaining {qs_count} papers'
|
||||
return f'{def_name}, sent {task_count} tasks, remaining {qs_count} papers'
|
||||
|
||||
|
||||
@shared_task(base=CustomTask)
|
||||
def download_pdf(self, paper_id):
|
||||
"""
|
||||
下载单个论文的PDF
|
||||
"""
|
||||
try:
|
||||
paper = Paper.objects.get(id=paper_id)
|
||||
except Paper.DoesNotExist:
|
||||
return f"Paper {paper_id} not found"
|
||||
|
||||
# 检查缓存，避免短时间内重复下载同一个paper
|
||||
cache_key = f"download_pdf_{paper_id}"
|
||||
if cache.get(cache_key):
|
||||
return "already_processing"
|
||||
|
||||
# 设置处理中标记，防止并发重复处理
|
||||
cache.set(cache_key, True, timeout=3600)
|
||||
|
||||
try:
|
||||
headers = get_random_headers()
|
||||
res = requests.get(paper.oa_url, headers=headers, timeout=(3, 15))
|
||||
except requests.RequestException as e:
|
||||
paper.save_fail_reason("oa_url_request_error")
|
||||
return f"request_error_final: {str(e)}"
|
||||
|
||||
if res.status_code == 200:
|
||||
# 检查是否是PDF文件：检查魔数 %PDF 或 content-type
|
||||
is_pdf = (
|
||||
res.content.startswith(b'%PDF') or
|
||||
res.headers.get("content-type", "").startswith("application/pdf") or
|
||||
res.headers.get("content-type", "") == "application/octet-stream"
|
||||
)
|
||||
if is_pdf and len(res.content) > 1024: # 至少1KB
|
||||
paper.save_file_pdf(res.content)
|
||||
paper.has_fulltext = True
|
||||
paper.has_fulltext_pdf = True
|
||||
paper.fetch_status = "fulltext_ready"
|
||||
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
|
||||
return "success"
|
||||
else:
|
||||
paper.save_fail_reason("oa_url_not_pdf")
|
||||
return "not_pdf"
|
||||
else:
|
||||
# 尝试openalex下载
|
||||
try:
|
||||
res = requests.get(url=f"https://content.openalex.org/works/{paper.openalex_id}.pdf",
|
||||
params={
|
||||
"api_key": OPENALEX_KEY
|
||||
})
|
||||
except requests.RequestException as e:
|
||||
paper.save_fail_reason("oa_url_not_pdf;openalex_pdf_error")
|
||||
return f"openalex_pdf_error: {str(e)}"
|
||||
if res.status_code == 200:
|
||||
paper.save_file_pdf(res.content)
|
||||
paper.has_fulltext = True
|
||||
paper.has_fulltext_pdf = True
|
||||
paper.fetch_status = "fulltext_ready"
|
||||
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
|
||||
return "success"
|
||||
else:
|
||||
paper.save_fail_reason("oa_url_not_pdf;openalex_pdf_error")
|
||||
return f"openalex_pdf_error: {res.status_code}"
|
||||
Loading…
Reference in New Issue