zcdsj
/
paper_server
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 429时等待一会2

This commit is contained in:
caoqianming 2026-01-29 11:02:06 +08:00
parent ae298d0e6d
commit 3afdbc995c
1 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/resm/tasks.py
View File

@ -316,10 +316,12 @@ def download_pdf(paper_id):
# 设置处理中标记，防止并发重复处理 # 设置处理中标记，防止并发重复处理
cache.set(cache_key, True, timeout=3600) cache.set(cache_key, True, timeout=3600)
current_from = "oa_url"
msg = save_pdf_from_oa_url(paper) msg = save_pdf_from_oa_url(paper)
if paper.has_fulltext_pdf is False: if paper.has_fulltext_pdf is False:
current_from = "openalex"
msg = save_pdf_from_openalex(paper) msg = save_pdf_from_openalex(paper)
return msg return msg, current_from
def save_pdf_from_oa_url(paper:Paper): def save_pdf_from_oa_url(paper:Paper):
@ -345,7 +347,7 @@ def save_pdf_from_oa_url(paper:Paper):
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"]) paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
return "success" return "success"
def save_pdf_from_openalex(paper:Paper): def save_pdf_from_openalex(paper:Paper, retry=1):
# 尝试openalex下载 # 尝试openalex下载
try: try:
res = requests.get(url=f"https://content.openalex.org/works/{paper.openalex_id}.pdf", res = requests.get(url=f"https://content.openalex.org/works/{paper.openalex_id}.pdf",
@ -363,8 +365,10 @@ def save_pdf_from_openalex(paper:Paper):
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"]) paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
return "success" return "success"
elif res.status_code == 429: elif res.status_code == 429:
time.sleep(random.randint(1, 10)) if retry > 1:
return save_pdf_from_openalex(paper) time.sleep(random.randint(1, 10))
retry -= 1
return save_pdf_from_openalex(paper, retry=retry)
# https://sci.bban.top/pdf/10.1016/j.conbuildmat.2020.121016.pdf?download=true # https://sci.bban.top/pdf/10.1016/j.conbuildmat.2020.121016.pdf?download=true