zcdsj
/
paper_server
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 优化fetch_status

This commit is contained in:
caoqianming 2026-01-30 10:37:29 +08:00
parent bfcc6d77fc
commit 1f5def2821
3 changed files with 82 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/resm/migrations/0005_alter_paper_fetch_status.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.2.27 on 2026-01-30 02:37
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('resm', '0004_alter_paper_fail_reason'),
]
operations = [
migrations.AlterField(
model_name='paper',
name='fetch_status',
field=models.CharField(blank=True, max_length=20, null=True),
),
]

20
apps/resm/models.py
View File

@ -27,11 +27,7 @@ class Paper(BaseModel):
has_fulltext = models.BooleanField(default=False, db_index=True)
has_fulltext_xml = models.BooleanField(default=False, db_index=True)
has_fulltext_pdf = models.BooleanField(default=False, db_index=True)
fetch_status = models.CharField(
max_length=20,
default="meta_only", # meta_only / downloading / abstract_ready / fulltext_ready / parsed / failed
db_index=True
)
fetch_status = models.CharField(max_length=20, null=True, blank=True) # downloading
fail_reason = models.TextField(null=True, blank=True)
source = models.CharField(
@ -63,11 +59,15 @@ class Paper(BaseModel):
with open(paper_file, "wb") as f:
f.write(content.encode("utf-8"))
def save_file_pdf(self, content):
def save_file_pdf(self, content, save_obj=False):
safe_doi = self.doi.replace("/", "_")
paper_file = os.path.join(self.init_save_dir(), f"{safe_doi}.pdf")
with open(paper_file, "wb") as f:
f.write(content)
if save_obj:
self.has_fulltext = True
self.has_fulltext_pdf = True
self.save(update_fields=["has_fulltext", "has_fulltext_pdf", "update_time"])
def save_fail_reason(self, reason):
if self.fail_reason:
@ -76,6 +76,14 @@ class Paper(BaseModel):
self.fail_reason = f";{reason}"
self.save(update_fields=["fail_reason", "update_time"])
def fetch(self, status:str):
self.fetch_status = status
self.save(update_fields=["fetch_status", "update_time"])
def fetch_end(self):
self.fetch_status = None
self.save(update_fields=["fetch_status", "update_time"])
class PaperAbstract(BaseModel):

75
apps/resm/tasks.py
View File

@ -140,11 +140,9 @@ def get_abstract_from_elsevier(number_of_task:int = 20):
for paper in qs[:number_of_task]:
if not show_task_run(def_name):
break
original_status = paper.fetch_status
if original_status == "downloading":
if paper.fetch_status == "downloading":
continue
paper.fetch_status = "downloading"
paper.save(update_fields=["fetch_status", "update_time"])
paper.fetch(status="downloading")
try:
try:
res = req.get(
@ -177,7 +175,6 @@ def get_abstract_from_elsevier(number_of_task:int = 20):
)
paper.has_abstract = True
paper.has_abstract_xml = True
paper.fetch_status = "abstract_ready"
else:
paper.save_fail_reason("elsevier_abstract_not_found")
continue
@ -191,19 +188,17 @@ def get_abstract_from_elsevier(number_of_task:int = 20):
if has_fulltext:
paper.has_fulltext = True
paper.has_fulltext_xml = True
paper.fetch_status = "fulltext_ready"
paper.save_file_xml(xml_str)
paper.save(update_fields=["has_abstract",
"has_abstract_xml", "has_fulltext",
"has_fulltext_xml", "update_time", "fetch_status"])
"has_fulltext_xml", "update_time"])
save_pdf_from_elsevier(paper)
elif res.status_code == 404:
paper.save_fail_reason("elsevier_doi_not_found")
finally:
if paper.fetch_status == "downloading":
paper.fetch_status = original_status
paper.save(update_fields=["fetch_status", "update_time"])
paper.fetch_end()
qs_count = qs.count()
if show_task_run(def_name) and qs_count > 0:
@ -274,7 +269,7 @@ def can_send_more(max_running):
@shared_task(base=CustomTask)
def send_download_fulltext_task(number_of_task=100):
qs = Paper.objects.filter(is_oa=True, has_fulltext=False, fail_reason=None).exclude(
qs = Paper.objects.filter(has_fulltext=False, fail_reason=None).exclude(
fetch_status='downloading'
)
if not qs.exists():
@ -286,7 +281,6 @@ def send_download_fulltext_task(number_of_task=100):
for paper in qs0[:number_of_task]:
if not can_send_more(number_of_task):
break
if paper.oa_url:
# 使用 countdown 错开请求时间，避免过多并发
countdown = task_count * 1 # 每个任务间隔1秒
current_app.send_task(
@ -306,21 +300,18 @@ def download_pdf(paper_id):
"""
下载单个论文的PDF
"""
paper = None
original_status = None
try:
paper = Paper.objects.get(id=paper_id)
original_status = paper.fetch_status
if original_status == "downloading":
return f"paper {paper_id} is already downloading"
# 将状态改为downloading
paper.fetch_status = 'downloading'
paper.save(update_fields=['fetch_status', 'update_time'])
if paper.fetch_status == "downloading":
return
paper.fetch("downloading")
msg = "没有下载渠道"
current_from = ""
if paper.oa_url:
if "https://doi.org/10.1016" in paper.oa_url:
current_from = "elsevier"
msg = save_pdf_from_elsevier(paper)
else:
current_from = "oa_url"
msg = save_pdf_from_oa_url(paper)
# if paper.has_fulltext_pdf is False and cache.get("openalex_api_exceed") is None:
@ -328,9 +319,7 @@ def download_pdf(paper_id):
# msg = save_pdf_from_openalex(paper)
return msg, current_from
finally:
if paper and paper.fetch_status == "downloading":
paper.fetch_status = original_status
paper.save(update_fields=['fetch_status', 'update_time'])
paper.fetch_end()
def save_pdf_from_oa_url(paper:Paper):
@ -349,12 +338,9 @@ def save_pdf_from_oa_url(paper:Paper):
res.headers.get("content-type", "") == "application/octet-stream"
)
if is_pdf and len(res.content) > 1024: # 至少1KB
paper.save_file_pdf(res.content)
paper.has_fulltext = True
paper.has_fulltext_pdf = True
paper.fetch_status = "fulltext_ready"
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
paper.save_file_pdf(res.content, save_obj=True)
return "success"
return f"oa_url_pdf_error: {res.status_code}"
def save_pdf_from_openalex(paper:Paper):
if cache.get("openalex_api_exceed"):
@ -366,19 +352,34 @@ def save_pdf_from_openalex(paper:Paper):
"api_key": OPENALEX_KEY
})
except requests.RequestException as e:
paper.save_fail_reason("openalex_pdf_error")
return f"openalex_pdf_error: {str(e)}"
if res.status_code == 200:
paper.save_file_pdf(res.content)
paper.has_fulltext = True
paper.has_fulltext_pdf = True
paper.fetch_status = "fulltext_ready"
paper.save(update_fields=["has_fulltext", "has_fulltext_pdf", "fetch_status", "update_time"])
paper.save_file_pdf(res.content, save_obj=True)
return "success"
elif res.status_code == 429:
if "Insufficient credits" in res.json().get("message", ""):
cache.set("openalex_api_exceed", True, timeout=3600)
return "Insufficient credits"
return "openalex_pdf_error: Insufficient credits"
def save_pdf_from_elsevier(paper:Paper):
params = {
"apiKey": ELSEVIER_APIKEY,
"httpAccept": "application/pdf"
}
try:
res = requests.get(
f"https://api.elsevier.com/content/article/doi/{paper.doi}",
params=params,
timeout=(3, 15)
)
except requests.RequestException as e:
return f"elsevier_request_error: {str(e)}"
if res.status_code == 200:
paper.save_file_pdf(res.content, save_obj=True)
return "success"
else:
return f"elsevier_status_error: {res.status_code}"
def save_pdf_from_scihub(paper:Paper):
pass