zcdsj
/
paper_server
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: get_abstract_from_elsevier 可自动触发自己

This commit is contained in:
caoqianming 2026-01-26 17:42:16 +08:00
parent a11dce1358
commit 5529f97ab8
1 changed files with 75 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
apps/resm/tasks.py
View File

@ -11,6 +11,7 @@ import requests
from lxml import etree from lxml import etree
from django.conf import settings from django.conf import settings
import os import os
from celery import current_app
config.email = "caoqianming@foxmail.com" config.email = "caoqianming@foxmail.com"
config.max_retries = 0 config.max_retries = 0
@ -61,11 +62,11 @@ def get_paper_meta_from_openalex(publication_year:int, search_key:str):
ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf' ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf'
@shared_task(base=CustomTask) @shared_task(base=CustomTask)
def get_abstract_from_elsevier(publication_year: int, number_of_task:int = 100): def get_abstract_from_elsevier(publication_year: int = None, number_of_task:int = 100):
qs = Paper.objects.filter( qs = Paper.objects.filter(has_abstract=True)
publication_year=publication_year, if publication_year is not None:
has_abstract=False qs = qs.filter(publication_year=publication_year)
).exclude( qs = qs.exclude(
fail_reason="elsevier_doi_not_found" fail_reason="elsevier_doi_not_found"
).order_by("publication_date") ).order_by("publication_date")
@ -77,70 +78,84 @@ def get_abstract_from_elsevier(publication_year: int, number_of_task:int = 100):
"httpAccept": "text/xml" "httpAccept": "text/xml"
} }
err_msg = "" err_msg = ""
req = requests.Session() if number_of_task is None:
for paper in qs[:number_of_task]: papers = qs.all()
try: else:
res = req.get( papers = qs[:number_of_task]
f"https://api.elsevier.com/content/article/doi/{paper.doi}", with requests.Session() as req:
params=params, for paper in papers:
timeout=(3, 15)
)
except requests.RequestException:
err_msg = "elsevier_request_error"
break
if res.status_code == 200:
xml_str = res.text
try: try:
root = etree.fromstring(xml_str.encode("utf-8")) res = req.get(
except etree.XMLSyntaxError: f"https://api.elsevier.com/content/article/doi/{paper.doi}",
paper.fail_reason = "elsevier_xml_error" params=params,
paper.save(update_fields=["fail_reason"]) timeout=(3, 15)
continue
ns = {"dc": "http://purl.org/dc/elements/1.1/",
"ce": "http://www.elsevier.com/xml/common/dtd"}
abstract = root.xpath("//dc:description/text()", namespaces=ns)
if abstract:
PaperAbstract.objects.update_or_create(
paper=paper,
defaults={
"abstract": abstract[0].strip(),
"source": "elsevier"
}
) )
paper.has_abstract = True except requests.RequestException:
paper.has_abstract_xml = True err_msg = "elsevier_request_error"
break
paras = root.xpath("//ce:para", namespaces=ns) if res.status_code == 200:
has_fulltext = len(paras) > 0 xml_str = res.text
if has_fulltext: try:
paper.has_fulltext = True root = etree.fromstring(xml_str.encode("utf-8"))
paper.has_fulltext_xml = True except etree.XMLSyntaxError:
paper.fail_reason = "elsevier_xml_error"
publication_date = paper.publication_date paper.save(update_fields=["fail_reason"])
paper_dir = os.path.join( continue
settings.BASE_DIR,
"media/papers",
str(publication_date.year),
str(publication_date.month),
str(publication_date.day)
)
os.makedirs(paper_dir, exist_ok=True)
safe_doi = paper.doi.replace("/", "_") ns = {"dc": "http://purl.org/dc/elements/1.1/",
paper_file = os.path.join(paper_dir, f"{safe_doi}.xml") "ce": "http://www.elsevier.com/xml/common/dtd"}
with open(paper_file, "wb") as f: abstract = root.xpath("//dc:description/text()", namespaces=ns)
f.write(xml_str.encode("utf-8")) if abstract:
PaperAbstract.objects.update_or_create(
paper=paper,
defaults={
"abstract": abstract[0].strip(),
"source": "elsevier"
}
)
paper.has_abstract = True
paper.has_abstract_xml = True
paper.fetch_status = "abstract_ready"
paper.save(update_fields=["has_abstract", "has_abstract_xml", "has_fulltext", "has_fulltext_xml", "update_time"]) paras = root.xpath("//ce:para", namespaces=ns)
has_fulltext = len(paras) > 0
if has_fulltext:
paper.has_fulltext = True
paper.has_fulltext_xml = True
paper.fetch_status = "fulltext_ready"
publication_date = paper.publication_date
paper_dir = os.path.join(
settings.BASE_DIR,
"media/papers",
str(publication_date.year),
str(publication_date.month),
str(publication_date.day)
)
os.makedirs(paper_dir, exist_ok=True)
elif res.status_code == 404: safe_doi = paper.doi.replace("/", "_")
paper.fail_reason = "elsevier_doi_not_found" paper_file = os.path.join(paper_dir, f"{safe_doi}.xml")
paper.save(update_fields=["fail_reason"]) with open(paper_file, "wb") as f:
f.write(xml_str.encode("utf-8"))
paper.save(update_fields=["has_abstract", "has_abstract_xml", "has_fulltext", "has_fulltext_xml", "update_time"])
elif res.status_code == 404:
paper.fail_reason = "elsevier_doi_not_found"
paper.save(update_fields=["fail_reason"])
remaining_count = qs.count() remaining_count = qs.count()
if remaining_count == 0: if remaining_count == 0:
return "done" return "done"
else: else:
current_app.send_task(
"apps.resm.tasks.get_abstract_from_elsevier",
kwargs={
"publication_year": publication_year,
"number_of_task": number_of_task,
},
countdown=5,
)
return f'{err_msg}, remaining {remaining_count} papers' return f'{err_msg}, remaining {remaining_count} papers'