zcdsj
/
paper_server
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: get_abstract_from_elsevier 通过缓存停止

This commit is contained in:
caoqianming 2026-01-27 08:57:47 +08:00
parent fce9039e9a
commit 58780125aa
1 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/resm/tasks.py
View File

@ -25,6 +25,7 @@ def get_paper_meta_from_openalex(publication_year:int, search_key:str, end_year:
cache_cursor = cache.get(cache_key, "*") cache_cursor = cache.get(cache_key, "*")
pager = Works().filter( pager = Works().filter(
publication_year=publication_year, publication_year=publication_year,
has_doi=True,
type="article" # 将 type 移到 filter 中 type="article" # 将 type 移到 filter 中
).search(search_key).select([ ).search(search_key).select([
"id", "doi", "title", "publication_date", "id", "doi", "title", "publication_date",
@ -63,8 +64,13 @@ def get_paper_meta_from_openalex(publication_year:int, search_key:str, end_year:
ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf' ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf'
def is_elsevier_abstract_task_enabled():
return cache.get("elsevier_abstract_task_enabled", True)
@shared_task(base=CustomTask) @shared_task(base=CustomTask)
def get_abstract_from_elsevier(publication_year: int = None, number_of_task:int = 100): def get_abstract_from_elsevier(publication_year: int = None, number_of_task:int = 100):
if not is_elsevier_abstract_task_enabled():
return "stoped"
qs = Paper.objects.filter(has_abstract=False) qs = Paper.objects.filter(has_abstract=False)
if publication_year is not None: if publication_year is not None:
qs = qs.filter(publication_year=publication_year) qs = qs.filter(publication_year=publication_year)
@ -82,6 +88,8 @@ def get_abstract_from_elsevier(publication_year: int = None, number_of_task:int
err_msg = "" err_msg = ""
with requests.Session() as req: with requests.Session() as req:
for paper in qs[:number_of_task]: for paper in qs[:number_of_task]:
if not is_elsevier_abstract_task_enabled():
break
try: try:
res = req.get( res = req.get(
f"https://api.elsevier.com/content/article/doi/{paper.doi}", f"https://api.elsevier.com/content/article/doi/{paper.doi}",
@ -143,7 +151,7 @@ def get_abstract_from_elsevier(publication_year: int = None, number_of_task:int
elif res.status_code == 404: elif res.status_code == 404:
paper.fail_reason = "elsevier_doi_not_found" paper.fail_reason = "elsevier_doi_not_found"
paper.save(update_fields=["fail_reason"]) paper.save(update_fields=["fail_reason"])
if is_elsevier_abstract_task_enabled():
current_app.send_task( current_app.send_task(
"apps.resm.tasks.get_abstract_from_elsevier", "apps.resm.tasks.get_abstract_from_elsevier",
kwargs={ kwargs={