feat: get_abstract_from_elsevier 可自动触发自己

apps/resm/tasks.py

@@ -11,6 +11,7 @@ import requests
 from lxml import etree
 from django.conf import settings
 import os
+from celery import current_app
 
 config.email = "caoqianming@foxmail.com"
 config.max_retries = 0
@@ -61,11 +62,11 @@ def get_paper_meta_from_openalex(publication_year:int, search_key:str):
 
 ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf'
 @shared_task(base=CustomTask)
-def get_abstract_from_elsevier(publication_year: int, number_of_task:int = 100):
-    qs = Paper.objects.filter(
-        publication_year=publication_year,
-        has_abstract=False
-    ).exclude(
+def get_abstract_from_elsevier(publication_year: int = None, number_of_task:int = 100):
+    qs = Paper.objects.filter(has_abstract=True)
+    if publication_year is not None:
+        qs = qs.filter(publication_year=publication_year)
+    qs = qs.exclude(
         fail_reason="elsevier_doi_not_found"
     ).order_by("publication_date")
 
@@ -77,8 +78,12 @@ def get_abstract_from_elsevier(publication_year: int, number_of_task:int = 100):
         "httpAccept": "text/xml"
     }
     err_msg = ""
-    req = requests.Session()
-    for paper in qs[:number_of_task]:
+    if number_of_task is None:
+        papers = qs.all()
+    else:
+        papers = qs[:number_of_task]
+    with requests.Session() as req:
+        for paper in papers:
             try:
                 res = req.get(
                     f"https://api.elsevier.com/content/article/doi/{paper.doi}",
@@ -111,12 +116,14 @@ def get_abstract_from_elsevier(publication_year: int, number_of_task:int = 100):
                     )
                     paper.has_abstract = True
                     paper.has_abstract_xml = True
+                    paper.fetch_status = "abstract_ready"
 
                 paras = root.xpath("//ce:para", namespaces=ns)
                 has_fulltext = len(paras) > 0
                 if has_fulltext:
                     paper.has_fulltext = True
                     paper.has_fulltext_xml = True
+                    paper.fetch_status = "fulltext_ready"
                     
                 publication_date = paper.publication_date
                 paper_dir = os.path.join(
@@ -143,4 +150,12 @@ def get_abstract_from_elsevier(publication_year: int, number_of_task:int = 100):
     if remaining_count == 0:
         return "done"
     else:
+        current_app.send_task(
+            "apps.resm.tasks.get_abstract_from_elsevier",
+            kwargs={
+                "publication_year": publication_year,
+                "number_of_task": number_of_task,
+            },
+            countdown=5,
+        )
         return f'{err_msg}, remaining {remaining_count} papers'