zcdsj
/
paper_server
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: zcdsj:b621b0409d6effa6c5cc4f2a259460c884a6a738
Branches Tags
zcdsj:main
..
compare: zcdsj:d813a5365272517cb96258d5851a1018e5631418
Branches Tags
zcdsj:main

No commits in common. "b621b0409d6effa6c5cc4f2a259460c884a6a738" and "d813a5365272517cb96258d5851a1018e5631418" have entirely different histories.
b621b0409d ... d813a53652

1 changed files with 9 additions and 11 deletions
Show Stats Expand all files Collapse all files

18
apps/resm/tasks.py
View File

@ -22,9 +22,9 @@ OPENALEX_KEY = "NPimoE2ecdWmfdhH8abxEp"
config.api_key = OPENALEX_KEY config.api_key = OPENALEX_KEY
ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf' ELSEVIER_APIKEY = 'aa8868cac9e27d6153ab0a0acd7b50bf'
ELSEVIER_HEADERS = { ELSEVIER_HEAEDERS = {
"X-ELS-Insttoken": "135fa874aea9f0de11cad187ccb4878c",
"X-ELS-APIKey": ELSEVIER_APIKEY, "X-ELS-APIKey": ELSEVIER_APIKEY,
"X-ELS-Insttoken": "135fa874aea9f0de11cad187ccb4878c "
} }
@shared_task(base=CustomTask) @shared_task(base=CustomTask)
@ -129,15 +129,14 @@ def get_abstract_from_elsevier(number_of_task:int = 20):
qs = qs.exclude( qs = qs.exclude(
fail_reason__contains="elsevier_doi_not_found" fail_reason__contains="elsevier_doi_not_found"
).exclude(fail_reason__contains="elsevier_abstract_not_found" ).exclude(fail_reason__contains="elsevier_abstract_not_found"
).exclude(fetch_status="downloading" ).exclude(fetch_status="downloading").order_by("publication_date")
).filter(doi__startswith="10.1016").order_by("publication_date")
if not qs.exists(): if not qs.exists():
return "done" return "done"
params = { params = {
"httpAccept": "text/xml", "apiKey": ELSEVIER_APIKEY,
"view": "FULL" "httpAccept": "text/xml"
} }
err_msg = "" err_msg = ""
count_abs = 0 count_abs = 0
@ -154,7 +153,6 @@ def get_abstract_from_elsevier(number_of_task:int = 20):
res = req.get( res = req.get(
f"https://api.elsevier.com/content/article/doi/{paper.doi}", f"https://api.elsevier.com/content/article/doi/{paper.doi}",
params=params, params=params,
headers = ELSEVIER_HEADERS,
timeout=(3, 15) timeout=(3, 15)
) )
except requests.RequestException: except requests.RequestException:
@ -196,6 +194,8 @@ def get_abstract_from_elsevier(number_of_task:int = 20):
if has_fulltext: if has_fulltext:
paper.has_fulltext = True paper.has_fulltext = True
paper.has_fulltext_xml = True paper.has_fulltext_xml = True
if paper.has_fulltext_pdf is False:
save_pdf_from_elsevier(paper)
count_fulltext += 1 count_fulltext += 1
paper.save_file_xml(xml_str) paper.save_file_xml(xml_str)
@ -355,9 +355,6 @@ def save_pdf_from_oa_url(paper:Paper):
if is_pdf and len(res.content) > 1024: # 至少1KB if is_pdf and len(res.content) > 1024: # 至少1KB
paper.save_file_pdf(res.content, save_obj=True) paper.save_file_pdf(res.content, save_obj=True)
return "success" return "success"
else:
paper.save_fail_reason("oa_url_not_pdf")
return "oa_url_not_pdf"
return f"oa_url_pdf_error: {res.status_code}" return f"oa_url_pdf_error: {res.status_code}"
def save_pdf_from_openalex(paper:Paper): def save_pdf_from_openalex(paper:Paper):
@ -382,6 +379,7 @@ def save_pdf_from_openalex(paper:Paper):
def save_pdf_from_elsevier(paper:Paper): def save_pdf_from_elsevier(paper:Paper):
params = { params = {
"apiKey": ELSEVIER_APIKEY,
"httpAccept": "application/pdf" "httpAccept": "application/pdf"
} }
try: try: