From 60cd157a40c14c3b3f95dd194cd7aad607634c26 Mon Sep 17 00:00:00 2001 From: caoqianming Date: Mon, 6 Jul 2026 14:30:54 +0800 Subject: [PATCH] =?UTF-8?q?feat(resm):=20OpenAlex=20=E5=85=83=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=8A=93=E5=8F=96=E6=94=AF=E6=8C=81=E4=B9=A6=E7=B1=8D?= =?UTF-8?q?/=E4=B9=A6=E7=AB=A0=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 OPENALEX_WORK_TYPES 常量(article|book|book-chapter),替换四处硬编码 type="article" 过滤 - select 字段增加 type,Paper.type 写入 OpenAlex 真实类型而非写死 article - get_paper_meta_from_openalex / backfill 增加 types 参数,断点 cache key 纳入 types(游标只对相同查询有效) Co-Authored-By: Claude Fable 5 --- apps/resm/tasks.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/apps/resm/tasks.py b/apps/resm/tasks.py index cb54933..2d4c9eb 100644 --- a/apps/resm/tasks.py +++ b/apps/resm/tasks.py @@ -58,9 +58,12 @@ def run_async(coro): OPENALEX_SELECT_FIELDS = [ "id", "doi", "title", "publication_date", "open_access", "authorships", "primary_location", "publication_year", - "display_name", "content_urls" + "display_name", "content_urls", "type" ] +# OpenAlex 抓取的 work 类型,"|" 表示 OR;记录的真实 type 会写回 Paper.type +OPENALEX_WORK_TYPES = "article|book|book-chapter" + def _apply_keyword_search(pager, keywords: str, search: str): """把 keywords / search 过滤条件套到 pyalex 的 pager 上。 @@ -87,7 +90,7 @@ def _build_paper_from_record(record, keywords: str, search: str) -> Paper: paper.o_keywords = keywords paper.o_search = search paper.source = "openalex" - paper.type = "article" + paper.type = (record.get("type") or "article")[:20] paper.openalex_id = record["id"].split("/")[-1] paper.doi = record["doi"].replace("https://doi.org/", "") paper.title = record["display_name"] @@ -146,14 +149,16 @@ def _crawl_openalex_query(base_pager, keywords: str, search: str, cache_key: str @shared_task(base=CustomTask) -def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None): +def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None, + types:str=OPENALEX_WORK_TYPES): if not (keywords or search): raise Exception("keywords or search must be provided") - cache_key = f"openalex_cursor_{publication_year}_{keywords}{search}" + # types 进 key:OpenAlex 游标只对完全相同的查询有效,换类型过滤必须换 checkpoint + cache_key = f"openalex_cursor_{publication_year}_{keywords}{search}_{types}" base = Works().filter( publication_year=publication_year, has_doi=True, - type="article" + type=types ) base = _apply_keyword_search(base, keywords, search) _crawl_openalex_query(base, keywords, search, cache_key=cache_key, @@ -168,7 +173,8 @@ def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:s "publication_year": publication_year + 1, "keywords": keywords, "search": search, - "end_year": end_year + "end_year": end_year, + "types": types }, countdown=5 ) @@ -186,7 +192,7 @@ def _fetch_openalex_published_since(keywords: str, search: str, from_publication """ base = Works().filter( has_doi=True, - type="article", + type=OPENALEX_WORK_TYPES, from_publication_date=from_publication_date, ) base = _apply_keyword_search(base, keywords, search) @@ -225,7 +231,8 @@ BACKFILL_STOP_KEY = "backfill_paper_meta_stop" @shared_task(base=CustomTask) def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication_date: str = None, - combo_index: int = 0, combos=None): + combo_index: int = 0, combos=None, + types: str = OPENALEX_WORK_TYPES): """按发表日期一次性回补论文索引,支持断点续传(应对 OpenAlex 配额限制)。 本任务只负责策略:抓哪些查询组合、按什么发表日期区间、怎么 chain。 @@ -252,7 +259,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication })] def ckey(kw, search): - return f"backfill_cursor_{from_publication_date}|{to_publication_date}|{kw}|{search}" + return f"backfill_cursor_{from_publication_date}|{to_publication_date}|{kw}|{search}|{types}" # 跳过已完成的组合 while combo_index < len(combos): @@ -267,7 +274,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication kw, search = combos[combo_index] cursor_key = ckey(kw, search) base = Works().filter( - has_doi=True, type="article", from_publication_date=from_publication_date, + has_doi=True, type=types, from_publication_date=from_publication_date, ) if to_publication_date: base = base.filter(to_publication_date=to_publication_date) @@ -294,6 +301,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication "to_publication_date": to_publication_date, "combo_index": combo_index + 1, "combos": combos, + "types": types, }, countdown=3, ) @@ -317,7 +325,7 @@ def monitor_papers(monitor_id: str = None): results = [] for m in qs: from_pub = (timezone.now() - timedelta(days=m.days or 30)).date().isoformat() - base = Works().filter(has_doi=True, type="article", from_publication_date=from_pub) + base = Works().filter(has_doi=True, type=OPENALEX_WORK_TYPES, from_publication_date=from_pub) kw, search = "", "" if m.type == PaperMonitor.TYPE_JOURNAL: base = base.filter(primary_location={"source": {"issn": m.value}})