feat(resm): OpenAlex 元数据抓取支持书籍/书章类型
- 新增 OPENALEX_WORK_TYPES 常量(article|book|book-chapter),替换四处硬编码 type="article" 过滤 - select 字段增加 type,Paper.type 写入 OpenAlex 真实类型而非写死 article - get_paper_meta_from_openalex / backfill 增加 types 参数,断点 cache key 纳入 types(游标只对相同查询有效) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
cc6db5ffd9
commit
60cd157a40
|
|
@ -58,9 +58,12 @@ def run_async(coro):
|
|||
OPENALEX_SELECT_FIELDS = [
|
||||
"id", "doi", "title", "publication_date",
|
||||
"open_access", "authorships", "primary_location", "publication_year",
|
||||
"display_name", "content_urls"
|
||||
"display_name", "content_urls", "type"
|
||||
]
|
||||
|
||||
# OpenAlex 抓取的 work 类型,"|" 表示 OR;记录的真实 type 会写回 Paper.type
|
||||
OPENALEX_WORK_TYPES = "article|book|book-chapter"
|
||||
|
||||
|
||||
def _apply_keyword_search(pager, keywords: str, search: str):
|
||||
"""把 keywords / search 过滤条件套到 pyalex 的 pager 上。
|
||||
|
|
@ -87,7 +90,7 @@ def _build_paper_from_record(record, keywords: str, search: str) -> Paper:
|
|||
paper.o_keywords = keywords
|
||||
paper.o_search = search
|
||||
paper.source = "openalex"
|
||||
paper.type = "article"
|
||||
paper.type = (record.get("type") or "article")[:20]
|
||||
paper.openalex_id = record["id"].split("/")[-1]
|
||||
paper.doi = record["doi"].replace("https://doi.org/", "")
|
||||
paper.title = record["display_name"]
|
||||
|
|
@ -146,14 +149,16 @@ def _crawl_openalex_query(base_pager, keywords: str, search: str, cache_key: str
|
|||
|
||||
|
||||
@shared_task(base=CustomTask)
|
||||
def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None):
|
||||
def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None,
|
||||
types:str=OPENALEX_WORK_TYPES):
|
||||
if not (keywords or search):
|
||||
raise Exception("keywords or search must be provided")
|
||||
cache_key = f"openalex_cursor_{publication_year}_{keywords}{search}"
|
||||
# types 进 key:OpenAlex 游标只对完全相同的查询有效,换类型过滤必须换 checkpoint
|
||||
cache_key = f"openalex_cursor_{publication_year}_{keywords}{search}_{types}"
|
||||
base = Works().filter(
|
||||
publication_year=publication_year,
|
||||
has_doi=True,
|
||||
type="article"
|
||||
type=types
|
||||
)
|
||||
base = _apply_keyword_search(base, keywords, search)
|
||||
_crawl_openalex_query(base, keywords, search, cache_key=cache_key,
|
||||
|
|
@ -168,7 +173,8 @@ def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:s
|
|||
"publication_year": publication_year + 1,
|
||||
"keywords": keywords,
|
||||
"search": search,
|
||||
"end_year": end_year
|
||||
"end_year": end_year,
|
||||
"types": types
|
||||
},
|
||||
countdown=5
|
||||
)
|
||||
|
|
@ -186,7 +192,7 @@ def _fetch_openalex_published_since(keywords: str, search: str, from_publication
|
|||
"""
|
||||
base = Works().filter(
|
||||
has_doi=True,
|
||||
type="article",
|
||||
type=OPENALEX_WORK_TYPES,
|
||||
from_publication_date=from_publication_date,
|
||||
)
|
||||
base = _apply_keyword_search(base, keywords, search)
|
||||
|
|
@ -225,7 +231,8 @@ BACKFILL_STOP_KEY = "backfill_paper_meta_stop"
|
|||
|
||||
@shared_task(base=CustomTask)
|
||||
def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication_date: str = None,
|
||||
combo_index: int = 0, combos=None):
|
||||
combo_index: int = 0, combos=None,
|
||||
types: str = OPENALEX_WORK_TYPES):
|
||||
"""按发表日期一次性回补论文索引,支持断点续传(应对 OpenAlex 配额限制)。
|
||||
|
||||
本任务只负责策略:抓哪些查询组合、按什么发表日期区间、怎么 chain。
|
||||
|
|
@ -252,7 +259,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication
|
|||
})]
|
||||
|
||||
def ckey(kw, search):
|
||||
return f"backfill_cursor_{from_publication_date}|{to_publication_date}|{kw}|{search}"
|
||||
return f"backfill_cursor_{from_publication_date}|{to_publication_date}|{kw}|{search}|{types}"
|
||||
|
||||
# 跳过已完成的组合
|
||||
while combo_index < len(combos):
|
||||
|
|
@ -267,7 +274,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication
|
|||
kw, search = combos[combo_index]
|
||||
cursor_key = ckey(kw, search)
|
||||
base = Works().filter(
|
||||
has_doi=True, type="article", from_publication_date=from_publication_date,
|
||||
has_doi=True, type=types, from_publication_date=from_publication_date,
|
||||
)
|
||||
if to_publication_date:
|
||||
base = base.filter(to_publication_date=to_publication_date)
|
||||
|
|
@ -294,6 +301,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication
|
|||
"to_publication_date": to_publication_date,
|
||||
"combo_index": combo_index + 1,
|
||||
"combos": combos,
|
||||
"types": types,
|
||||
},
|
||||
countdown=3,
|
||||
)
|
||||
|
|
@ -317,7 +325,7 @@ def monitor_papers(monitor_id: str = None):
|
|||
results = []
|
||||
for m in qs:
|
||||
from_pub = (timezone.now() - timedelta(days=m.days or 30)).date().isoformat()
|
||||
base = Works().filter(has_doi=True, type="article", from_publication_date=from_pub)
|
||||
base = Works().filter(has_doi=True, type=OPENALEX_WORK_TYPES, from_publication_date=from_pub)
|
||||
kw, search = "", ""
|
||||
if m.type == PaperMonitor.TYPE_JOURNAL:
|
||||
base = base.filter(primary_location={"source": {"issn": m.value}})
|
||||
|
|
|
|||
Loading…
Reference in New Issue