feat(resm): OpenAlex 元数据抓取支持书籍/书章类型
- 新增 OPENALEX_WORK_TYPES 常量(article|book|book-chapter),替换四处硬编码 type="article" 过滤 - select 字段增加 type,Paper.type 写入 OpenAlex 真实类型而非写死 article - get_paper_meta_from_openalex / backfill 增加 types 参数,断点 cache key 纳入 types(游标只对相同查询有效) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
cc6db5ffd9
commit
60cd157a40
|
|
@ -58,9 +58,12 @@ def run_async(coro):
|
||||||
OPENALEX_SELECT_FIELDS = [
|
OPENALEX_SELECT_FIELDS = [
|
||||||
"id", "doi", "title", "publication_date",
|
"id", "doi", "title", "publication_date",
|
||||||
"open_access", "authorships", "primary_location", "publication_year",
|
"open_access", "authorships", "primary_location", "publication_year",
|
||||||
"display_name", "content_urls"
|
"display_name", "content_urls", "type"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# OpenAlex 抓取的 work 类型,"|" 表示 OR;记录的真实 type 会写回 Paper.type
|
||||||
|
OPENALEX_WORK_TYPES = "article|book|book-chapter"
|
||||||
|
|
||||||
|
|
||||||
def _apply_keyword_search(pager, keywords: str, search: str):
|
def _apply_keyword_search(pager, keywords: str, search: str):
|
||||||
"""把 keywords / search 过滤条件套到 pyalex 的 pager 上。
|
"""把 keywords / search 过滤条件套到 pyalex 的 pager 上。
|
||||||
|
|
@ -87,7 +90,7 @@ def _build_paper_from_record(record, keywords: str, search: str) -> Paper:
|
||||||
paper.o_keywords = keywords
|
paper.o_keywords = keywords
|
||||||
paper.o_search = search
|
paper.o_search = search
|
||||||
paper.source = "openalex"
|
paper.source = "openalex"
|
||||||
paper.type = "article"
|
paper.type = (record.get("type") or "article")[:20]
|
||||||
paper.openalex_id = record["id"].split("/")[-1]
|
paper.openalex_id = record["id"].split("/")[-1]
|
||||||
paper.doi = record["doi"].replace("https://doi.org/", "")
|
paper.doi = record["doi"].replace("https://doi.org/", "")
|
||||||
paper.title = record["display_name"]
|
paper.title = record["display_name"]
|
||||||
|
|
@ -146,14 +149,16 @@ def _crawl_openalex_query(base_pager, keywords: str, search: str, cache_key: str
|
||||||
|
|
||||||
|
|
||||||
@shared_task(base=CustomTask)
|
@shared_task(base=CustomTask)
|
||||||
def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None):
|
def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:str="", end_year:int=None,
|
||||||
|
types:str=OPENALEX_WORK_TYPES):
|
||||||
if not (keywords or search):
|
if not (keywords or search):
|
||||||
raise Exception("keywords or search must be provided")
|
raise Exception("keywords or search must be provided")
|
||||||
cache_key = f"openalex_cursor_{publication_year}_{keywords}{search}"
|
# types 进 key:OpenAlex 游标只对完全相同的查询有效,换类型过滤必须换 checkpoint
|
||||||
|
cache_key = f"openalex_cursor_{publication_year}_{keywords}{search}_{types}"
|
||||||
base = Works().filter(
|
base = Works().filter(
|
||||||
publication_year=publication_year,
|
publication_year=publication_year,
|
||||||
has_doi=True,
|
has_doi=True,
|
||||||
type="article"
|
type=types
|
||||||
)
|
)
|
||||||
base = _apply_keyword_search(base, keywords, search)
|
base = _apply_keyword_search(base, keywords, search)
|
||||||
_crawl_openalex_query(base, keywords, search, cache_key=cache_key,
|
_crawl_openalex_query(base, keywords, search, cache_key=cache_key,
|
||||||
|
|
@ -168,7 +173,8 @@ def get_paper_meta_from_openalex(publication_year:int, keywords:str="", search:s
|
||||||
"publication_year": publication_year + 1,
|
"publication_year": publication_year + 1,
|
||||||
"keywords": keywords,
|
"keywords": keywords,
|
||||||
"search": search,
|
"search": search,
|
||||||
"end_year": end_year
|
"end_year": end_year,
|
||||||
|
"types": types
|
||||||
},
|
},
|
||||||
countdown=5
|
countdown=5
|
||||||
)
|
)
|
||||||
|
|
@ -186,7 +192,7 @@ def _fetch_openalex_published_since(keywords: str, search: str, from_publication
|
||||||
"""
|
"""
|
||||||
base = Works().filter(
|
base = Works().filter(
|
||||||
has_doi=True,
|
has_doi=True,
|
||||||
type="article",
|
type=OPENALEX_WORK_TYPES,
|
||||||
from_publication_date=from_publication_date,
|
from_publication_date=from_publication_date,
|
||||||
)
|
)
|
||||||
base = _apply_keyword_search(base, keywords, search)
|
base = _apply_keyword_search(base, keywords, search)
|
||||||
|
|
@ -225,7 +231,8 @@ BACKFILL_STOP_KEY = "backfill_paper_meta_stop"
|
||||||
|
|
||||||
@shared_task(base=CustomTask)
|
@shared_task(base=CustomTask)
|
||||||
def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication_date: str = None,
|
def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication_date: str = None,
|
||||||
combo_index: int = 0, combos=None):
|
combo_index: int = 0, combos=None,
|
||||||
|
types: str = OPENALEX_WORK_TYPES):
|
||||||
"""按发表日期一次性回补论文索引,支持断点续传(应对 OpenAlex 配额限制)。
|
"""按发表日期一次性回补论文索引,支持断点续传(应对 OpenAlex 配额限制)。
|
||||||
|
|
||||||
本任务只负责策略:抓哪些查询组合、按什么发表日期区间、怎么 chain。
|
本任务只负责策略:抓哪些查询组合、按什么发表日期区间、怎么 chain。
|
||||||
|
|
@ -252,7 +259,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication
|
||||||
})]
|
})]
|
||||||
|
|
||||||
def ckey(kw, search):
|
def ckey(kw, search):
|
||||||
return f"backfill_cursor_{from_publication_date}|{to_publication_date}|{kw}|{search}"
|
return f"backfill_cursor_{from_publication_date}|{to_publication_date}|{kw}|{search}|{types}"
|
||||||
|
|
||||||
# 跳过已完成的组合
|
# 跳过已完成的组合
|
||||||
while combo_index < len(combos):
|
while combo_index < len(combos):
|
||||||
|
|
@ -267,7 +274,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication
|
||||||
kw, search = combos[combo_index]
|
kw, search = combos[combo_index]
|
||||||
cursor_key = ckey(kw, search)
|
cursor_key = ckey(kw, search)
|
||||||
base = Works().filter(
|
base = Works().filter(
|
||||||
has_doi=True, type="article", from_publication_date=from_publication_date,
|
has_doi=True, type=types, from_publication_date=from_publication_date,
|
||||||
)
|
)
|
||||||
if to_publication_date:
|
if to_publication_date:
|
||||||
base = base.filter(to_publication_date=to_publication_date)
|
base = base.filter(to_publication_date=to_publication_date)
|
||||||
|
|
@ -294,6 +301,7 @@ def backfill_paper_meta_from_openalex(from_publication_date: str, to_publication
|
||||||
"to_publication_date": to_publication_date,
|
"to_publication_date": to_publication_date,
|
||||||
"combo_index": combo_index + 1,
|
"combo_index": combo_index + 1,
|
||||||
"combos": combos,
|
"combos": combos,
|
||||||
|
"types": types,
|
||||||
},
|
},
|
||||||
countdown=3,
|
countdown=3,
|
||||||
)
|
)
|
||||||
|
|
@ -317,7 +325,7 @@ def monitor_papers(monitor_id: str = None):
|
||||||
results = []
|
results = []
|
||||||
for m in qs:
|
for m in qs:
|
||||||
from_pub = (timezone.now() - timedelta(days=m.days or 30)).date().isoformat()
|
from_pub = (timezone.now() - timedelta(days=m.days or 30)).date().isoformat()
|
||||||
base = Works().filter(has_doi=True, type="article", from_publication_date=from_pub)
|
base = Works().filter(has_doi=True, type=OPENALEX_WORK_TYPES, from_publication_date=from_pub)
|
||||||
kw, search = "", ""
|
kw, search = "", ""
|
||||||
if m.type == PaperMonitor.TYPE_JOURNAL:
|
if m.type == PaperMonitor.TYPE_JOURNAL:
|
||||||
base = base.filter(primary_location={"source": {"issn": m.value}})
|
base = base.filter(primary_location={"source": {"issn": m.value}})
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue