"""一次性修复: 把误标为全文 PDF 的 Elsevier "摘要预览页"(1 页)纠正回未下载状态。 背景: Elsevier Article API 对未授权 / in-press 文章, application/pdf 端点会返回仅含 摘要的 1 页预览 PDF(魔数仍是 %PDF、体积也不小), 而全文 XML 却能正常拿到。旧抓取 逻辑只校验魔数 + 体积, 误将预览页落库并置 has_fulltext_pdf=True。 本命令重新核对本地 PDF 的页数, 对 <= 1 页者: - has_fulltext_pdf 置回 False - 若该论文有 XML 全文(has_fulltext_xml=True), 保留 has_fulltext=True; 否则(此前只有这张假预览页冒充全文)一并把 has_fulltext 回退为 False, 让它能重新进入下载链路去找真正的全文。 - 追加 fail_reason 'elsevier_pdf_preview_only' (供 Elsevier 补抓队列排除, 避免无限重试) - 可选: 删除本地预览 PDF 文件 (--delete-file) 文件读取依赖本地存在 PDF (在跑抓取的服务器上执行)。建议先 --dry-run 看统计。 用法: python manage.py fix_preview_pdf --dry-run python manage.py fix_preview_pdf --delete-file """ import os from django.core.management.base import BaseCommand from apps.resm.models import Paper from apps.resm.tasks import _pdf_page_count class Command(BaseCommand): help = "纠正被误标为全文的 Elsevier 摘要预览 PDF(1 页)" def add_arguments(self, parser): parser.add_argument("--dry-run", action="store_true", help="只统计, 不写库 / 不删文件") parser.add_argument("--limit", type=int, default=0, help="最多处理多少条 (0=不限)") parser.add_argument("--delete-file", action="store_true", help="同时删除本地预览 PDF 文件") def handle(self, *args, **opts): dry = opts["dry_run"] limit = opts["limit"] del_file = opts["delete_file"] qs = Paper.objects.filter( has_fulltext_pdf=True, doi__startswith="10.1016" ).order_by("id") total = qs.count() self.stdout.write( f"候选(has_fulltext_pdf=True 且 DOI 以 10.1016 开头): {total}") checked = fixed = only_pdf = missing = unreadable = 0 for paper in qs.iterator(): if limit and checked >= limit: break checked += 1 path = paper.init_paper_path("pdf") if not os.path.exists(path): missing += 1 continue try: with open(path, "rb") as f: content = f.read() except OSError: unreadable += 1 continue pages = _pdf_page_count(content) if pages is None: unreadable += 1 continue if pages > 1: continue # 真全文, 跳过 fixed += 1 only_pdf_case = not paper.has_fulltext_xml if only_pdf_case: only_pdf += 1 self.stdout.write( f"[preview {pages}p]{' (only-pdf)' if only_pdf_case else ''} " f"{paper.doi} {path}") if dry: continue paper.has_fulltext_pdf = False update_fields = ["has_fulltext_pdf", "update_time"] # 没有 XML 全文时, 之前的 has_fulltext 只是被这张假预览页置上的, 一并回退 if not paper.has_fulltext_xml: paper.has_fulltext = False update_fields.insert(0, "has_fulltext") paper.save(update_fields=update_fields) if "elsevier_pdf_preview_only" not in (paper.fail_reason or ""): paper.save_fail_reason("elsevier_pdf_preview_only") if del_file: try: os.remove(path) except OSError: pass self.stdout.write(self.style.SUCCESS( f"检查={checked} 预览页修复={fixed} (其中无XML全文/一并回退has_fulltext={only_pdf}) " f"文件缺失={missing} 无法解析={unreadable}" + (" (dry-run, 未写库)" if dry else "") ))