From e80367414a9428308bdc31884791ff7e770b2824 Mon Sep 17 00:00:00 2001 From: caoqianming Date: Wed, 19 Aug 2026 08:38:26 +0800 Subject: [PATCH] fix(resm): reject unsafe PDF title candidates --- CLAUDE.md | 5 +- .../rollback_pdf_title_corrections.py | 54 +++++++++++++++++++ apps/resm/test_title_utils.py | 24 +++++++++ apps/resm/title_utils.py | 22 +++++++- 4 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 apps/resm/management/commands/rollback_pdf_title_corrections.py diff --git a/CLAUDE.md b/CLAUDE.md index 59ad25e..ae4d35f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -181,7 +181,10 @@ Title quality is tracked on `Paper` with the original value retained only when a is applied. `python manage.py audit_paper_titles` performs a dry-run audit by default; `--apply` writes only high-confidence XML corrections unless `--allow-medium` is explicitly provided for reviewed PDF candidates. The audit checks physical XML/PDF files even when -legacy `has_fulltext_*` flags are stale. +legacy `has_fulltext_*` flags are stale. PDF first-page layout guesses always remain low +confidence and are never bulk-applied; PDF metadata also rejects filenames and generic titles. +`rollback_pdf_title_corrections` restores PDF-derived corrections from `title_raw` and is a +dry-run unless `--apply` is passed. Download behavior is stateful: diff --git a/apps/resm/management/commands/rollback_pdf_title_corrections.py b/apps/resm/management/commands/rollback_pdf_title_corrections.py new file mode 100644 index 0000000..638430e --- /dev/null +++ b/apps/resm/management/commands/rollback_pdf_title_corrections.py @@ -0,0 +1,54 @@ +"""回滚误用 PDF 候选矫正的标题。""" +from django.core.management.base import BaseCommand +from django.utils import timezone + +from apps.resm.models import Paper +from apps.resm.title_utils import assess_title + + +class Command(BaseCommand): + help = "将 PDF 自动矫正标题恢复为 title_raw；默认 dry-run" + + def add_arguments(self, parser): + parser.add_argument("--doi", action="append", default=[], help="只回滚指定 DOI，可重复") + parser.add_argument("--keep-doi", action="append", default=[], help="保留的 DOI，可重复") + parser.add_argument("--apply", action="store_true", help="实际回滚；默认只预览") + + def handle(self, *args, **opts): + keep = [doi.strip().lower() for doi in opts["keep_doi"] if doi.strip()] + selected = [doi.strip().lower() for doi in opts["doi"] if doi.strip()] + qs = Paper.objects.filter( + title_quality_status=Paper.TITLE_CORRECTED, + title_source__in=("pdf_metadata", "pdf_first_page"), + title_raw__isnull=False, + ).exclude(title_raw="").order_by("id") + if selected: + qs = qs.filter(doi__in=selected) + if keep: + qs = qs.exclude(doi__in=keep) + + count = 0 + for paper in qs.iterator(chunk_size=500): + count += 1 + self.stdout.write( + f"[rollback] {paper.doi} {paper.title!r} -> {paper.title_raw!r}" + ) + if not opts["apply"]: + continue + restored = paper.title_raw + paper.title = restored + paper.title_raw = None + paper.title_source = None + paper.title_quality_status = ( + Paper.TITLE_SUSPECT if assess_title(restored).suspect + else Paper.TITLE_UNCHECKED + ) + paper.title_verified_at = None + paper.update_time = timezone.now() + paper.save(update_fields=[ + "title", "title_raw", "title_source", "title_quality_status", + "title_verified_at", "update_time", + ]) + + mode = "apply" if opts["apply"] else "dry-run" + self.stdout.write(self.style.SUCCESS(f"完成({mode}) rollback={count}")) diff --git a/apps/resm/test_title_utils.py b/apps/resm/test_title_utils.py index 2066291..6c2aa8d 100644 --- a/apps/resm/test_title_utils.py +++ b/apps/resm/test_title_utils.py @@ -97,6 +97,30 @@ class PdfTitleExtractionTests(unittest.TestCase): self.assertEqual(candidate.confidence, "medium") self.assertEqual(candidate.evidence, "pdf_metadata+doi") + def test_rejects_layout_filename_and_keeps_first_page_low_confidence(self): + handle = tempfile.NamedTemporaryFile("wb", suffix=".pdf", delete=False) + self.addCleanup(lambda: os.path.exists(handle.name) and os.unlink(handle.name)) + handle.write(b"%PDF-placeholder") + handle.close() + + page = types.SimpleNamespace( + extract_text=lambda: ( + "Correct Article Title\nMohamed Shamseldin\n" + "https://doi.org/10.1234/example.1\nAbstract" + ) + ) + reader = types.SimpleNamespace( + pages=[page], metadata=types.SimpleNamespace(title="article-layout.indd") + ) + module = types.SimpleNamespace(PdfReader=lambda *args, **kwargs: reader) + with patch.dict("sys.modules", {"pypdf": module}): + candidate = extract_title_from_pdf( + handle.name, "10.1234/example.1", "Mohamed Shamseldin" + ) + self.assertIsNotNone(candidate) + self.assertEqual(candidate.source, "pdf_first_page") + self.assertEqual(candidate.confidence, "low") + if __name__ == "__main__": unittest.main() diff --git a/apps/resm/title_utils.py b/apps/resm/title_utils.py index d7200dd..18492c2 100644 --- a/apps/resm/title_utils.py +++ b/apps/resm/title_utils.py @@ -25,7 +25,9 @@ _WEB_MARKERS = ( _DOCUMENT_MARKERS = ("author:", "authors:", "abstract:", "document type:") _GENERIC_PDF_TITLES = { "untitled", "microsoft word", "document", "article", "full text", "pdf", + "no job name", } +_FILE_TITLE_RE = re.compile(r"(?:\.eps|\.indd|\.docx?|\.pdf|\.tex)$", re.I) @dataclass(frozen=True) @@ -106,6 +108,21 @@ def is_usable_title(value: Optional[str]) -> bool: return True +def _is_usable_pdf_metadata_title(value: Optional[str]) -> bool: + """PDF Title 属性经常是排版文件名，使用比普通标题更严格的门槛。""" + title = clean_title(value) + assessment = assess_title(title) + if not is_usable_title(title): + return False + if assessment.plain_length < 15 or assessment.word_count < 3: + return False + if title.casefold() in _GENERIC_PDF_TITLES or _FILE_TITLE_RE.search(title): + return False + if re.fullmatch(r"(?:issn\s*)?[\dXx-]{7,}", title): + return False + return True + + def is_better_title(current: Optional[str], candidate: Optional[str]) -> bool: old = assess_title(current) new = assess_title(candidate) @@ -237,7 +254,7 @@ def extract_title_from_pdf( identity = _identity_evidence(page_text, expected_doi or "", first_author or "") meta_title = clean_title(getattr(metadata, "title", "") if metadata else "") - if meta_title and meta_title.casefold() not in _GENERIC_PDF_TITLES and is_usable_title(meta_title): + if _is_usable_pdf_metadata_title(meta_title): return FulltextTitleCandidate( title=meta_title, source="pdf_metadata", @@ -250,7 +267,8 @@ def extract_title_from_pdf( return FulltextTitleCandidate( title=page_title, source="pdf_first_page", - confidence="medium" if identity else "low", + # 首页文本顺序受版式影响很大，只输出人工复核候选，绝不批量自动采用。 + confidence="low", evidence=f"pdf_first_page+{identity}" if identity else "pdf_first_page_only", ) return None