fix(resm): reject unsafe PDF title candidates

This commit is contained in:
caoqianming 2026-08-19 08:38:26 +08:00
parent b0b26a4ab5
commit e80367414a
4 changed files with 102 additions and 3 deletions

View File

@ -181,7 +181,10 @@ Title quality is tracked on `Paper` with the original value retained only when a
is applied. `python manage.py audit_paper_titles` performs a dry-run audit by default; is applied. `python manage.py audit_paper_titles` performs a dry-run audit by default;
`--apply` writes only high-confidence XML corrections unless `--allow-medium` is explicitly `--apply` writes only high-confidence XML corrections unless `--allow-medium` is explicitly
provided for reviewed PDF candidates. The audit checks physical XML/PDF files even when provided for reviewed PDF candidates. The audit checks physical XML/PDF files even when
legacy `has_fulltext_*` flags are stale. legacy `has_fulltext_*` flags are stale. PDF first-page layout guesses always remain low
confidence and are never bulk-applied; PDF metadata also rejects filenames and generic titles.
`rollback_pdf_title_corrections` restores PDF-derived corrections from `title_raw` and is a
dry-run unless `--apply` is passed.
Download behavior is stateful: Download behavior is stateful:

View File

@ -0,0 +1,54 @@
"""回滚误用 PDF 候选矫正的标题。"""
from django.core.management.base import BaseCommand
from django.utils import timezone
from apps.resm.models import Paper
from apps.resm.title_utils import assess_title
class Command(BaseCommand):
help = "将 PDF 自动矫正标题恢复为 title_raw默认 dry-run"
def add_arguments(self, parser):
parser.add_argument("--doi", action="append", default=[], help="只回滚指定 DOI可重复")
parser.add_argument("--keep-doi", action="append", default=[], help="保留的 DOI可重复")
parser.add_argument("--apply", action="store_true", help="实际回滚;默认只预览")
def handle(self, *args, **opts):
keep = [doi.strip().lower() for doi in opts["keep_doi"] if doi.strip()]
selected = [doi.strip().lower() for doi in opts["doi"] if doi.strip()]
qs = Paper.objects.filter(
title_quality_status=Paper.TITLE_CORRECTED,
title_source__in=("pdf_metadata", "pdf_first_page"),
title_raw__isnull=False,
).exclude(title_raw="").order_by("id")
if selected:
qs = qs.filter(doi__in=selected)
if keep:
qs = qs.exclude(doi__in=keep)
count = 0
for paper in qs.iterator(chunk_size=500):
count += 1
self.stdout.write(
f"[rollback] {paper.doi} {paper.title!r} -> {paper.title_raw!r}"
)
if not opts["apply"]:
continue
restored = paper.title_raw
paper.title = restored
paper.title_raw = None
paper.title_source = None
paper.title_quality_status = (
Paper.TITLE_SUSPECT if assess_title(restored).suspect
else Paper.TITLE_UNCHECKED
)
paper.title_verified_at = None
paper.update_time = timezone.now()
paper.save(update_fields=[
"title", "title_raw", "title_source", "title_quality_status",
"title_verified_at", "update_time",
])
mode = "apply" if opts["apply"] else "dry-run"
self.stdout.write(self.style.SUCCESS(f"完成({mode}) rollback={count}"))

View File

@ -97,6 +97,30 @@ class PdfTitleExtractionTests(unittest.TestCase):
self.assertEqual(candidate.confidence, "medium") self.assertEqual(candidate.confidence, "medium")
self.assertEqual(candidate.evidence, "pdf_metadata+doi") self.assertEqual(candidate.evidence, "pdf_metadata+doi")
def test_rejects_layout_filename_and_keeps_first_page_low_confidence(self):
handle = tempfile.NamedTemporaryFile("wb", suffix=".pdf", delete=False)
self.addCleanup(lambda: os.path.exists(handle.name) and os.unlink(handle.name))
handle.write(b"%PDF-placeholder")
handle.close()
page = types.SimpleNamespace(
extract_text=lambda: (
"Correct Article Title\nMohamed Shamseldin\n"
"https://doi.org/10.1234/example.1\nAbstract"
)
)
reader = types.SimpleNamespace(
pages=[page], metadata=types.SimpleNamespace(title="article-layout.indd")
)
module = types.SimpleNamespace(PdfReader=lambda *args, **kwargs: reader)
with patch.dict("sys.modules", {"pypdf": module}):
candidate = extract_title_from_pdf(
handle.name, "10.1234/example.1", "Mohamed Shamseldin"
)
self.assertIsNotNone(candidate)
self.assertEqual(candidate.source, "pdf_first_page")
self.assertEqual(candidate.confidence, "low")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

View File

@ -25,7 +25,9 @@ _WEB_MARKERS = (
_DOCUMENT_MARKERS = ("author:", "authors:", "abstract:", "document type:") _DOCUMENT_MARKERS = ("author:", "authors:", "abstract:", "document type:")
_GENERIC_PDF_TITLES = { _GENERIC_PDF_TITLES = {
"untitled", "microsoft word", "document", "article", "full text", "pdf", "untitled", "microsoft word", "document", "article", "full text", "pdf",
"no job name",
} }
_FILE_TITLE_RE = re.compile(r"(?:\.eps|\.indd|\.docx?|\.pdf|\.tex)$", re.I)
@dataclass(frozen=True) @dataclass(frozen=True)
@ -106,6 +108,21 @@ def is_usable_title(value: Optional[str]) -> bool:
return True return True
def _is_usable_pdf_metadata_title(value: Optional[str]) -> bool:
"""PDF Title 属性经常是排版文件名,使用比普通标题更严格的门槛。"""
title = clean_title(value)
assessment = assess_title(title)
if not is_usable_title(title):
return False
if assessment.plain_length < 15 or assessment.word_count < 3:
return False
if title.casefold() in _GENERIC_PDF_TITLES or _FILE_TITLE_RE.search(title):
return False
if re.fullmatch(r"(?:issn\s*)?[\dXx-]{7,}", title):
return False
return True
def is_better_title(current: Optional[str], candidate: Optional[str]) -> bool: def is_better_title(current: Optional[str], candidate: Optional[str]) -> bool:
old = assess_title(current) old = assess_title(current)
new = assess_title(candidate) new = assess_title(candidate)
@ -237,7 +254,7 @@ def extract_title_from_pdf(
identity = _identity_evidence(page_text, expected_doi or "", first_author or "") identity = _identity_evidence(page_text, expected_doi or "", first_author or "")
meta_title = clean_title(getattr(metadata, "title", "") if metadata else "") meta_title = clean_title(getattr(metadata, "title", "") if metadata else "")
if meta_title and meta_title.casefold() not in _GENERIC_PDF_TITLES and is_usable_title(meta_title): if _is_usable_pdf_metadata_title(meta_title):
return FulltextTitleCandidate( return FulltextTitleCandidate(
title=meta_title, title=meta_title,
source="pdf_metadata", source="pdf_metadata",
@ -250,7 +267,8 @@ def extract_title_from_pdf(
return FulltextTitleCandidate( return FulltextTitleCandidate(
title=page_title, title=page_title,
source="pdf_first_page", source="pdf_first_page",
confidence="medium" if identity else "low", # 首页文本顺序受版式影响很大,只输出人工复核候选,绝不批量自动采用。
confidence="low",
evidence=f"pdf_first_page+{identity}" if identity else "pdf_first_page_only", evidence=f"pdf_first_page+{identity}" if identity else "pdf_first_page_only",
) )
return None return None