fix(resm): reject unsafe PDF title candidates
This commit is contained in:
parent
b0b26a4ab5
commit
e80367414a
|
|
@ -181,7 +181,10 @@ Title quality is tracked on `Paper` with the original value retained only when a
|
||||||
is applied. `python manage.py audit_paper_titles` performs a dry-run audit by default;
|
is applied. `python manage.py audit_paper_titles` performs a dry-run audit by default;
|
||||||
`--apply` writes only high-confidence XML corrections unless `--allow-medium` is explicitly
|
`--apply` writes only high-confidence XML corrections unless `--allow-medium` is explicitly
|
||||||
provided for reviewed PDF candidates. The audit checks physical XML/PDF files even when
|
provided for reviewed PDF candidates. The audit checks physical XML/PDF files even when
|
||||||
legacy `has_fulltext_*` flags are stale.
|
legacy `has_fulltext_*` flags are stale. PDF first-page layout guesses always remain low
|
||||||
|
confidence and are never bulk-applied; PDF metadata also rejects filenames and generic titles.
|
||||||
|
`rollback_pdf_title_corrections` restores PDF-derived corrections from `title_raw` and is a
|
||||||
|
dry-run unless `--apply` is passed.
|
||||||
|
|
||||||
Download behavior is stateful:
|
Download behavior is stateful:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
"""回滚误用 PDF 候选矫正的标题。"""
|
||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from apps.resm.models import Paper
|
||||||
|
from apps.resm.title_utils import assess_title
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = "将 PDF 自动矫正标题恢复为 title_raw;默认 dry-run"
|
||||||
|
|
||||||
|
def add_arguments(self, parser):
|
||||||
|
parser.add_argument("--doi", action="append", default=[], help="只回滚指定 DOI,可重复")
|
||||||
|
parser.add_argument("--keep-doi", action="append", default=[], help="保留的 DOI,可重复")
|
||||||
|
parser.add_argument("--apply", action="store_true", help="实际回滚;默认只预览")
|
||||||
|
|
||||||
|
def handle(self, *args, **opts):
|
||||||
|
keep = [doi.strip().lower() for doi in opts["keep_doi"] if doi.strip()]
|
||||||
|
selected = [doi.strip().lower() for doi in opts["doi"] if doi.strip()]
|
||||||
|
qs = Paper.objects.filter(
|
||||||
|
title_quality_status=Paper.TITLE_CORRECTED,
|
||||||
|
title_source__in=("pdf_metadata", "pdf_first_page"),
|
||||||
|
title_raw__isnull=False,
|
||||||
|
).exclude(title_raw="").order_by("id")
|
||||||
|
if selected:
|
||||||
|
qs = qs.filter(doi__in=selected)
|
||||||
|
if keep:
|
||||||
|
qs = qs.exclude(doi__in=keep)
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
for paper in qs.iterator(chunk_size=500):
|
||||||
|
count += 1
|
||||||
|
self.stdout.write(
|
||||||
|
f"[rollback] {paper.doi} {paper.title!r} -> {paper.title_raw!r}"
|
||||||
|
)
|
||||||
|
if not opts["apply"]:
|
||||||
|
continue
|
||||||
|
restored = paper.title_raw
|
||||||
|
paper.title = restored
|
||||||
|
paper.title_raw = None
|
||||||
|
paper.title_source = None
|
||||||
|
paper.title_quality_status = (
|
||||||
|
Paper.TITLE_SUSPECT if assess_title(restored).suspect
|
||||||
|
else Paper.TITLE_UNCHECKED
|
||||||
|
)
|
||||||
|
paper.title_verified_at = None
|
||||||
|
paper.update_time = timezone.now()
|
||||||
|
paper.save(update_fields=[
|
||||||
|
"title", "title_raw", "title_source", "title_quality_status",
|
||||||
|
"title_verified_at", "update_time",
|
||||||
|
])
|
||||||
|
|
||||||
|
mode = "apply" if opts["apply"] else "dry-run"
|
||||||
|
self.stdout.write(self.style.SUCCESS(f"完成({mode}) rollback={count}"))
|
||||||
|
|
@ -97,6 +97,30 @@ class PdfTitleExtractionTests(unittest.TestCase):
|
||||||
self.assertEqual(candidate.confidence, "medium")
|
self.assertEqual(candidate.confidence, "medium")
|
||||||
self.assertEqual(candidate.evidence, "pdf_metadata+doi")
|
self.assertEqual(candidate.evidence, "pdf_metadata+doi")
|
||||||
|
|
||||||
|
def test_rejects_layout_filename_and_keeps_first_page_low_confidence(self):
|
||||||
|
handle = tempfile.NamedTemporaryFile("wb", suffix=".pdf", delete=False)
|
||||||
|
self.addCleanup(lambda: os.path.exists(handle.name) and os.unlink(handle.name))
|
||||||
|
handle.write(b"%PDF-placeholder")
|
||||||
|
handle.close()
|
||||||
|
|
||||||
|
page = types.SimpleNamespace(
|
||||||
|
extract_text=lambda: (
|
||||||
|
"Correct Article Title\nMohamed Shamseldin\n"
|
||||||
|
"https://doi.org/10.1234/example.1\nAbstract"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
reader = types.SimpleNamespace(
|
||||||
|
pages=[page], metadata=types.SimpleNamespace(title="article-layout.indd")
|
||||||
|
)
|
||||||
|
module = types.SimpleNamespace(PdfReader=lambda *args, **kwargs: reader)
|
||||||
|
with patch.dict("sys.modules", {"pypdf": module}):
|
||||||
|
candidate = extract_title_from_pdf(
|
||||||
|
handle.name, "10.1234/example.1", "Mohamed Shamseldin"
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(candidate)
|
||||||
|
self.assertEqual(candidate.source, "pdf_first_page")
|
||||||
|
self.assertEqual(candidate.confidence, "low")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,9 @@ _WEB_MARKERS = (
|
||||||
_DOCUMENT_MARKERS = ("author:", "authors:", "abstract:", "document type:")
|
_DOCUMENT_MARKERS = ("author:", "authors:", "abstract:", "document type:")
|
||||||
_GENERIC_PDF_TITLES = {
|
_GENERIC_PDF_TITLES = {
|
||||||
"untitled", "microsoft word", "document", "article", "full text", "pdf",
|
"untitled", "microsoft word", "document", "article", "full text", "pdf",
|
||||||
|
"no job name",
|
||||||
}
|
}
|
||||||
|
_FILE_TITLE_RE = re.compile(r"(?:\.eps|\.indd|\.docx?|\.pdf|\.tex)$", re.I)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|
@ -106,6 +108,21 @@ def is_usable_title(value: Optional[str]) -> bool:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _is_usable_pdf_metadata_title(value: Optional[str]) -> bool:
|
||||||
|
"""PDF Title 属性经常是排版文件名,使用比普通标题更严格的门槛。"""
|
||||||
|
title = clean_title(value)
|
||||||
|
assessment = assess_title(title)
|
||||||
|
if not is_usable_title(title):
|
||||||
|
return False
|
||||||
|
if assessment.plain_length < 15 or assessment.word_count < 3:
|
||||||
|
return False
|
||||||
|
if title.casefold() in _GENERIC_PDF_TITLES or _FILE_TITLE_RE.search(title):
|
||||||
|
return False
|
||||||
|
if re.fullmatch(r"(?:issn\s*)?[\dXx-]{7,}", title):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def is_better_title(current: Optional[str], candidate: Optional[str]) -> bool:
|
def is_better_title(current: Optional[str], candidate: Optional[str]) -> bool:
|
||||||
old = assess_title(current)
|
old = assess_title(current)
|
||||||
new = assess_title(candidate)
|
new = assess_title(candidate)
|
||||||
|
|
@ -237,7 +254,7 @@ def extract_title_from_pdf(
|
||||||
|
|
||||||
identity = _identity_evidence(page_text, expected_doi or "", first_author or "")
|
identity = _identity_evidence(page_text, expected_doi or "", first_author or "")
|
||||||
meta_title = clean_title(getattr(metadata, "title", "") if metadata else "")
|
meta_title = clean_title(getattr(metadata, "title", "") if metadata else "")
|
||||||
if meta_title and meta_title.casefold() not in _GENERIC_PDF_TITLES and is_usable_title(meta_title):
|
if _is_usable_pdf_metadata_title(meta_title):
|
||||||
return FulltextTitleCandidate(
|
return FulltextTitleCandidate(
|
||||||
title=meta_title,
|
title=meta_title,
|
||||||
source="pdf_metadata",
|
source="pdf_metadata",
|
||||||
|
|
@ -250,7 +267,8 @@ def extract_title_from_pdf(
|
||||||
return FulltextTitleCandidate(
|
return FulltextTitleCandidate(
|
||||||
title=page_title,
|
title=page_title,
|
||||||
source="pdf_first_page",
|
source="pdf_first_page",
|
||||||
confidence="medium" if identity else "low",
|
# 首页文本顺序受版式影响很大,只输出人工复核候选,绝不批量自动采用。
|
||||||
|
confidence="low",
|
||||||
evidence=f"pdf_first_page+{identity}" if identity else "pdf_first_page_only",
|
evidence=f"pdf_first_page+{identity}" if identity else "pdf_first_page_only",
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue