feat(resm): correct suspect titles from fulltext
This commit is contained in:
parent
3ab18f3ed9
commit
43e8af74a4
|
|
@ -174,6 +174,13 @@ The paper fetch pipeline in `apps/resm/tasks.py` currently includes:
|
||||||
- PDF fetch from Elsevier
|
- PDF fetch from Elsevier
|
||||||
- Sci-Hub fallback
|
- Sci-Hub fallback
|
||||||
- task fan-out and stuck-download release
|
- task fan-out and stuck-download release
|
||||||
|
- suspicious-title detection, targeted OpenAlex title refresh, and high-confidence
|
||||||
|
correction from fulltext XML when the XML DOI matches the paper
|
||||||
|
|
||||||
|
Title quality is tracked on `Paper` with the original value retained only when a correction
|
||||||
|
is applied. `python manage.py audit_paper_titles` performs a dry-run audit by default;
|
||||||
|
`--apply` writes only high-confidence XML corrections unless `--allow-medium` is explicitly
|
||||||
|
provided for reviewed PDF candidates.
|
||||||
|
|
||||||
Download behavior is stateful:
|
Download behavior is stateful:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,176 @@
|
||||||
|
"""审计异常标题，并可用全文中的标题安全矫正。"""
|
||||||
|
import csv
|
||||||
|
import os
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
from django.db.models.functions import Length
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from apps.resm.models import Paper
|
||||||
|
from apps.resm.title_utils import (
|
||||||
|
FulltextTitleCandidate,
|
||||||
|
assess_title,
|
||||||
|
clean_title,
|
||||||
|
extract_title_from_pdf,
|
||||||
|
extract_title_from_xml,
|
||||||
|
is_better_title,
|
||||||
|
is_usable_title,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _paper_path(paper, ext):
|
||||||
|
safe_doi = paper.doi.replace("/", "_")
|
||||||
|
if paper.publication_date is None:
|
||||||
|
directory = os.path.join(settings.BASE_DIR, "media", "papers", "unknown")
|
||||||
|
else:
|
||||||
|
d = paper.publication_date
|
||||||
|
directory = os.path.join(
|
||||||
|
settings.BASE_DIR, "media", "papers", str(d.year), str(d.month), str(d.day)
|
||||||
|
)
|
||||||
|
return os.path.join(directory, f"{safe_doi}.{ext}")
|
||||||
|
|
||||||
|
|
||||||
|
def _fulltext_candidate(paper):
|
||||||
|
candidates = []
|
||||||
|
if paper.has_fulltext_xml:
|
||||||
|
candidate = extract_title_from_xml(_paper_path(paper, "xml"), paper.doi)
|
||||||
|
if candidate:
|
||||||
|
candidates.append(candidate)
|
||||||
|
if paper.has_fulltext_pdf:
|
||||||
|
candidate = extract_title_from_pdf(
|
||||||
|
_paper_path(paper, "pdf"), paper.doi, paper.first_author or ""
|
||||||
|
)
|
||||||
|
if candidate:
|
||||||
|
candidates.append(candidate)
|
||||||
|
rank = {"high": 3, "medium": 2, "low": 1}
|
||||||
|
return max(candidates, key=lambda item: rank[item.confidence], default=None)
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = "审计摘要化/网页化标题，并可用 fulltext XML/PDF 标题矫正"
|
||||||
|
|
||||||
|
def add_arguments(self, parser):
|
||||||
|
parser.add_argument("--doi", action="append", default=[], help="只处理指定 DOI，可重复")
|
||||||
|
parser.add_argument("--min-length", type=int, default=300, help="初筛标题最小长度")
|
||||||
|
parser.add_argument("--limit", type=int, default=0, help="最多检查多少条，0=不限")
|
||||||
|
parser.add_argument("--apply", action="store_true", help="实际写库；默认仅审计")
|
||||||
|
parser.add_argument(
|
||||||
|
"--allow-medium", action="store_true",
|
||||||
|
help="允许采用 medium 置信候选；默认仅 XML+DOI 的 high 候选自动矫正",
|
||||||
|
)
|
||||||
|
parser.add_argument("--csv", default="", help="将审计结果写入 CSV")
|
||||||
|
parser.add_argument("--title", default="", help="单个 --doi 的人工确认标题")
|
||||||
|
parser.add_argument("--source", default="manual", help="人工标题来源标记")
|
||||||
|
|
||||||
|
def handle(self, *args, **opts):
|
||||||
|
dois = [doi.strip().lower() for doi in opts["doi"] if doi.strip()]
|
||||||
|
manual_title = clean_title(opts["title"])
|
||||||
|
if manual_title and len(dois) != 1:
|
||||||
|
raise CommandError("--title 必须与且只能与一个 --doi 一起使用")
|
||||||
|
if manual_title and not is_usable_title(manual_title):
|
||||||
|
raise CommandError("--title 不是可接受的标题")
|
||||||
|
|
||||||
|
qs = Paper.objects.all().order_by("id")
|
||||||
|
if dois:
|
||||||
|
qs = qs.filter(doi__in=dois)
|
||||||
|
else:
|
||||||
|
qs = qs.annotate(title_len=Length("title")).filter(
|
||||||
|
title_len__gte=max(1, opts["min_length"])
|
||||||
|
)
|
||||||
|
if opts["limit"]:
|
||||||
|
qs = qs[:opts["limit"]]
|
||||||
|
|
||||||
|
csv_file = None
|
||||||
|
writer = None
|
||||||
|
if opts["csv"]:
|
||||||
|
csv_file = open(opts["csv"], "w", newline="", encoding="utf-8-sig")
|
||||||
|
writer = csv.DictWriter(csv_file, fieldnames=(
|
||||||
|
"doi", "openalex_id", "old_title", "reasons", "candidate_title",
|
||||||
|
"candidate_source", "confidence", "evidence", "action",
|
||||||
|
))
|
||||||
|
writer.writeheader()
|
||||||
|
|
||||||
|
checked = suspect_count = corrected = candidates = 0
|
||||||
|
try:
|
||||||
|
for paper in qs.iterator(chunk_size=500):
|
||||||
|
checked += 1
|
||||||
|
assessment = assess_title(paper.title)
|
||||||
|
if not assessment.suspect and not manual_title:
|
||||||
|
continue
|
||||||
|
suspect_count += int(assessment.suspect)
|
||||||
|
|
||||||
|
if manual_title:
|
||||||
|
candidate = FulltextTitleCandidate(
|
||||||
|
title=manual_title,
|
||||||
|
source=opts["source"][:30],
|
||||||
|
confidence="high",
|
||||||
|
evidence="manual_confirmed",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
candidate = _fulltext_candidate(paper)
|
||||||
|
if candidate:
|
||||||
|
candidates += 1
|
||||||
|
|
||||||
|
allowed = candidate and (
|
||||||
|
candidate.confidence == "high"
|
||||||
|
or (opts["allow_medium"] and candidate.confidence == "medium")
|
||||||
|
)
|
||||||
|
should_correct = bool(
|
||||||
|
candidate and allowed
|
||||||
|
and (manual_title or is_better_title(paper.title, candidate.title))
|
||||||
|
)
|
||||||
|
action = "candidate"
|
||||||
|
if not candidate:
|
||||||
|
action = "no_candidate"
|
||||||
|
elif not allowed:
|
||||||
|
action = "needs_review"
|
||||||
|
elif not should_correct:
|
||||||
|
action = "rejected"
|
||||||
|
|
||||||
|
if opts["apply"]:
|
||||||
|
if should_correct:
|
||||||
|
if not paper.title_raw:
|
||||||
|
paper.title_raw = paper.title
|
||||||
|
paper.title = candidate.title
|
||||||
|
paper.title_source = candidate.source
|
||||||
|
paper.title_quality_status = Paper.TITLE_CORRECTED
|
||||||
|
paper.title_verified_at = timezone.now()
|
||||||
|
paper.save(update_fields=[
|
||||||
|
"title", "title_raw", "title_source", "title_quality_status",
|
||||||
|
"title_verified_at", "update_time",
|
||||||
|
])
|
||||||
|
corrected += 1
|
||||||
|
action = "corrected"
|
||||||
|
elif assessment.suspect and paper.title_quality_status == Paper.TITLE_UNCHECKED:
|
||||||
|
paper.title_quality_status = Paper.TITLE_SUSPECT
|
||||||
|
paper.save(update_fields=["title_quality_status", "update_time"])
|
||||||
|
action = "marked_suspect"
|
||||||
|
|
||||||
|
row = {
|
||||||
|
"doi": paper.doi,
|
||||||
|
"openalex_id": paper.openalex_id or "",
|
||||||
|
"old_title": paper.title_raw if action == "corrected" else paper.title,
|
||||||
|
"reasons": ",".join(assessment.reasons),
|
||||||
|
"candidate_title": candidate.title if candidate else "",
|
||||||
|
"candidate_source": candidate.source if candidate else "",
|
||||||
|
"confidence": candidate.confidence if candidate else "",
|
||||||
|
"evidence": candidate.evidence if candidate else "",
|
||||||
|
"action": action,
|
||||||
|
}
|
||||||
|
if writer:
|
||||||
|
writer.writerow(row)
|
||||||
|
self.stdout.write(
|
||||||
|
f"[{action}] {paper.doi} len={assessment.plain_length} "
|
||||||
|
f"reasons={row['reasons']} candidate={row['candidate_source'] or '-'} "
|
||||||
|
f"confidence={row['confidence'] or '-'}"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
if csv_file:
|
||||||
|
csv_file.close()
|
||||||
|
|
||||||
|
mode = "apply" if opts["apply"] else "dry-run"
|
||||||
|
self.stdout.write(self.style.SUCCESS(
|
||||||
|
f"完成({mode}) checked={checked} suspect={suspect_count} "
|
||||||
|
f"fulltext_candidates={candidates} corrected={corrected}"
|
||||||
|
))
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
# resm_paper 现有数百万行；质量状态索引用 CONCURRENTLY 创建，避免长时间阻塞写入。
|
||||||
|
atomic = False
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("resm", "0011_seed_briefing_monitors"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="paper",
|
||||||
|
name="title_raw",
|
||||||
|
field=models.TextField(blank=True, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="paper",
|
||||||
|
name="title_source",
|
||||||
|
field=models.CharField(blank=True, max_length=30, null=True),
|
||||||
|
),
|
||||||
|
migrations.SeparateDatabaseAndState(
|
||||||
|
database_operations=[
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="paper",
|
||||||
|
name="title_quality_status",
|
||||||
|
field=models.CharField(
|
||||||
|
choices=[
|
||||||
|
("unchecked", "未检查"),
|
||||||
|
("suspect", "疑似异常"),
|
||||||
|
("verified", "已验证"),
|
||||||
|
("corrected", "已矫正"),
|
||||||
|
],
|
||||||
|
default="unchecked",
|
||||||
|
max_length=20,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
state_operations=[
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="paper",
|
||||||
|
name="title_quality_status",
|
||||||
|
field=models.CharField(
|
||||||
|
choices=[
|
||||||
|
("unchecked", "未检查"),
|
||||||
|
("suspect", "疑似异常"),
|
||||||
|
("verified", "已验证"),
|
||||||
|
("corrected", "已矫正"),
|
||||||
|
],
|
||||||
|
db_index=True,
|
||||||
|
default="unchecked",
|
||||||
|
max_length=20,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
migrations.RunSQL(
|
||||||
|
sql=(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS "
|
||||||
|
"resm_paper_title_quality_status_idx "
|
||||||
|
"ON resm_paper (title_quality_status);"
|
||||||
|
),
|
||||||
|
reverse_sql=(
|
||||||
|
"DROP INDEX CONCURRENTLY IF EXISTS "
|
||||||
|
"resm_paper_title_quality_status_idx;"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="paper",
|
||||||
|
name="title_verified_at",
|
||||||
|
field=models.DateTimeField(blank=True, null=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -5,12 +5,30 @@ import os
|
||||||
# Create your models here.
|
# Create your models here.
|
||||||
|
|
||||||
class Paper(BaseModel):
|
class Paper(BaseModel):
|
||||||
|
TITLE_UNCHECKED = "unchecked"
|
||||||
|
TITLE_SUSPECT = "suspect"
|
||||||
|
TITLE_VERIFIED = "verified"
|
||||||
|
TITLE_CORRECTED = "corrected"
|
||||||
|
TITLE_QUALITY_CHOICES = (
|
||||||
|
(TITLE_UNCHECKED, "未检查"),
|
||||||
|
(TITLE_SUSPECT, "疑似异常"),
|
||||||
|
(TITLE_VERIFIED, "已验证"),
|
||||||
|
(TITLE_CORRECTED, "已矫正"),
|
||||||
|
)
|
||||||
# ===== 全局唯一标识 =====
|
# ===== 全局唯一标识 =====
|
||||||
openalex_id = models.TextField(unique=True, verbose_name="OpenAlex ID", null=True, blank=True)
|
openalex_id = models.TextField(unique=True, verbose_name="OpenAlex ID", null=True, blank=True)
|
||||||
doi = models.TextField(unique=True, verbose_name="DOI")
|
doi = models.TextField(unique=True, verbose_name="DOI")
|
||||||
# ===== 基本信息 =====
|
# ===== 基本信息 =====
|
||||||
type = models.CharField(max_length=20, db_index=True)
|
type = models.CharField(max_length=20, db_index=True)
|
||||||
title = models.TextField()
|
title = models.TextField()
|
||||||
|
# title_raw 仅在矫正时保存旧值，避免为全部历史数据复制一份大字段。
|
||||||
|
title_raw = models.TextField(null=True, blank=True)
|
||||||
|
title_source = models.CharField(max_length=30, null=True, blank=True)
|
||||||
|
title_quality_status = models.CharField(
|
||||||
|
max_length=20, choices=TITLE_QUALITY_CHOICES,
|
||||||
|
default=TITLE_UNCHECKED, db_index=True,
|
||||||
|
)
|
||||||
|
title_verified_at = models.DateTimeField(null=True, blank=True)
|
||||||
publication_date = models.DateField(null=True, blank=True)
|
publication_date = models.DateField(null=True, blank=True)
|
||||||
publication_year = models.IntegerField(db_index=True)
|
publication_year = models.IntegerField(db_index=True)
|
||||||
# ===== 作者（最小可用集）=====
|
# ===== 作者（最小可用集）=====
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ from celery import current_app
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
import random
|
import random
|
||||||
from .pdf_utils import _is_elsevier_preview_pdf
|
from .pdf_utils import _is_elsevier_preview_pdf
|
||||||
|
from .title_utils import assess_title, extract_title_from_xml, is_better_title
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
import asyncio
|
import asyncio
|
||||||
import sys
|
import sys
|
||||||
|
|
@ -93,7 +94,10 @@ def _build_paper_from_record(record, keywords: str, search: str) -> Paper:
|
||||||
paper.type = (record.get("type") or "article")[:20]
|
paper.type = (record.get("type") or "article")[:20]
|
||||||
paper.openalex_id = record["id"].split("/")[-1]
|
paper.openalex_id = record["id"].split("/")[-1]
|
||||||
paper.doi = record["doi"].replace("https://doi.org/", "")
|
paper.doi = record["doi"].replace("https://doi.org/", "")
|
||||||
paper.title = record["display_name"]
|
paper.title = record.get("title") or record["display_name"]
|
||||||
|
paper.title_source = "openalex"
|
||||||
|
if assess_title(paper.title).suspect:
|
||||||
|
paper.title_quality_status = Paper.TITLE_SUSPECT
|
||||||
paper.publication_date = record["publication_date"]
|
paper.publication_date = record["publication_date"]
|
||||||
paper.publication_year = record["publication_year"]
|
paper.publication_year = record["publication_year"]
|
||||||
if record["open_access"]:
|
if record["open_access"]:
|
||||||
|
|
@ -226,6 +230,42 @@ def update_paper_meta_from_openalex(days: int = 30, per_combo_max: int = None):
|
||||||
return f"openalex update: combos={n_combos}, new_papers={new_papers}, since={from_publication_date}"
|
return f"openalex update: combos={n_combos}, new_papers={new_papers}, since={from_publication_date}"
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task(base=CustomTask)
|
||||||
|
def refresh_suspect_titles_from_openalex(limit: int = 100):
|
||||||
|
"""定向刷新疑似异常标题，吸收 OpenAlex/Crossref 后续元数据修正。
|
||||||
|
|
||||||
|
常规抓取使用 bulk_create(ignore_conflicts=True)，不会更新已有 DOI；本任务只检查
|
||||||
|
已标记 suspect 的记录，并且仅在新标题通过质量检查且明显优于旧值时覆盖。
|
||||||
|
"""
|
||||||
|
qs = Paper.objects.filter(
|
||||||
|
title_quality_status=Paper.TITLE_SUSPECT,
|
||||||
|
openalex_id__isnull=False,
|
||||||
|
).exclude(openalex_id="").order_by("id")[:max(1, limit)]
|
||||||
|
checked = corrected = failed = 0
|
||||||
|
for paper in qs:
|
||||||
|
checked += 1
|
||||||
|
try:
|
||||||
|
record = Works()[paper.openalex_id]
|
||||||
|
except Exception:
|
||||||
|
failed += 1
|
||||||
|
continue
|
||||||
|
candidate = (record or {}).get("title") or (record or {}).get("display_name")
|
||||||
|
if not is_better_title(paper.title, candidate):
|
||||||
|
continue
|
||||||
|
if not paper.title_raw:
|
||||||
|
paper.title_raw = paper.title
|
||||||
|
paper.title = assess_title(candidate).title
|
||||||
|
paper.title_source = "openalex_refresh"
|
||||||
|
paper.title_quality_status = Paper.TITLE_CORRECTED
|
||||||
|
paper.title_verified_at = timezone.now()
|
||||||
|
paper.save(update_fields=[
|
||||||
|
"title", "title_raw", "title_source", "title_quality_status",
|
||||||
|
"title_verified_at", "update_time",
|
||||||
|
])
|
||||||
|
corrected += 1
|
||||||
|
return f"title refresh: checked={checked}, corrected={corrected}, failed={failed}"
|
||||||
|
|
||||||
|
|
||||||
BACKFILL_STOP_KEY = "backfill_paper_meta_stop"
|
BACKFILL_STOP_KEY = "backfill_paper_meta_stop"
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -374,6 +414,9 @@ def _build_paper_from_sd_result(r, qs_text: str):
|
||||||
paper.o_search = qs_text
|
paper.o_search = qs_text
|
||||||
paper.doi = str(doi).replace("https://doi.org/", "")
|
paper.doi = str(doi).replace("https://doi.org/", "")
|
||||||
paper.title = title
|
paper.title = title
|
||||||
|
paper.title_source = "elsevier"
|
||||||
|
if assess_title(paper.title).suspect:
|
||||||
|
paper.title_quality_status = Paper.TITLE_SUSPECT
|
||||||
paper.publication_date = pub_date if len(str(pub_date)) == 10 else None
|
paper.publication_date = pub_date if len(str(pub_date)) == 10 else None
|
||||||
paper.publication_year = year
|
paper.publication_year = year
|
||||||
paper.publication_name = r.get("sourceTitle")
|
paper.publication_name = r.get("sourceTitle")
|
||||||
|
|
@ -625,8 +668,24 @@ def _elsevier_fetch_xml(req, paper):
|
||||||
paper.has_fulltext_xml = True
|
paper.has_fulltext_xml = True
|
||||||
|
|
||||||
paper.save_file_xml(xml_str)
|
paper.save_file_xml(xml_str)
|
||||||
paper.save(update_fields=["has_abstract", "has_abstract_xml",
|
update_fields = ["has_abstract", "has_abstract_xml",
|
||||||
"has_fulltext", "has_fulltext_xml", "update_time"])
|
"has_fulltext", "has_fulltext_xml", "update_time"]
|
||||||
|
# XML 标题节点且 DOI 匹配才是 high；仅在旧标题已呈摘要化/网页化时自动覆盖。
|
||||||
|
if paper.title_quality_status == Paper.TITLE_SUSPECT or assess_title(paper.title).suspect:
|
||||||
|
candidate = extract_title_from_xml(paper.init_paper_path("xml"), paper.doi)
|
||||||
|
if (candidate and candidate.confidence == "high"
|
||||||
|
and is_better_title(paper.title, candidate.title)):
|
||||||
|
if not paper.title_raw:
|
||||||
|
paper.title_raw = paper.title
|
||||||
|
paper.title = candidate.title
|
||||||
|
paper.title_source = candidate.source
|
||||||
|
paper.title_quality_status = Paper.TITLE_CORRECTED
|
||||||
|
paper.title_verified_at = timezone.now()
|
||||||
|
update_fields.extend([
|
||||||
|
"title", "title_raw", "title_source", "title_quality_status",
|
||||||
|
"title_verified_at",
|
||||||
|
])
|
||||||
|
paper.save(update_fields=update_fields)
|
||||||
return True, has_fulltext, None
|
return True, has_fulltext, None
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,102 @@
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import types
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from apps.resm.title_utils import (
|
||||||
|
assess_title,
|
||||||
|
extract_title_from_pdf,
|
||||||
|
extract_title_from_xml,
|
||||||
|
is_better_title,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TitleAssessmentTests(unittest.TestCase):
|
||||||
|
def test_normal_title_is_not_suspect(self):
|
||||||
|
result = assess_title("Optimal Nonlinear PID Speed Control for an Electric Vehicle")
|
||||||
|
self.assertFalse(result.suspect)
|
||||||
|
|
||||||
|
def test_abstract_stored_as_title_is_suspect(self):
|
||||||
|
text = (
|
||||||
|
"In many situations, a beam opening is necessary near a plastic hinge. "
|
||||||
|
"However, few studies investigated the resulting behavior. "
|
||||||
|
"In this study, nine full-scale joints were tested with several configurations. "
|
||||||
|
"The results show that additional reinforcement improves strength and ductility. "
|
||||||
|
) * 2
|
||||||
|
result = assess_title(text)
|
||||||
|
self.assertTrue(result.suspect)
|
||||||
|
self.assertIn("abstract_like_prose", result.reasons)
|
||||||
|
|
||||||
|
def test_mathml_markup_does_not_trigger_raw_length_false_positive(self):
|
||||||
|
markup = "<mml:math>" + "<mml:mi>x</mml:mi>" * 100 + "</mml:math>"
|
||||||
|
result = assess_title(f"Behavior of {markup} under pressure")
|
||||||
|
self.assertFalse(result.suspect)
|
||||||
|
|
||||||
|
def test_short_candidate_is_better_than_abstract(self):
|
||||||
|
old = ("This study investigates reinforced concrete joints. " * 20).strip()
|
||||||
|
new = "Cyclic Behavior of Reinforced Concrete Beam-Column Joints"
|
||||||
|
self.assertTrue(is_better_title(old, new))
|
||||||
|
|
||||||
|
|
||||||
|
class XmlTitleExtractionTests(unittest.TestCase):
|
||||||
|
def _xml_file(self, body):
|
||||||
|
handle = tempfile.NamedTemporaryFile("w", suffix=".xml", delete=False, encoding="utf-8")
|
||||||
|
self.addCleanup(lambda: os.path.exists(handle.name) and os.unlink(handle.name))
|
||||||
|
handle.write(body)
|
||||||
|
handle.close()
|
||||||
|
return handle.name
|
||||||
|
|
||||||
|
def test_extracts_elsevier_title_and_verifies_doi(self):
|
||||||
|
path = self._xml_file("""<?xml version="1.0"?>
|
||||||
|
<response xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||||
|
xmlns:prism="http://prismstandard.org/namespaces/basic/2.0/">
|
||||||
|
<coredata>
|
||||||
|
<prism:doi>10.1234/example.1</prism:doi>
|
||||||
|
<dc:title>Correct & Verified Article Title</dc:title>
|
||||||
|
</coredata>
|
||||||
|
</response>""")
|
||||||
|
candidate = extract_title_from_xml(path, "10.1234/example.1")
|
||||||
|
self.assertIsNotNone(candidate)
|
||||||
|
self.assertEqual(candidate.title, "Correct & Verified Article Title")
|
||||||
|
self.assertEqual(candidate.confidence, "high")
|
||||||
|
|
||||||
|
def test_rejects_xml_for_another_doi(self):
|
||||||
|
path = self._xml_file("""<article>
|
||||||
|
<article-meta><article-id pub-id-type="doi">10.1234/wrong</article-id>
|
||||||
|
<title-group><article-title>A Correct Looking Title</article-title></title-group>
|
||||||
|
</article-meta></article>""")
|
||||||
|
candidate = extract_title_from_xml(path, "10.1234/expected")
|
||||||
|
self.assertIsNone(candidate)
|
||||||
|
|
||||||
|
|
||||||
|
class PdfTitleExtractionTests(unittest.TestCase):
|
||||||
|
def test_pdf_metadata_requires_identity_evidence_and_stays_medium(self):
|
||||||
|
handle = tempfile.NamedTemporaryFile("wb", suffix=".pdf", delete=False)
|
||||||
|
self.addCleanup(lambda: os.path.exists(handle.name) and os.unlink(handle.name))
|
||||||
|
handle.write(b"%PDF-placeholder")
|
||||||
|
handle.close()
|
||||||
|
|
||||||
|
page = types.SimpleNamespace(
|
||||||
|
extract_text=lambda: (
|
||||||
|
"Correct Article Title\nMohamed Shamseldin\n"
|
||||||
|
"https://doi.org/10.1234/example.1\nAbstract"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
reader = types.SimpleNamespace(
|
||||||
|
pages=[page],
|
||||||
|
metadata=types.SimpleNamespace(title="Correct Article Title"),
|
||||||
|
)
|
||||||
|
module = types.SimpleNamespace(PdfReader=lambda *args, **kwargs: reader)
|
||||||
|
with patch.dict("sys.modules", {"pypdf": module}):
|
||||||
|
candidate = extract_title_from_pdf(
|
||||||
|
handle.name, "10.1234/example.1", "Mohamed Shamseldin"
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(candidate)
|
||||||
|
self.assertEqual(candidate.title, "Correct Article Title")
|
||||||
|
self.assertEqual(candidate.confidence, "medium")
|
||||||
|
self.assertEqual(candidate.evidence, "pdf_metadata+doi")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|
@ -0,0 +1,256 @@
|
||||||
|
"""论文标题质量检测与全文标题提取工具。
|
||||||
|
|
||||||
|
本模块不依赖 Django，管理命令、抓取任务和单元测试都可以复用。全文来源的
|
||||||
|
可信度按 XML(标题节点且 DOI 匹配) > PDF 元数据(首页身份匹配) > PDF 首页版式
|
||||||
|
推断排序；调用方默认只应自动采用 high 置信度候选。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from html import unescape
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
_TAG_RE = re.compile(r"<[^>]+>")
|
||||||
|
_SPACE_RE = re.compile(r"\s+")
|
||||||
|
_WORD_RE = re.compile(r"[\w'-]+", re.UNICODE)
|
||||||
|
_DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
|
||||||
|
_WEB_MARKERS = (
|
||||||
|
"journal help", "user username", "remember me login", "current issue",
|
||||||
|
"paper submission", "guidelines for authors", "view subscribe",
|
||||||
|
)
|
||||||
|
_DOCUMENT_MARKERS = ("author:", "authors:", "abstract:", "document type:")
|
||||||
|
_GENERIC_PDF_TITLES = {
|
||||||
|
"untitled", "microsoft word", "document", "article", "full text", "pdf",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class TitleAssessment:
|
||||||
|
title: str
|
||||||
|
plain_length: int
|
||||||
|
word_count: int
|
||||||
|
sentence_count: int
|
||||||
|
suspect: bool
|
||||||
|
reasons: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FulltextTitleCandidate:
|
||||||
|
title: str
|
||||||
|
source: str
|
||||||
|
confidence: str
|
||||||
|
evidence: str
|
||||||
|
|
||||||
|
|
||||||
|
def clean_title(value: Optional[str]) -> str:
|
||||||
|
"""去标签、解实体并合并空白，保留公式中的可见文本。"""
|
||||||
|
if not value:
|
||||||
|
return ""
|
||||||
|
text = unescape(_TAG_RE.sub(" ", str(value)))
|
||||||
|
text = _SPACE_RE.sub(" ", text).strip(" \t\r\n\ufeff")
|
||||||
|
if text.lower().startswith("title:"):
|
||||||
|
text = text[6:].strip()
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_doi(value: Optional[str]) -> str:
|
||||||
|
if not value:
|
||||||
|
return ""
|
||||||
|
value = unescape(str(value)).strip().lower()
|
||||||
|
value = re.sub(r"^(?:doi:\s*|https?://(?:dx\.)?doi\.org/)", "", value)
|
||||||
|
match = _DOI_RE.search(value)
|
||||||
|
return match.group(0).rstrip(".,;)").lower() if match else value.rstrip(".,;)")
|
||||||
|
|
||||||
|
|
||||||
|
def assess_title(value: Optional[str]) -> TitleAssessment:
|
||||||
|
plain = clean_title(value)
|
||||||
|
lower = plain.lower()
|
||||||
|
words = len(_WORD_RE.findall(plain))
|
||||||
|
sentences = len(re.findall(r"[.!?](?:\s|$)", plain))
|
||||||
|
reasons = []
|
||||||
|
|
||||||
|
# 去标签后的长度才用于判断，避免 MathML 本身把正常标题撑到几千字符。
|
||||||
|
if len(plain) >= 300 and words >= 50 and sentences >= 3:
|
||||||
|
reasons.append("abstract_like_prose")
|
||||||
|
web_hits = sum(marker in lower for marker in _WEB_MARKERS)
|
||||||
|
if len(plain) >= 200 and web_hits >= 2:
|
||||||
|
reasons.append("webpage_text")
|
||||||
|
doc_hits = sum(marker in lower for marker in _DOCUMENT_MARKERS)
|
||||||
|
if len(plain) >= 300 and doc_hits >= 2:
|
||||||
|
reasons.append("document_metadata")
|
||||||
|
if len(plain) >= 800 and words >= 80:
|
||||||
|
reasons.append("extreme_length")
|
||||||
|
|
||||||
|
return TitleAssessment(
|
||||||
|
title=plain,
|
||||||
|
plain_length=len(plain),
|
||||||
|
word_count=words,
|
||||||
|
sentence_count=sentences,
|
||||||
|
suspect=bool(reasons),
|
||||||
|
reasons=tuple(reasons),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_usable_title(value: Optional[str]) -> bool:
|
||||||
|
assessment = assess_title(value)
|
||||||
|
if not assessment.title or assessment.suspect:
|
||||||
|
return False
|
||||||
|
if assessment.plain_length < 5 or assessment.plain_length > 500:
|
||||||
|
return False
|
||||||
|
if assessment.word_count > 70 or assessment.sentence_count > 2:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def is_better_title(current: Optional[str], candidate: Optional[str]) -> bool:
|
||||||
|
old = assess_title(current)
|
||||||
|
new = assess_title(candidate)
|
||||||
|
if not old.suspect or not is_usable_title(new.title):
|
||||||
|
return False
|
||||||
|
normalized_old = re.sub(r"\W+", "", old.title).casefold()
|
||||||
|
normalized_new = re.sub(r"\W+", "", new.title).casefold()
|
||||||
|
if not normalized_new or normalized_new == normalized_old:
|
||||||
|
return False
|
||||||
|
return new.plain_length < old.plain_length * 0.75
|
||||||
|
|
||||||
|
|
||||||
|
def _element_text(element) -> str:
|
||||||
|
return clean_title(" ".join(element.itertext()))
|
||||||
|
|
||||||
|
|
||||||
|
def extract_title_from_xml(
|
||||||
|
path: str,
|
||||||
|
expected_doi: Optional[str] = None,
|
||||||
|
) -> Optional[FulltextTitleCandidate]:
|
||||||
|
"""从 Elsevier/JATS/常见全文 XML 中提取文章标题。
|
||||||
|
|
||||||
|
XML 内存在 DOI 时必须与目标论文一致；不一致直接拒绝，防止历史文件错配。
|
||||||
|
"""
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from lxml import etree
|
||||||
|
|
||||||
|
parser = etree.XMLParser(recover=True, huge_tree=True, resolve_entities=False)
|
||||||
|
root = etree.parse(path, parser).getroot()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
expected = normalize_doi(expected_doi)
|
||||||
|
doi_values = []
|
||||||
|
for node in root.xpath(
|
||||||
|
"//*[local-name()='doi' or local-name()='identifier' "
|
||||||
|
"or (local-name()='article-id' and "
|
||||||
|
"translate(@pub-id-type, 'DOI', 'doi')='doi')]"
|
||||||
|
):
|
||||||
|
value = normalize_doi(_element_text(node))
|
||||||
|
if value.startswith("10."):
|
||||||
|
doi_values.append(value)
|
||||||
|
if expected and doi_values and expected not in doi_values:
|
||||||
|
return None
|
||||||
|
|
||||||
|
title_xpaths = (
|
||||||
|
"//*[local-name()='coredata']/*[local-name()='title']",
|
||||||
|
"//*[local-name()='article-meta']/*[local-name()='title-group']/*[local-name()='article-title']",
|
||||||
|
"//*[local-name()='article-title']",
|
||||||
|
"/*/*[local-name()='coredata']/*[local-name()='title']",
|
||||||
|
)
|
||||||
|
for xpath in title_xpaths:
|
||||||
|
for node in root.xpath(xpath):
|
||||||
|
title = _element_text(node)
|
||||||
|
if not is_usable_title(title):
|
||||||
|
continue
|
||||||
|
doi_matched = bool(expected and expected in doi_values)
|
||||||
|
return FulltextTitleCandidate(
|
||||||
|
title=title,
|
||||||
|
source="fulltext_xml",
|
||||||
|
confidence="high" if doi_matched else "medium",
|
||||||
|
evidence="xml_title_node+doi" if doi_matched else "xml_title_node",
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _identity_evidence(text: str, expected_doi: str, first_author: str) -> str:
|
||||||
|
if expected_doi:
|
||||||
|
found = {normalize_doi(v) for v in _DOI_RE.findall(text)}
|
||||||
|
if normalize_doi(expected_doi) in found:
|
||||||
|
return "doi"
|
||||||
|
if first_author:
|
||||||
|
surname = clean_title(first_author).split()[-1].casefold()
|
||||||
|
if len(surname) >= 3 and surname in text.casefold():
|
||||||
|
return "author"
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _title_from_first_page(text: str, first_author: str) -> str:
|
||||||
|
"""保守地从首页文本猜标题，仅返回低/中置信候选。"""
|
||||||
|
lines = [clean_title(line) for line in text.splitlines()]
|
||||||
|
lines = [line for line in lines if line]
|
||||||
|
if not lines:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
surname = clean_title(first_author).split()[-1].casefold() if first_author else ""
|
||||||
|
author_index = next(
|
||||||
|
(i for i, line in enumerate(lines[:30]) if len(surname) >= 3 and surname in line.casefold()),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if author_index is None or author_index == 0:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
candidates = []
|
||||||
|
for line in lines[max(0, author_index - 4):author_index]:
|
||||||
|
lower = line.casefold()
|
||||||
|
if _DOI_RE.search(line) or "journal" in lower or "issn" in lower:
|
||||||
|
continue
|
||||||
|
if re.fullmatch(r"[\d\s|:/.-]+", line):
|
||||||
|
continue
|
||||||
|
candidates.append(line)
|
||||||
|
title = clean_title(" ".join(candidates))
|
||||||
|
return title if is_usable_title(title) else ""
|
||||||
|
|
||||||
|
|
||||||
|
def extract_title_from_pdf(
|
||||||
|
path: str,
|
||||||
|
expected_doi: Optional[str] = None,
|
||||||
|
first_author: Optional[str] = None,
|
||||||
|
) -> Optional[FulltextTitleCandidate]:
|
||||||
|
"""从 PDF metadata 或首页文本提取标题候选。
|
||||||
|
|
||||||
|
PDF metadata 只有在首页 DOI/作者能确认文件身份时才达到 medium；首页版式推断
|
||||||
|
即使 DOI 匹配也保持 medium，默认不会被管理命令自动写库。
|
||||||
|
"""
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
logging.getLogger("pypdf").setLevel(logging.CRITICAL)
|
||||||
|
from pypdf import PdfReader
|
||||||
|
|
||||||
|
reader = PdfReader(path, strict=False)
|
||||||
|
page_text = reader.pages[0].extract_text() or "" if reader.pages else ""
|
||||||
|
metadata = reader.metadata
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
identity = _identity_evidence(page_text, expected_doi or "", first_author or "")
|
||||||
|
meta_title = clean_title(getattr(metadata, "title", "") if metadata else "")
|
||||||
|
if meta_title and meta_title.casefold() not in _GENERIC_PDF_TITLES and is_usable_title(meta_title):
|
||||||
|
return FulltextTitleCandidate(
|
||||||
|
title=meta_title,
|
||||||
|
source="pdf_metadata",
|
||||||
|
confidence="medium" if identity else "low",
|
||||||
|
evidence=f"pdf_metadata+{identity}" if identity else "pdf_metadata_only",
|
||||||
|
)
|
||||||
|
|
||||||
|
page_title = _title_from_first_page(page_text, first_author or "")
|
||||||
|
if page_title:
|
||||||
|
return FulltextTitleCandidate(
|
||||||
|
title=page_title,
|
||||||
|
source="pdf_first_page",
|
||||||
|
confidence="medium" if identity else "low",
|
||||||
|
evidence=f"pdf_first_page+{identity}" if identity else "pdf_first_page_only",
|
||||||
|
)
|
||||||
|
return None
|
||||||
Loading…
Reference in New Issue