"""论文标题质量检测与全文标题提取工具。

本模块不依赖 Django，管理命令、抓取任务和单元测试都可以复用。全文来源的
可信度按 XML(标题节点且 DOI 匹配) > PDF 元数据(首页身份匹配) > PDF 首页版式
推断排序；调用方默认只应自动采用 high 置信度候选。
"""
from __future__ import annotations

from dataclasses import dataclass
from html import unescape
import logging
import os
import re
from typing import Optional


_TAG_RE = re.compile(r"<[^>]+>")
_SPACE_RE = re.compile(r"\s+")
_WORD_RE = re.compile(r"[\w'-]+", re.UNICODE)
_DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
_WEB_MARKERS = (
    "journal help", "user username", "remember me login", "current issue",
    "paper submission", "guidelines for authors", "view subscribe",
)
_DOCUMENT_MARKERS = ("author:", "authors:", "abstract:", "document type:")
_GENERIC_PDF_TITLES = {
    "untitled", "microsoft word", "document", "article", "full text", "pdf",
    "no job name",
}
_FILE_TITLE_RE = re.compile(r"(?:\.eps|\.indd|\.docx?|\.pdf|\.tex)$", re.I)


@dataclass(frozen=True)
class TitleAssessment:
    title: str
    plain_length: int
    word_count: int
    sentence_count: int
    suspect: bool
    reasons: tuple[str, ...]


@dataclass(frozen=True)
class FulltextTitleCandidate:
    title: str
    source: str
    confidence: str
    evidence: str


def clean_title(value: Optional[str]) -> str:
    """去标签、解实体并合并空白，保留公式中的可见文本。"""
    if not value:
        return ""
    text = unescape(_TAG_RE.sub(" ", str(value)))
    text = _SPACE_RE.sub(" ", text).strip(" \t\r\n\ufeff")
    if text.lower().startswith("title:"):
        text = text[6:].strip()
    return text


def normalize_doi(value: Optional[str]) -> str:
    if not value:
        return ""
    value = unescape(str(value)).strip().lower()
    value = re.sub(r"^(?:doi:\s*|https?://(?:dx\.)?doi\.org/)", "", value)
    match = _DOI_RE.search(value)
    return match.group(0).rstrip(".,;)").lower() if match else value.rstrip(".,;)")


def assess_title(value: Optional[str]) -> TitleAssessment:
    plain = clean_title(value)
    lower = plain.lower()
    words = len(_WORD_RE.findall(plain))
    sentences = len(re.findall(r"[.!?](?:\s|$)", plain))
    reasons = []

    # 去标签后的长度才用于判断，避免 MathML 本身把正常标题撑到几千字符。
    if len(plain) >= 300 and words >= 50 and sentences >= 3:
        reasons.append("abstract_like_prose")
    web_hits = sum(marker in lower for marker in _WEB_MARKERS)
    if len(plain) >= 200 and web_hits >= 2:
        reasons.append("webpage_text")
    doc_hits = sum(marker in lower for marker in _DOCUMENT_MARKERS)
    if len(plain) >= 300 and doc_hits >= 2:
        reasons.append("document_metadata")
    if len(plain) >= 800 and words >= 80:
        reasons.append("extreme_length")

    return TitleAssessment(
        title=plain,
        plain_length=len(plain),
        word_count=words,
        sentence_count=sentences,
        suspect=bool(reasons),
        reasons=tuple(reasons),
    )


def is_usable_title(value: Optional[str]) -> bool:
    assessment = assess_title(value)
    if not assessment.title or assessment.suspect:
        return False
    if assessment.plain_length < 5 or assessment.plain_length > 500:
        return False
    if assessment.word_count > 70 or assessment.sentence_count > 2:
        return False
    return True


def _is_usable_pdf_metadata_title(value: Optional[str]) -> bool:
    """PDF Title 属性经常是排版文件名，使用比普通标题更严格的门槛。"""
    title = clean_title(value)
    assessment = assess_title(title)
    if not is_usable_title(title):
        return False
    if assessment.plain_length < 15 or assessment.word_count < 3:
        return False
    if title.casefold() in _GENERIC_PDF_TITLES or _FILE_TITLE_RE.search(title):
        return False
    if re.fullmatch(r"(?:issn\s*)?[\dXx-]{7,}", title):
        return False
    return True


def is_better_title(current: Optional[str], candidate: Optional[str]) -> bool:
    old = assess_title(current)
    new = assess_title(candidate)
    if not old.suspect or not is_usable_title(new.title):
        return False
    normalized_old = re.sub(r"\W+", "", old.title).casefold()
    normalized_new = re.sub(r"\W+", "", new.title).casefold()
    if not normalized_new or normalized_new == normalized_old:
        return False
    return new.plain_length < old.plain_length * 0.75


def _element_text(element) -> str:
    return clean_title(" ".join(element.itertext()))


def extract_title_from_xml(
    path: str,
    expected_doi: Optional[str] = None,
) -> Optional[FulltextTitleCandidate]:
    """从 Elsevier/JATS/常见全文 XML 中提取文章标题。

    XML 内存在 DOI 时必须与目标论文一致；不一致直接拒绝，防止历史文件错配。
    """
    if not os.path.exists(path):
        return None
    try:
        from lxml import etree

        parser = etree.XMLParser(recover=True, huge_tree=True, resolve_entities=False)
        root = etree.parse(path, parser).getroot()
    except Exception:
        return None

    expected = normalize_doi(expected_doi)
    doi_values = []
    for node in root.xpath(
        "//*[local-name()='doi' or local-name()='identifier' "
        "or (local-name()='article-id' and "
        "translate(@pub-id-type, 'DOI', 'doi')='doi')]"
    ):
        value = normalize_doi(_element_text(node))
        if value.startswith("10."):
            doi_values.append(value)
    if expected and doi_values and expected not in doi_values:
        return None

    title_xpaths = (
        "//*[local-name()='coredata']/*[local-name()='title']",
        "//*[local-name()='article-meta']/*[local-name()='title-group']/*[local-name()='article-title']",
        "//*[local-name()='article-title']",
        "/*/*[local-name()='coredata']/*[local-name()='title']",
    )
    for xpath in title_xpaths:
        for node in root.xpath(xpath):
            title = _element_text(node)
            if not is_usable_title(title):
                continue
            doi_matched = bool(expected and expected in doi_values)
            return FulltextTitleCandidate(
                title=title,
                source="fulltext_xml",
                confidence="high" if doi_matched else "medium",
                evidence="xml_title_node+doi" if doi_matched else "xml_title_node",
            )
    return None


def _identity_evidence(text: str, expected_doi: str, first_author: str) -> str:
    if expected_doi:
        found = {normalize_doi(v) for v in _DOI_RE.findall(text)}
        if normalize_doi(expected_doi) in found:
            return "doi"
    if first_author:
        surname = clean_title(first_author).split()[-1].casefold()
        if len(surname) >= 3 and surname in text.casefold():
            return "author"
    return ""


def _title_from_first_page(text: str, first_author: str) -> str:
    """保守地从首页文本猜标题，仅返回低/中置信候选。"""
    lines = [clean_title(line) for line in text.splitlines()]
    lines = [line for line in lines if line]
    if not lines:
        return ""

    surname = clean_title(first_author).split()[-1].casefold() if first_author else ""
    author_index = next(
        (i for i, line in enumerate(lines[:30]) if len(surname) >= 3 and surname in line.casefold()),
        None,
    )
    if author_index is None or author_index == 0:
        return ""

    candidates = []
    for line in lines[max(0, author_index - 4):author_index]:
        lower = line.casefold()
        if _DOI_RE.search(line) or "journal" in lower or "issn" in lower:
            continue
        if re.fullmatch(r"[\d\s|:/.-]+", line):
            continue
        candidates.append(line)
    title = clean_title(" ".join(candidates))
    return title if is_usable_title(title) else ""


def extract_title_from_pdf(
    path: str,
    expected_doi: Optional[str] = None,
    first_author: Optional[str] = None,
) -> Optional[FulltextTitleCandidate]:
    """从 PDF metadata 或首页文本提取标题候选。

    PDF metadata 只有在首页 DOI/作者能确认文件身份时才达到 medium；首页版式推断
    即使 DOI 匹配也保持 medium，默认不会被管理命令自动写库。
    """
    if not os.path.exists(path):
        return None
    try:
        logging.getLogger("pypdf").setLevel(logging.CRITICAL)
        from pypdf import PdfReader

        reader = PdfReader(path, strict=False)
        page_text = reader.pages[0].extract_text() or "" if reader.pages else ""
        metadata = reader.metadata
    except Exception:
        return None

    identity = _identity_evidence(page_text, expected_doi or "", first_author or "")
    meta_title = clean_title(getattr(metadata, "title", "") if metadata else "")
    if _is_usable_pdf_metadata_title(meta_title):
        return FulltextTitleCandidate(
            title=meta_title,
            source="pdf_metadata",
            confidence="medium" if identity else "low",
            evidence=f"pdf_metadata+{identity}" if identity else "pdf_metadata_only",
        )

    page_title = _title_from_first_page(page_text, first_author or "")
    if page_title:
        return FulltextTitleCandidate(
            title=page_title,
            source="pdf_first_page",
            # 首页文本顺序受版式影响很大，只输出人工复核候选，绝不批量自动采用。
            confidence="low",
            evidence=f"pdf_first_page+{identity}" if identity else "pdf_first_page_only",
        )
    return None