139 lines
5.4 KiB
Python
139 lines
5.4 KiB
Python
from django.db import models
|
|
from apps.utils.models import BaseModel, CommonAModel
|
|
from django.conf import settings
|
|
import os
|
|
# Create your models here.
|
|
|
|
class Paper(BaseModel):
|
|
# ===== 全局唯一标识 =====
|
|
openalex_id = models.TextField(unique=True, verbose_name="OpenAlex ID", null=True, blank=True)
|
|
doi = models.TextField(unique=True, verbose_name="DOI")
|
|
# ===== 基本信息 =====
|
|
type = models.CharField(max_length=20, db_index=True)
|
|
title = models.TextField()
|
|
publication_date = models.DateField(null=True, blank=True)
|
|
publication_year = models.IntegerField(db_index=True)
|
|
# ===== 作者（最小可用集）=====
|
|
first_author = models.TextField(null=True, blank=True)
|
|
first_author_institution = models.TextField(null=True, blank=True)
|
|
# ===== 期刊 =====
|
|
publication_name = models.TextField(null=True, blank=True)
|
|
# ===== OA 元信息 =====
|
|
is_oa = models.BooleanField(default=False, db_index=True)
|
|
oa_url = models.TextField(null=True, blank=True)
|
|
# ===== 状态位（调度核心）=====
|
|
has_abstract = models.BooleanField(default=False, db_index=True)
|
|
has_abstract_xml = models.BooleanField(default=False, db_index=True)
|
|
has_fulltext = models.BooleanField(default=False, db_index=True)
|
|
has_fulltext_xml = models.BooleanField(default=False, db_index=True)
|
|
has_fulltext_pdf = models.BooleanField(default=False, db_index=True)
|
|
fetch_status = models.CharField(max_length=20, null=True, blank=True) # downloading
|
|
fail_reason = models.TextField(null=True, blank=True)
|
|
|
|
source = models.CharField(
|
|
max_length=20,
|
|
default="openalex",
|
|
verbose_name="元数据来源"
|
|
)
|
|
o_search = models.TextField(default="cement")
|
|
o_keywords = models.TextField(null=True, blank=True)
|
|
|
|
def init_save_dir(self):
|
|
publication_date = self.publication_date
|
|
if publication_date is None:
|
|
paper_dir = os.path.join(settings.BASE_DIR, "media/papers", "unknown")
|
|
else:
|
|
paper_dir = os.path.join(
|
|
settings.BASE_DIR,
|
|
"media/papers",
|
|
str(publication_date.year),
|
|
str(publication_date.month),
|
|
str(publication_date.day)
|
|
)
|
|
os.makedirs(paper_dir, exist_ok=True)
|
|
return paper_dir
|
|
|
|
def init_paper_path(self, type:str):
|
|
paper_dir = self.init_save_dir()
|
|
safe_doi = self.doi.replace("/", "_")
|
|
if type == "xml":
|
|
paper_file = os.path.join(paper_dir, f"{safe_doi}.xml")
|
|
elif type == "pdf":
|
|
paper_file = os.path.join(paper_dir, f"{safe_doi}.pdf")
|
|
else:
|
|
raise ValueError("type must be xml or pdf")
|
|
return paper_file
|
|
|
|
def save_file_xml(self, content):
|
|
paper_file = self.init_paper_path("xml")
|
|
with open(paper_file, "wb") as f:
|
|
f.write(content.encode("utf-8"))
|
|
|
|
def save_file_pdf(self, content, save_obj=False):
|
|
paper_file = self.init_paper_path("pdf")
|
|
with open(paper_file, "wb") as f:
|
|
f.write(content)
|
|
if save_obj:
|
|
self.has_fulltext = True
|
|
self.has_fulltext_pdf = True
|
|
self.save(update_fields=["has_fulltext", "has_fulltext_pdf", "update_time"])
|
|
|
|
def save_fail_reason(self, reason):
|
|
if self.fail_reason:
|
|
self.fail_reason += f";{reason}"
|
|
else:
|
|
self.fail_reason = f";{reason}"
|
|
self.save(update_fields=["fail_reason", "update_time"])
|
|
|
|
def fetch(self, status:str):
|
|
self.fetch_status = status
|
|
self.save(update_fields=["fetch_status", "update_time"])
|
|
|
|
def fetch_end(self):
|
|
self.fetch_status = None
|
|
self.save(update_fields=["fetch_status", "update_time"])
|
|
|
|
|
|
|
|
class PaperAbstract(BaseModel):
|
|
paper = models.OneToOneField(
|
|
Paper,
|
|
on_delete=models.CASCADE,
|
|
related_name="abstract"
|
|
)
|
|
|
|
abstract = models.TextField()
|
|
source = models.CharField(
|
|
max_length=20,
|
|
verbose_name="摘要来源" # openalex / elsevier / crossref
|
|
)
|
|
|
|
|
|
class PaperMonitor(CommonAModel):
|
|
"""论文监控订阅:监控任务遍历启用项,按 type 拼 OpenAlex 过滤,用
|
|
from_publication_date 拉最近 days 天的最新论文元数据入库(走通用核心,自动去重)。
|
|
期刊监控与关键词监控共用本表,靠 type 区分。"""
|
|
TYPE_JOURNAL = "journal"
|
|
TYPE_SEARCH = "search"
|
|
TYPE_KEYWORD = "keyword"
|
|
TYPE_CHOICES = (
|
|
(TYPE_JOURNAL, "期刊(ISSN)"),
|
|
(TYPE_SEARCH, "搜索词(标题/摘要)"),
|
|
(TYPE_KEYWORD, "OpenAlex关键词ID"),
|
|
)
|
|
type = models.CharField("监控类型", max_length=20, choices=TYPE_CHOICES, db_index=True)
|
|
value = models.CharField("监控值", max_length=500) # ISSN / 搜索词 / keyword id
|
|
name = models.CharField("名称", max_length=200, null=True, blank=True)
|
|
note = models.CharField("方向标注", max_length=100, null=True, blank=True) # 如 无机非金属材料
|
|
is_active = models.BooleanField("启用", default=True, db_index=True)
|
|
days = models.IntegerField("回看窗口(天)", default=30)
|
|
last_run = models.DateTimeField("上次运行时间", null=True, blank=True)
|
|
last_count = models.IntegerField("上次拉取篇数(窗口内)", default=0)
|
|
|
|
class Meta:
|
|
verbose_name = "论文监控"
|
|
verbose_name_plural = verbose_name
|
|
|
|
def __str__(self):
|
|
return f"{self.get_type_display()}:{self.name or self.value}"
|