278 lines
10 KiB
Python
278 lines
10 KiB
Python
"""外部系统大响应的 task 私有缓存与结构化分段读取。"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import time
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from uuid import uuid4
|
||
|
||
from core.file_store import atomic_write_text
|
||
|
||
RESULT_CACHE_SUBDIR = ".zcbot_external_results"
|
||
RESULT_CACHE_ROOT = ".zcbot_cache"
|
||
RESULT_TTL_SECONDS = 24 * 60 * 60
|
||
MAX_STORED_RESULT_BYTES = 10 * 1024 * 1024
|
||
MAX_TASK_CACHE_BYTES = 50 * 1024 * 1024
|
||
MAX_USER_CACHE_BYTES = 200 * 1024 * 1024
|
||
_REF_RE = re.compile(r"^extres_[0-9a-f]{32}$")
|
||
_PREVIEW_MAX_DEPTH = 6
|
||
_PREVIEW_MAX_NODES = 120
|
||
_PREVIEW_MAX_DICT_KEYS = 30
|
||
_PREVIEW_MAX_LIST_ITEMS = 5
|
||
_PREVIEW_MAX_STRING_CHARS = 500
|
||
|
||
|
||
class ExternalResultError(RuntimeError):
|
||
pass
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class StoredExternalResult:
|
||
result_ref: str
|
||
original_bytes: int
|
||
expires_in_seconds: int
|
||
|
||
|
||
def _json_bytes(value: Any) -> bytes:
|
||
return json.dumps(value, ensure_ascii=False, default=str).encode("utf-8")
|
||
|
||
|
||
def _pointer_escape(value: str) -> str:
|
||
return value.replace("~", "~0").replace("/", "~1")
|
||
|
||
|
||
def build_result_preview(value: Any) -> tuple[Any, list[dict[str, Any]]]:
|
||
"""生成合法、定界的 JSON 预览，并列出可继续分段读取的位置。"""
|
||
reads: list[dict[str, Any]] = []
|
||
budget = [_PREVIEW_MAX_NODES]
|
||
|
||
def visit(item: Any, pointer: str, depth: int) -> Any:
|
||
if budget[0] <= 0:
|
||
reads.append({"json_pointer": pointer, "reason": "node_budget"})
|
||
return {"$preview_truncated": True}
|
||
budget[0] -= 1
|
||
if depth > _PREVIEW_MAX_DEPTH:
|
||
reads.append({"json_pointer": pointer, "reason": "depth"})
|
||
return {"$preview_truncated": True}
|
||
if isinstance(item, dict):
|
||
pairs = list(item.items())
|
||
shown = pairs[:_PREVIEW_MAX_DICT_KEYS]
|
||
result: Any = {
|
||
str(key): visit(
|
||
child,
|
||
pointer + "/" + _pointer_escape(str(key)),
|
||
depth + 1,
|
||
)
|
||
for key, child in shown
|
||
}
|
||
if len(pairs) > len(shown):
|
||
result["$preview_omitted_keys"] = len(pairs) - len(shown)
|
||
reads.append({
|
||
"json_pointer": pointer,
|
||
"reason": "keys",
|
||
"total_keys": len(pairs),
|
||
})
|
||
return result
|
||
if isinstance(item, list):
|
||
shown = item[:_PREVIEW_MAX_LIST_ITEMS]
|
||
result = [
|
||
visit(child, pointer + "/" + str(index), depth + 1)
|
||
for index, child in enumerate(shown)
|
||
]
|
||
if len(item) > len(shown):
|
||
reads.append({
|
||
"json_pointer": pointer,
|
||
"reason": "items",
|
||
"total_items": len(item),
|
||
"next_offset": len(shown),
|
||
})
|
||
return result
|
||
if isinstance(item, str) and len(item) > _PREVIEW_MAX_STRING_CHARS:
|
||
reads.append({
|
||
"json_pointer": pointer,
|
||
"reason": "string",
|
||
"total_chars": len(item),
|
||
})
|
||
return item[:_PREVIEW_MAX_STRING_CHARS] + "...[preview]"
|
||
return item
|
||
|
||
return visit(value, "", 0), reads
|
||
|
||
|
||
def resolve_json_pointer(value: Any, pointer: str) -> Any:
|
||
"""解析 RFC 6901 JSON Pointer；空串表示完整根值。"""
|
||
pointer = (pointer or "").strip()
|
||
if not pointer:
|
||
return value
|
||
if not pointer.startswith("/"):
|
||
raise ExternalResultError("json_pointer 必须为空或以 / 开头")
|
||
current = value
|
||
for raw_part in pointer[1:].split("/"):
|
||
part = raw_part.replace("~1", "/").replace("~0", "~")
|
||
if isinstance(current, dict):
|
||
if part not in current:
|
||
raise ExternalResultError(f"json_pointer 不存在: {pointer}")
|
||
current = current[part]
|
||
elif isinstance(current, list):
|
||
try:
|
||
if not part.isdigit():
|
||
raise ValueError(part)
|
||
index = int(part)
|
||
current = current[index]
|
||
except (ValueError, IndexError) as exc:
|
||
raise ExternalResultError(f"json_pointer 不存在: {pointer}") from exc
|
||
else:
|
||
raise ExternalResultError(f"json_pointer 不存在: {pointer}")
|
||
return current
|
||
|
||
|
||
class ExternalResultStore:
|
||
def __init__(self, user_root: Path, task_id: str):
|
||
task_id = str(task_id).strip()
|
||
if not task_id or task_id in {".", ".."} or not re.fullmatch(r"[\w.-]+", task_id):
|
||
raise ExternalResultError("task_id 无效")
|
||
self.user_root = Path(user_root).resolve()
|
||
self.cache_root = self.user_root / RESULT_CACHE_ROOT
|
||
self.root = self.cache_root / task_id / "external_results"
|
||
# 0.62.1 曾把 24h 缓存写在该目录；reader 保留一版兼容窗口。
|
||
self.legacy_cache_root = self.user_root / RESULT_CACHE_SUBDIR
|
||
self.legacy_root = self.legacy_cache_root / task_id
|
||
|
||
def _path(self, result_ref: str) -> Path:
|
||
if not _REF_RE.fullmatch(result_ref or ""):
|
||
raise ExternalResultError("result_ref 无效")
|
||
return self.root / f"{result_ref}.json"
|
||
|
||
def _candidate_paths(self, result_ref: str) -> list[Path]:
|
||
current = self._path(result_ref)
|
||
return [current, self.legacy_root / current.name]
|
||
|
||
def _sweep(self) -> None:
|
||
cutoff = time.time() - RESULT_TTL_SECONDS
|
||
paths = list(self.cache_root.glob("*/external_results/extres_*.json"))
|
||
paths.extend(self.legacy_cache_root.glob("*/extres_*.json"))
|
||
for path in paths:
|
||
try:
|
||
if path.stat().st_mtime < cutoff:
|
||
path.unlink()
|
||
try:
|
||
path.parent.rmdir()
|
||
except OSError:
|
||
pass
|
||
except OSError:
|
||
continue
|
||
|
||
def store(
|
||
self,
|
||
system_id: str,
|
||
result: dict[str, Any],
|
||
*,
|
||
provenance: dict[str, Any] | None = None,
|
||
) -> StoredExternalResult:
|
||
encoded = _json_bytes(result)
|
||
if len(encoded) > MAX_STORED_RESULT_BYTES:
|
||
raise ExternalResultError(
|
||
f"外部系统响应超过安全保存上限({MAX_STORED_RESULT_BYTES} bytes)"
|
||
)
|
||
self.root.mkdir(parents=True, exist_ok=True)
|
||
self._sweep()
|
||
existing = list(self.root.glob("extres_*.json"))
|
||
existing.extend(self.legacy_root.glob("extres_*.json"))
|
||
existing.sort(key=lambda path: path.stat().st_mtime)
|
||
total = sum(path.stat().st_size for path in existing)
|
||
while existing and total + len(encoded) > MAX_TASK_CACHE_BYTES:
|
||
oldest = existing.pop(0)
|
||
try:
|
||
size = oldest.stat().st_size
|
||
oldest.unlink()
|
||
total -= size
|
||
except OSError:
|
||
continue
|
||
if total + len(encoded) > MAX_TASK_CACHE_BYTES:
|
||
raise ExternalResultError("当前任务的外部结果缓存已达上限")
|
||
user_files = list(self.cache_root.glob("*/external_results/extres_*.json"))
|
||
user_files.extend(self.legacy_cache_root.glob("*/extres_*.json"))
|
||
user_files.sort(key=lambda path: path.stat().st_mtime)
|
||
user_total = sum(path.stat().st_size for path in user_files)
|
||
while user_files and user_total + len(encoded) > MAX_USER_CACHE_BYTES:
|
||
oldest = user_files.pop(0)
|
||
try:
|
||
size = oldest.stat().st_size
|
||
oldest.unlink()
|
||
user_total -= size
|
||
except OSError:
|
||
continue
|
||
if user_total + len(encoded) > MAX_USER_CACHE_BYTES:
|
||
raise ExternalResultError("当前用户的外部结果缓存已达上限")
|
||
result_ref = f"extres_{uuid4().hex}"
|
||
envelope = {
|
||
"version": 1,
|
||
"created_at": int(time.time()),
|
||
"system_id": str(system_id),
|
||
"provenance": provenance or {},
|
||
"result": result,
|
||
}
|
||
atomic_write_text(
|
||
self._path(result_ref),
|
||
json.dumps(envelope, ensure_ascii=False, default=str),
|
||
)
|
||
return StoredExternalResult(
|
||
result_ref=result_ref,
|
||
original_bytes=len(encoded),
|
||
expires_in_seconds=RESULT_TTL_SECONDS,
|
||
)
|
||
|
||
def load(self, result_ref: str) -> dict[str, Any]:
|
||
self._sweep()
|
||
candidates = self._candidate_paths(result_ref)
|
||
path = next((candidate for candidate in candidates if candidate.is_file()), candidates[0])
|
||
try:
|
||
if time.time() - path.stat().st_mtime > RESULT_TTL_SECONDS:
|
||
path.unlink()
|
||
raise ExternalResultError("result_ref 已过期")
|
||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||
except FileNotFoundError as exc:
|
||
raise ExternalResultError("result_ref 不存在或已过期") from exc
|
||
except (OSError, ValueError) as exc:
|
||
raise ExternalResultError("result_ref 无法读取") from exc
|
||
if not isinstance(payload, dict) or not isinstance(payload.get("result"), dict):
|
||
raise ExternalResultError("result_ref 内容无效")
|
||
return payload
|
||
|
||
|
||
def project_result(
|
||
value: Any,
|
||
*,
|
||
offset: int = 0,
|
||
limit: int = 50,
|
||
fields: list[str] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""对缓存节点分页并可选投影字典字段。"""
|
||
offset = max(0, int(offset))
|
||
limit = max(1, min(int(limit), 200))
|
||
if fields is not None and not isinstance(fields, list):
|
||
raise ExternalResultError("fields 必须是字符串数组")
|
||
wanted = [str(field) for field in (fields or [])[:50]]
|
||
|
||
def select(item: Any) -> Any:
|
||
if not wanted or not isinstance(item, dict):
|
||
return item
|
||
return {field: item.get(field) for field in wanted if field in item}
|
||
|
||
if isinstance(value, list):
|
||
page = [select(item) for item in value[offset : offset + limit]]
|
||
return {
|
||
"type": "array",
|
||
"total_items": len(value),
|
||
"offset": offset,
|
||
"limit": limit,
|
||
"has_more": offset + len(page) < len(value),
|
||
"data": page,
|
||
}
|
||
if isinstance(value, dict):
|
||
return {"type": "object", "data": select(value)}
|
||
return {"type": type(value).__name__, "data": value}
|