446 lines
18 KiB
Python
446 lines
18 KiB
Python
"""管理后台端点(admin-only):/v1/admin/*。
|
||
|
||
`register_admin_routes(app, require_admin)` 在 create_app 内调用,把管理路由挂上去,
|
||
整组走 `Depends(require_admin)`(JWT 有效 + users.role=='admin',否则 403)。
|
||
|
||
第一版只有总览(监控指标):单个 `GET /v1/admin/overview` 一次返回全部 section
|
||
(runtime / tasks / users / usage / storage),前端定时轮询这一个端点即可。runtime 读
|
||
app.state 内存(轻);其余走 DB 聚合(GROUP BY,无 N+1)。指标只读、不落库。
|
||
|
||
后续管理动作(建用户 / 改角色 / 配置磁盘配额等)在此模块续挂 /v1/admin/users、
|
||
/v1/admin/config 等,前端 admin.html 加 tab。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import importlib
|
||
from types import ModuleType
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Any
|
||
from uuid import UUID
|
||
|
||
from fastapi import Depends, FastAPI, HTTPException
|
||
from pydantic import BaseModel, Field
|
||
from sqlalchemy import func, select, update
|
||
|
||
from core.storage import session_scope
|
||
from core.storage import usage_report
|
||
from core.storage.models import Task, UsageEvent, User, UserDiskUsage
|
||
|
||
from .broker import broker
|
||
|
||
resource: ModuleType | None
|
||
try:
|
||
resource = importlib.import_module("resource")
|
||
except ImportError: # pragma: no cover - Windows
|
||
resource = None
|
||
|
||
|
||
def _rss_peak_mb():
|
||
"""进程峰值 RSS(MB)。Linux 走 ru_maxrss(KB);Windows dev 返 None(降级)。"""
|
||
if resource is None:
|
||
return None
|
||
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
|
||
|
||
|
||
def _range_cutoff(now: datetime, range_key: str):
|
||
"""时间筛选 → cutoff datetime(或 None=全部)。range_key: all / 7d / 30d。"""
|
||
if range_key == "7d":
|
||
return now - timedelta(days=7)
|
||
if range_key == "30d":
|
||
return now - timedelta(days=30)
|
||
return None # all / 未知 → 不筛
|
||
|
||
|
||
def _runtime_section(app: FastAPI) -> dict:
|
||
"""实时运行态:从 app.state 读内存,无 DB。
|
||
|
||
active_runs 逼近 max_workers 即线程池排队(新 run 的 SSE 会卡)—— 前端据此变色。
|
||
"""
|
||
inflight = getattr(app.state, "inflight", None)
|
||
active = len(inflight) if inflight is not None else 0
|
||
max_workers = getattr(app.state, "run_max_workers", None)
|
||
return {
|
||
"active_runs": active,
|
||
"max_workers": max_workers,
|
||
"sse_subs": broker.total_subscribers(),
|
||
"rss_peak_mb": _rss_peak_mb(),
|
||
}
|
||
|
||
|
||
def _tasks_section(s: Any) -> dict:
|
||
"""task 计数:总数 + 按 status + 按 run_status 分布。"""
|
||
total = s.execute(select(func.count()).select_from(Task)).scalar_one()
|
||
by_status = {
|
||
st: n
|
||
for st, n in s.execute(
|
||
select(Task.status, func.count()).group_by(Task.status)
|
||
).all()
|
||
}
|
||
by_run_status = {
|
||
st: n
|
||
for st, n in s.execute(
|
||
select(Task.run_status, func.count()).group_by(Task.run_status)
|
||
).all()
|
||
}
|
||
return {"total": total, "by_status": by_status, "by_run_status": by_run_status}
|
||
|
||
|
||
def _users_section(s: Any, cutoff_7d: datetime) -> dict:
|
||
"""用户:总数 + 近 7d 有用量事件的活跃用户数。"""
|
||
total = s.execute(select(func.count()).select_from(User)).scalar_one()
|
||
active_7d = s.execute(
|
||
select(func.count(func.distinct(UsageEvent.user_id))).where(
|
||
UsageEvent.created_at >= cutoff_7d
|
||
)
|
||
).scalar_one()
|
||
return {"total": total, "active_7d": active_7d}
|
||
|
||
|
||
def _storage_page(s: Any, page: int, page_size: int) -> dict:
|
||
"""分页的各用户磁盘用量(bytes desc + user_id 兜底);附 per-user 配额。
|
||
|
||
数据源 user_disk_usage(后台扫描快照,只含扫过的用户);total 为该表行数。
|
||
"""
|
||
from core.agent_builder import load_config
|
||
from core.storage.disk_quota import parse_bytes
|
||
|
||
quota = parse_bytes((load_config().get("quotas") or {}).get("disk_bytes_per_user"))
|
||
total = s.execute(select(func.count()).select_from(UserDiskUsage)).scalar_one()
|
||
rows = [
|
||
{
|
||
"user_id": str(uid),
|
||
"email": email or "",
|
||
"name": name or "",
|
||
"user_name": uname or "",
|
||
"bytes_used": int(b or 0),
|
||
"file_count": int(fc or 0),
|
||
"scanned_at": scanned.isoformat() if scanned else None,
|
||
}
|
||
for uid, email, name, uname, b, fc, scanned in s.execute(
|
||
select(
|
||
UserDiskUsage.user_id,
|
||
User.email,
|
||
User.name,
|
||
User.user_name,
|
||
UserDiskUsage.bytes_used,
|
||
UserDiskUsage.file_count,
|
||
UserDiskUsage.scanned_at,
|
||
)
|
||
.join(User, User.user_id == UserDiskUsage.user_id, isouter=True)
|
||
.order_by(UserDiskUsage.bytes_used.desc(), UserDiskUsage.user_id)
|
||
.limit(page_size)
|
||
.offset(page * page_size)
|
||
).all()
|
||
]
|
||
return {
|
||
"page": page, "page_size": page_size, "total": total,
|
||
"quota_bytes": quota, "rows": rows,
|
||
}
|
||
|
||
|
||
def _model_catalog() -> list[dict]:
|
||
"""全部可门控模型清单 [{id, display_name, kind}]:文本(config/models/*.yaml)+
|
||
图/视频(config/media/*.yaml,跨 provider)。给档位编辑 UI 画图例(id → 显示名)。
|
||
"""
|
||
from core.capabilities import ModelCapabilities
|
||
from core.paths import ROOT
|
||
import yaml as _yaml
|
||
|
||
out: list[dict] = []
|
||
models_dir = ROOT / "config" / "models"
|
||
if models_dir.is_dir():
|
||
for path in sorted(models_dir.glob("*.yaml")):
|
||
try:
|
||
data = _yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||
except Exception:
|
||
continue
|
||
family = data.get("family") or path.stem
|
||
for variant in (data.get("variants") or {}).keys():
|
||
profile = f"{family}.{variant}"
|
||
try:
|
||
caps = ModelCapabilities.load(profile, models_dir)
|
||
except (ValueError, FileNotFoundError):
|
||
continue
|
||
out.append({"id": profile, "display_name": caps.display_name or profile, "kind": "text"})
|
||
media_dir = ROOT / "config" / "media"
|
||
seen: set[str] = set()
|
||
if media_dir.is_dir():
|
||
for path in sorted(media_dir.glob("*.yaml")):
|
||
try:
|
||
mdata = _yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||
except Exception:
|
||
continue
|
||
for kind in ("image", "video"):
|
||
for key, cfg in (mdata.get(kind) or {}).items():
|
||
if isinstance(cfg, dict) and key not in seen:
|
||
seen.add(key)
|
||
out.append({"id": key, "display_name": cfg.get("display_name") or key, "kind": kind})
|
||
return out
|
||
|
||
|
||
class SetPlanRequest(BaseModel):
|
||
plan: str = "" # 档位名(config/agent.yaml model_tiers 的 key);空串 = 清空 → 落 default 档
|
||
|
||
|
||
class ExternalSystemDefinitionRequest(BaseModel):
|
||
provider: str = "factory_mes"
|
||
name: str
|
||
base_url: str
|
||
openapi_url: str
|
||
login_path: str = "/api/auth/token/"
|
||
auth_type: str = "password_jwt"
|
||
username_field: str = "username"
|
||
password_field: str = "password"
|
||
token_field: str = "access"
|
||
auth_header_name: str = "Authorization"
|
||
auth_header_template: str = "Bearer {token}"
|
||
operation_mode: str | None = None
|
||
operation_policies: dict[str, str] = Field(default_factory=dict)
|
||
timeout_seconds: float = 15
|
||
max_result_bytes: int = 65536
|
||
max_total_result_bytes: int = 262144
|
||
max_page_size: int = 200
|
||
verify_tls: bool = True
|
||
query_guidance: str = ""
|
||
recommended_operation_ids: list[str] = Field(
|
||
default_factory=lambda: ["bi_dataset_list", "bi_dataset_exec"]
|
||
)
|
||
enabled: bool = True
|
||
visibility: str = "selected"
|
||
selected_user_ids: list[UUID] = Field(default_factory=list)
|
||
|
||
|
||
def _external_definition_config(body: ExternalSystemDefinitionRequest) -> dict[str, Any]:
|
||
config = {
|
||
"base_url": body.base_url,
|
||
"openapi_url": body.openapi_url,
|
||
"login_path": body.login_path,
|
||
"auth_type": body.auth_type,
|
||
"username_field": body.username_field,
|
||
"password_field": body.password_field,
|
||
"token_field": body.token_field,
|
||
"auth_header_name": body.auth_header_name,
|
||
"auth_header_template": body.auth_header_template,
|
||
"operation_policies": body.operation_policies,
|
||
"timeout_seconds": body.timeout_seconds,
|
||
"max_result_bytes": body.max_result_bytes,
|
||
"max_total_result_bytes": body.max_total_result_bytes,
|
||
"max_page_size": body.max_page_size,
|
||
"verify_tls": body.verify_tls,
|
||
"query_guidance": body.query_guidance,
|
||
"recommended_operation_ids": body.recommended_operation_ids,
|
||
}
|
||
if body.operation_mode is not None:
|
||
config["operation_mode"] = body.operation_mode
|
||
return config
|
||
|
||
|
||
def register_admin_routes(app: FastAPI, require_admin) -> None:
|
||
"""把 /v1/admin/* 管理路由挂到 app 上,整组走 require_admin gate。"""
|
||
|
||
@app.get("/v1/admin/overview", tags=["admin"])
|
||
def admin_overview(user_id: UUID = Depends(require_admin)):
|
||
"""管理总览(固定形态,供轮询):runtime/tasks/users/usage 总用量+近7d趋势。admin-only。
|
||
|
||
按模型 / 各用户用量 / 存储 是带筛选/分页的独立端点,不在此 bundle。
|
||
"""
|
||
now = datetime.now(timezone.utc)
|
||
cutoff_7d = now - timedelta(days=7)
|
||
with session_scope() as s:
|
||
return {
|
||
"generated_at": now.isoformat(),
|
||
"runtime": _runtime_section(app),
|
||
"tasks": _tasks_section(s),
|
||
"users": _users_section(s, cutoff_7d),
|
||
"usage": usage_report.usage_overview(s, cutoff_7d),
|
||
}
|
||
|
||
@app.get("/v1/admin/external-system-definitions", tags=["admin"])
|
||
def admin_external_system_definitions(user_id: UUID = Depends(require_admin)):
|
||
from core.external_systems.service import list_external_system_definitions
|
||
return {"results": list_external_system_definitions()}
|
||
|
||
@app.get("/v1/admin/external-system-users", tags=["admin"])
|
||
def admin_external_system_users(user_id: UUID = Depends(require_admin)):
|
||
with session_scope() as s:
|
||
rows = s.execute(
|
||
select(User.user_id, User.name, User.user_name, User.email)
|
||
.order_by(User.name, User.user_name, User.email, User.user_id)
|
||
).all()
|
||
return {"results": [
|
||
{
|
||
"user_id": str(uid),
|
||
"label": name or user_name or email or str(uid)[:8],
|
||
"email": email or "",
|
||
}
|
||
for uid, name, user_name, email in rows
|
||
]}
|
||
|
||
@app.post("/v1/admin/external-system-definitions", tags=["admin"])
|
||
def admin_create_external_system_definition(
|
||
body: ExternalSystemDefinitionRequest,
|
||
user_id: UUID = Depends(require_admin),
|
||
):
|
||
from core.external_systems.service import (
|
||
ExternalSystemError,
|
||
create_external_system_definition,
|
||
)
|
||
try:
|
||
return create_external_system_definition(
|
||
user_id,
|
||
provider=body.provider,
|
||
name=body.name,
|
||
config=_external_definition_config(body),
|
||
enabled=body.enabled,
|
||
visibility=body.visibility,
|
||
selected_user_ids=body.selected_user_ids,
|
||
)
|
||
except ExternalSystemError as exc:
|
||
raise HTTPException(400, str(exc)) from exc
|
||
|
||
@app.put("/v1/admin/external-system-definitions/{definition_id}", tags=["admin"])
|
||
def admin_update_external_system_definition(
|
||
definition_id: UUID,
|
||
body: ExternalSystemDefinitionRequest,
|
||
user_id: UUID = Depends(require_admin),
|
||
):
|
||
from core.external_systems.service import (
|
||
ExternalSystemError,
|
||
update_external_system_definition,
|
||
)
|
||
try:
|
||
return update_external_system_definition(
|
||
definition_id,
|
||
name=body.name,
|
||
config=_external_definition_config(body),
|
||
enabled=body.enabled,
|
||
visibility=body.visibility,
|
||
selected_user_ids=body.selected_user_ids,
|
||
)
|
||
except ExternalSystemError as exc:
|
||
code = 404 if str(exc) == "external system definition not found" else 400
|
||
raise HTTPException(code, str(exc)) from exc
|
||
|
||
@app.delete("/v1/admin/external-system-definitions/{definition_id}", tags=["admin"])
|
||
def admin_delete_external_system_definition(
|
||
definition_id: UUID,
|
||
user_id: UUID = Depends(require_admin),
|
||
):
|
||
from core.external_systems.service import (
|
||
ExternalSystemError,
|
||
delete_external_system_definition,
|
||
)
|
||
try:
|
||
if not delete_external_system_definition(definition_id):
|
||
raise HTTPException(404, "external system definition not found")
|
||
except ExternalSystemError as exc:
|
||
raise HTTPException(409, str(exc)) from exc
|
||
return {"deleted": True}
|
||
|
||
@app.get("/v1/admin/usage/models", tags=["admin"])
|
||
def admin_usage_models(
|
||
range: str = "all", sort: str = "cost", user_id: UUID = Depends(require_admin)
|
||
):
|
||
"""按模型用量。range: all/7d/30d;sort: cost/tokens。admin-only。"""
|
||
now = datetime.now(timezone.utc)
|
||
with session_scope() as s:
|
||
return {
|
||
"range": range, "sort": sort,
|
||
"rows": usage_report.models_usage(s, _range_cutoff(now, range), sort),
|
||
}
|
||
|
||
@app.get("/v1/admin/usage/users", tags=["admin"])
|
||
def admin_usage_users(
|
||
page: int = 0, page_size: int = 20, range: str = "all", sort: str = "cost",
|
||
user_id: UUID = Depends(require_admin),
|
||
):
|
||
"""各用户 token 用量(分页 + 时间筛选 + 排序)。admin-only。
|
||
|
||
page 0-based;page_size 夹到 [1,100];range all/7d/30d;sort cost/tokens。
|
||
含零用量用户(全表 LEFT JOIN);总用量在 overview.usage.total。
|
||
"""
|
||
page = max(0, page)
|
||
page_size = min(100, max(1, page_size))
|
||
now = datetime.now(timezone.utc)
|
||
with session_scope() as s:
|
||
d = usage_report.user_usage_page(s, page, page_size, _range_cutoff(now, range), sort)
|
||
d["range"] = range
|
||
d["sort"] = sort
|
||
return d
|
||
|
||
@app.get("/v1/admin/storage/users", tags=["admin"])
|
||
def admin_storage_users(
|
||
page: int = 0, page_size: int = 20, user_id: UUID = Depends(require_admin)
|
||
):
|
||
"""各用户磁盘用量(分页,bytes desc)。admin-only。page 0-based;page_size [1,100]。"""
|
||
page = max(0, page)
|
||
page_size = min(100, max(1, page_size))
|
||
with session_scope() as s:
|
||
return _storage_page(s, page, page_size)
|
||
|
||
@app.get("/v1/admin/tool-failures", tags=["admin"])
|
||
def admin_tool_failures(
|
||
days: int = 7, min_count: int = 3, min_tasks: int = 1,
|
||
user_id: UUID = Depends(require_admin),
|
||
):
|
||
"""工具失败聚集(core/toolfail 同款扫描,页面用低阈值看全量;
|
||
巡检推送走 5 次/2 task 的高阈值)。admin-only。"""
|
||
from core.toolfail import scan_tool_failures
|
||
days = min(90, max(1, days))
|
||
return {
|
||
"days": days,
|
||
"clusters": scan_tool_failures(
|
||
days=days, min_count=max(1, min_count), min_tasks=max(1, min_tasks)
|
||
),
|
||
}
|
||
|
||
@app.get("/v1/admin/tool-wire-health", tags=["admin"])
|
||
def admin_tool_wire_health(
|
||
days: int = 7, user_id: UUID = Depends(require_admin),
|
||
):
|
||
"""工具参数 wire 损坏的抢救率，按模型档+工具聚合；admin-only、纯只读。"""
|
||
from core.toolfail import scan_tool_wire_health
|
||
return scan_tool_wire_health(days=days)
|
||
|
||
@app.get("/v1/admin/tiers", tags=["admin"])
|
||
def admin_tiers(user_id: UUID = Depends(require_admin)):
|
||
"""模型档位定义 + 全模型目录。admin-only。
|
||
|
||
UI:用户行的「档位」下拉用 tier 名;图例把每档 member id 映射成显示名。
|
||
default_tier 标出 plan 为空 / 未知时落的档。role=admin 始终全开(不在 tiers 里体现)。
|
||
"""
|
||
from core.model_access import DEFAULT_TIER
|
||
from core.agent_builder import load_config
|
||
tiers = load_config().get("model_tiers") or {}
|
||
return {
|
||
"tiers": tiers, # {name: [model_id, ...]}
|
||
"default_tier": DEFAULT_TIER,
|
||
"catalog": _model_catalog(), # [{id, display_name, kind}]
|
||
}
|
||
|
||
@app.patch("/v1/admin/users/{uid}/plan", tags=["admin"])
|
||
def admin_set_user_plan(
|
||
uid: str, body: SetPlanRequest, user_id: UUID = Depends(require_admin)
|
||
):
|
||
"""设置某用户的模型档位(写 users.plan)。admin-only。
|
||
|
||
plan 必须是 config/agent.yaml model_tiers 里存在的档位名;空串 = 清空(落 default 档)。
|
||
非法档位 → 400;用户不存在 → 404。
|
||
"""
|
||
from core.agent_builder import load_config
|
||
try:
|
||
target = UUID(uid)
|
||
except ValueError:
|
||
raise HTTPException(400, f"invalid user id: {uid!r}")
|
||
plan = (body.plan or "").strip()
|
||
tiers = load_config().get("model_tiers") or {}
|
||
if plan and plan not in tiers:
|
||
raise HTTPException(400, f"unknown tier {plan!r}; available: {sorted(tiers)}")
|
||
with session_scope() as s:
|
||
result = s.execute(
|
||
update(User).where(User.user_id == target).values(plan=plan or None)
|
||
)
|
||
if getattr(result, "rowcount", 0) == 0:
|
||
raise HTTPException(404, f"user not found: {uid}")
|
||
return {"user_id": str(target), "plan": plan}
|