326 lines
14 KiB
Python
326 lines
14 KiB
Python
"""web 层维护性后台协程(从 app.py lifespan 析出,2026-07-23)。
|
||
|
||
磁盘配额扫描 / 线程池监控 / 工具失败巡检 / sandbox 池初始化+reaper /
|
||
bg proc 清扫 / 孤儿 run 收割 / 优雅 drain。每个 start_* 返回 asyncio.Task
|
||
(或 None=未启用),关停统一走 `cancel_and_wait`。
|
||
定时任务引擎另见 scheduler_runner.py,微信入站另见 wechat_runner.py。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import importlib
|
||
import os
|
||
from types import ModuleType
|
||
from typing import Optional
|
||
|
||
from sqlalchemy import or_, update
|
||
|
||
from core.storage import session_scope
|
||
from core.storage.models import Task
|
||
|
||
from .broker import broker
|
||
from .common import INSTANCE
|
||
|
||
resource: ModuleType | None
|
||
try:
|
||
resource = importlib.import_module("resource")
|
||
except ImportError: # pragma: no cover - Windows
|
||
resource = None
|
||
|
||
|
||
async def cancel_and_wait(task: Optional[asyncio.Task]) -> None:
|
||
"""取消一个后台 task 并吞掉收尾异常(None 直接跳过)。"""
|
||
if task is None:
|
||
return
|
||
task.cancel()
|
||
try:
|
||
await task
|
||
except (asyncio.CancelledError, Exception):
|
||
pass
|
||
|
||
|
||
def reap_stale_runs() -> None:
|
||
"""Stale-run reaper:上次进程 crash 留下的 "running" / "cancelling" 已无 BG 线程
|
||
继续,启动时标 error,让对应 task 重新可发消息(否则 gate 永挂)。
|
||
蓝绿双实例(RUN.md B 档):ZCBOT_INSTANCE 在时只收 run_owner=自己(上次本色
|
||
实例的孤儿)或 NULL(0020 前的遗留行)的,不动另一实例正在跑的 run;单实例
|
||
部署 INSTANCE="" → 全量收,行为不变。
|
||
"""
|
||
with session_scope() as s:
|
||
stmt = update(Task).where(Task.run_status.in_(("running", "cancelling")))
|
||
if INSTANCE:
|
||
stmt = stmt.where(
|
||
or_(Task.run_owner == INSTANCE, Task.run_owner.is_(None))
|
||
)
|
||
result = s.execute(
|
||
stmt.values(
|
||
run_status="error",
|
||
run_error="server restarted before run finished",
|
||
)
|
||
)
|
||
rowcount = int(getattr(result, "rowcount", 0) or 0)
|
||
if rowcount:
|
||
print(f"[startup] reaped {rowcount} stale active run(s)")
|
||
|
||
|
||
def start_disk_scanner(cfg: dict) -> asyncio.Task:
|
||
"""磁盘配额后台扫描(§7.5 #4 应用层 gate)── 不依赖 docker backend,host
|
||
backend 也跑(/v1/files/upload 也走配额 gate)。yaml `quotas.disk_scan_interval_seconds`
|
||
默 900s = 15min;limit_bytes ≤ 0 视为不限,scan 仍跑(用量统计有用),check 短路放行。
|
||
"""
|
||
from core.agent_builder import resolve_workspace
|
||
from core.storage.disk_quota import scan_all_users
|
||
disk_user_root = resolve_workspace(None, cfg) / "users"
|
||
quotas_cfg = cfg.get("quotas") or {}
|
||
interval = int(quotas_cfg.get("disk_scan_interval_seconds") or 900)
|
||
|
||
async def _disk_scanner() -> None:
|
||
loop = asyncio.get_running_loop()
|
||
# 启动时跑一次,后续按 interval。首次扫完 check 才能命中。
|
||
try:
|
||
n = await loop.run_in_executor(None, scan_all_users, disk_user_root)
|
||
if n:
|
||
print(f"[disk_scanner] initial scan: {n} user(s)")
|
||
except Exception as e:
|
||
print(f"[disk_scanner] initial scan error: {type(e).__name__}: {e}")
|
||
while True:
|
||
try:
|
||
await asyncio.sleep(interval)
|
||
n = await loop.run_in_executor(None, scan_all_users, disk_user_root)
|
||
if n:
|
||
print(f"[disk_scanner] scanned {n} user(s)")
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as e:
|
||
print(f"[disk_scanner] error: {type(e).__name__}: {e}")
|
||
|
||
return asyncio.create_task(_disk_scanner(), name="disk-scanner")
|
||
|
||
|
||
def start_stats_logger(app, run_max_workers: int) -> asyncio.Task:
|
||
"""并发/线程池监控(§8.4):周期采样,只在有负载/刷新峰值时打,空闲不刷屏。
|
||
|
||
active_runs 来自 inflight(已提交未完成的 run,含排队中);逼近 max_workers 即
|
||
线程池排队,新 run 的 SSE 会卡着不吐 token。查看:journalctl -u zcbot | grep '\\[stats\\]'
|
||
"""
|
||
def _rss_peak_mb() -> Optional[float]:
|
||
if resource is None:
|
||
return None # Windows dev:降级,不打 rss
|
||
# Linux ru_maxrss 单位 KB,是峰值/high-water(单调不降 —— 看内存涨势够用)
|
||
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
|
||
|
||
async def _stats_logger() -> None:
|
||
peak = 0
|
||
while True:
|
||
try:
|
||
await asyncio.sleep(60)
|
||
active = len(app.state.inflight)
|
||
if active > peak:
|
||
peak = active
|
||
warn = " [WARN >= max_workers,已在排队]" if active >= run_max_workers else ""
|
||
print(f"[stats] new peak active_runs={active} "
|
||
f"max_workers={run_max_workers}{warn}")
|
||
if active > 0:
|
||
rss = _rss_peak_mb()
|
||
rss_s = f" rss_peak={rss:.0f}MB" if rss is not None else ""
|
||
print(f"[stats] active_runs={active} "
|
||
f"max_workers={run_max_workers} "
|
||
f"sse_subs={broker.total_subscribers()}{rss_s}")
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as e:
|
||
print(f"[stats] error: {type(e).__name__}: {e}")
|
||
|
||
return asyncio.create_task(_stats_logger(), name="stats-logger")
|
||
|
||
|
||
def start_toolfail_scanner() -> Optional[asyncio.Task]:
|
||
"""工具失败聚集巡检(0.58.11)── 反 9dcae061 教训(mmdc 在生产挂 90 天
|
||
0 成功无人知,靠人工扫 DB 才发现):每天扫近 7 天 tool 错误消息,同签名
|
||
>=5 次且跨 >=2 task 判聚集,发 ZCBOT_DEVELOPER_EMAIL(复用 SMTP_*;未配
|
||
邮箱/SMTP 则只打日志)。签名进程内去重,且只发近 24h 仍活跃的聚集
|
||
(count_24h>0)—— 去重集是内存态,重启/蓝绿清零,高频部署期若不加活跃
|
||
过滤,每次部署都会把 7 天窗口内早已安静的存量聚集重发一遍(0.58.19 实改;
|
||
还在烧的重启后再提醒一次是刻意保留的)。ZCBOT_TOOLFAIL_SCAN_INTERVAL
|
||
秒,默 86400,<=0 整体关掉。
|
||
"""
|
||
dev_email = os.getenv("ZCBOT_DEVELOPER_EMAIL", "").strip()
|
||
interval = int(os.getenv("ZCBOT_TOOLFAIL_SCAN_INTERVAL", "").strip() or 86400)
|
||
if interval <= 0:
|
||
return None
|
||
|
||
async def _toolfail_scanner() -> None:
|
||
from core.toolfail import format_alert, scan_tool_failures
|
||
from tools.send_email import send_email_smtp, smtp_configured
|
||
loop = asyncio.get_running_loop()
|
||
alerted: set = set()
|
||
while True:
|
||
try:
|
||
clusters = await loop.run_in_executor(None, scan_tool_failures)
|
||
fresh = [c for c in clusters
|
||
if c.get("count_24h")
|
||
and (c["tool"], c["signature"]) not in alerted]
|
||
if fresh:
|
||
alerted.update((c["tool"], c["signature"]) for c in fresh)
|
||
for c in fresh:
|
||
print(f"[toolfail] {c['tool']}/{c['kind']} x{c['count']} "
|
||
f"tasks={c['task_count']}: {c['signature'][:80]}")
|
||
if dev_email and smtp_configured():
|
||
await loop.run_in_executor(
|
||
None, send_email_smtp, dev_email,
|
||
f"[zcbot] 工具失败聚集 {len(fresh)} 类",
|
||
format_alert(fresh, 7),
|
||
)
|
||
else:
|
||
print("[toolfail] ZCBOT_DEVELOPER_EMAIL/SMTP 未配,仅日志")
|
||
await asyncio.sleep(interval)
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as e:
|
||
print(f"[toolfail] error: {type(e).__name__}: {e}")
|
||
await asyncio.sleep(interval)
|
||
|
||
return asyncio.create_task(_toolfail_scanner(), name="toolfail-scanner")
|
||
|
||
|
||
def start_provider_scanner() -> asyncio.Task:
|
||
"""每分钟寻找到期 Provider；实际一轮由 PostgreSQL advisory lock 单实例执行。"""
|
||
async def _scanner() -> None:
|
||
from core.provider_credentials.monitor import run_due_checks
|
||
loop = asyncio.get_running_loop()
|
||
while True:
|
||
try:
|
||
await asyncio.sleep(5)
|
||
checked = await loop.run_in_executor(None, run_due_checks)
|
||
if checked:
|
||
print(f"[provider] checked {checked} provider(s)")
|
||
await asyncio.sleep(55)
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as exc:
|
||
print(f"[provider] scanner error: {type(exc).__name__}")
|
||
await asyncio.sleep(60)
|
||
return asyncio.create_task(_scanner(), name="provider-scanner")
|
||
|
||
|
||
def init_sandbox(app, cfg: dict) -> Optional[asyncio.Task]:
|
||
"""Sandbox pool(§7.5):仅当 ZCBOT_SANDBOX_BACKEND=docker 时启用。
|
||
|
||
启动钩子:① init_pool(创建 docker network + pool 实例)② shutdown_all 清
|
||
前驱孤儿(上次进程留下的 zcbot-sandbox-* 容器,内存 _last_active 为空,
|
||
全清重启)③ 后台 reaper task,每 60s 跑 reap_idle。返回 reaper task;
|
||
host backend 返 None。init 失败 fail-fast(Stage C 协议:hardening 缺失
|
||
视为部署未完成,不退化到 host —— 否则误以为有沙盒实则在裸跑)。
|
||
"""
|
||
backend = os.getenv("ZCBOT_SANDBOX_BACKEND", "host").lower()
|
||
if backend != "docker":
|
||
return None
|
||
from core.agent_builder import resolve_workspace
|
||
from core.paths import ROOT
|
||
from core.sandbox import init_pool
|
||
from core.sandbox.check import detect_fs_quota
|
||
user_root_base = resolve_workspace(None, cfg) / "users"
|
||
# §7.5 #4 fs quota 探测:不阻塞启动(应用层周期扫描已有),仅打 WARN
|
||
# 提醒外部用户开放前必须升级到 xfs prjquota / ext4 project / zfs。
|
||
try:
|
||
level, msg = detect_fs_quota(user_root_base.resolve())
|
||
print(f"[startup] {'[ok]' if level == 'ok' else '[warn]'} {msg}")
|
||
except Exception as e:
|
||
print(f"[startup] [warn] fs quota detect failed: {type(e).__name__}: {e}")
|
||
try:
|
||
# repo_root=ROOT 让 SandboxPool 把 <repo>/skills 只读 mount 进容器
|
||
# (fs 工具进容器后 read SKILL references 需要)
|
||
# sandbox_cfg=yaml `sandbox` 段(memory/cpus/pids_limit 可调)
|
||
pool = init_pool(
|
||
user_root_base, repo_root=ROOT,
|
||
sandbox_cfg=cfg.get("sandbox") or {},
|
||
)
|
||
removed = pool.shutdown_all()
|
||
if removed:
|
||
print(f"[startup] swept {len(removed)} stale sandbox container(s)")
|
||
|
||
async def _reaper() -> None:
|
||
loop = asyncio.get_running_loop()
|
||
while True:
|
||
try:
|
||
await asyncio.sleep(60)
|
||
removed = await loop.run_in_executor(None, pool.reap_idle)
|
||
if removed:
|
||
print(f"[reaper] reaped {len(removed)} idle sandbox container(s)")
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as e:
|
||
print(f"[reaper] error: {type(e).__name__}: {e}")
|
||
|
||
app.state.sandbox_pool = pool
|
||
return asyncio.create_task(_reaper(), name="sandbox-reaper")
|
||
except Exception as e:
|
||
raise RuntimeError(
|
||
f"ZCBOT_SANDBOX_BACKEND=docker but sandbox init failed: {e}"
|
||
)
|
||
|
||
|
||
def start_proc_sweeper(cfg: dict) -> asyncio.Task:
|
||
"""bg proc 清扫(DESIGN §8.12):终态 proc 目录过 TTL 删除 + 已结束的
|
||
zcbot-proc-* 容器回收。host / docker 两种 backend 都要跑(host 模式只做
|
||
文件清扫,docker CLI 不在时 sweep 内部静默跳过容器部分)。每小时一次;
|
||
幂等,蓝绿双实例同时跑无害。启动后先跑一轮,把上个进程周期留下的
|
||
已结束容器/过期目录收掉。
|
||
"""
|
||
from core.agent_builder import resolve_workspace
|
||
from core.procs import dispatch_queued, sweep as _procs_sweep
|
||
users_base = resolve_workspace(None, cfg) / "users"
|
||
|
||
async def _proc_sweeper() -> None:
|
||
loop = asyncio.get_running_loop()
|
||
while True:
|
||
try:
|
||
stats = await loop.run_in_executor(
|
||
None, _procs_sweep, users_base
|
||
)
|
||
from core.sandbox import get_pool
|
||
pool = get_pool()
|
||
started = await loop.run_in_executor(None, dispatch_queued, users_base, pool) if pool is not None else 0
|
||
if stats["removed_dirs"] or stats["reaped_containers"]:
|
||
print(f"[proc-sweep] dirs={stats['removed_dirs']} "
|
||
f"containers={stats['reaped_containers']}")
|
||
if started:
|
||
print(f"[proc-sweep] started queued={started}")
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as e:
|
||
print(f"[proc-sweep] error: {type(e).__name__}: {e}")
|
||
await asyncio.sleep(30)
|
||
|
||
return asyncio.create_task(_proc_sweeper(), name="proc-sweeper")
|
||
|
||
|
||
async def drain_inflight(app, drain_timeout: int, cancel_grace: int) -> None:
|
||
"""优雅 drain:先拒新 run(draining 已置位),等在跑的 run 收尾,超时转协作式 cancel。
|
||
|
||
单实例形态下消除"restart 误杀 in-flight run 标 error"。新 POST /messages
|
||
期间返 503(客户端退避重试覆盖)。drain_timeout 内自然跑完 → idle 零 error;
|
||
超时的 broker.request_cancel → 下个 chunk 间隙退(标 idle);cancel_grace 后仍
|
||
没退的留给 systemd SIGKILL,下次启动 reaper 标 error(最坏退化 = 改前行为)。
|
||
★ systemd TimeoutStopSec 必须 > drain_timeout + cancel_grace + 余量(见 RUN.md)。
|
||
"""
|
||
inflight = app.state.inflight
|
||
if not inflight:
|
||
return
|
||
print(f"[shutdown] draining {len(inflight)} in-flight run(s), "
|
||
f"timeout={drain_timeout}s")
|
||
_, pending = await asyncio.wait(
|
||
list(inflight.keys()), timeout=drain_timeout
|
||
)
|
||
if pending:
|
||
print(f"[shutdown] {len(pending)} run(s) over drain timeout; "
|
||
f"signalling cooperative cancel")
|
||
for t in pending:
|
||
cid = inflight.get(t)
|
||
if cid is not None:
|
||
broker.request_cancel(cid)
|
||
_, still = await asyncio.wait(pending, timeout=cancel_grace)
|
||
if still:
|
||
print(f"[shutdown] {len(still)} run(s) still active after "
|
||
f"cancel grace; SIGKILL takes over, next start reaps them")
|