297 lines
13 KiB
Python
297 lines
13 KiB
Python
"""web 层维护性后台协程(从 app.py lifespan 析出,2026-07-23)。
|
|
|
|
磁盘配额扫描 / 线程池监控 / 工具失败巡检 / sandbox 池初始化+reaper /
|
|
bg proc 清扫 / 孤儿 run 收割 / 优雅 drain。每个 start_* 返回 asyncio.Task
|
|
(或 None=未启用),关停统一走 `cancel_and_wait`。
|
|
定时任务引擎另见 scheduler_runner.py,微信入站另见 wechat_runner.py。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
from typing import Optional
|
|
|
|
from sqlalchemy import or_, update
|
|
|
|
from core.storage import session_scope
|
|
from core.storage.models import Task
|
|
|
|
from .broker import broker
|
|
from .common import INSTANCE
|
|
|
|
try:
|
|
import resource # Unix only;Windows dev 无此模块,RSS 监控自动降级跳过
|
|
except ImportError: # pragma: no cover - Windows
|
|
resource = None
|
|
|
|
|
|
async def cancel_and_wait(task: Optional[asyncio.Task]) -> None:
|
|
"""取消一个后台 task 并吞掉收尾异常(None 直接跳过)。"""
|
|
if task is None:
|
|
return
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except (asyncio.CancelledError, Exception):
|
|
pass
|
|
|
|
|
|
def reap_stale_runs() -> None:
|
|
"""Stale-run reaper:上次进程 crash 留下的 "running" / "cancelling" 已无 BG 线程
|
|
继续,启动时标 error,让对应 task 重新可发消息(否则 gate 永挂)。
|
|
蓝绿双实例(RUN.md B 档):ZCBOT_INSTANCE 在时只收 run_owner=自己(上次本色
|
|
实例的孤儿)或 NULL(0020 前的遗留行)的,不动另一实例正在跑的 run;单实例
|
|
部署 INSTANCE="" → 全量收,行为不变。
|
|
"""
|
|
with session_scope() as s:
|
|
stmt = update(Task).where(Task.run_status.in_(("running", "cancelling")))
|
|
if INSTANCE:
|
|
stmt = stmt.where(
|
|
or_(Task.run_owner == INSTANCE, Task.run_owner.is_(None))
|
|
)
|
|
result = s.execute(
|
|
stmt.values(
|
|
run_status="error",
|
|
run_error="server restarted before run finished",
|
|
)
|
|
)
|
|
if result.rowcount:
|
|
print(f"[startup] reaped {result.rowcount} stale active run(s)")
|
|
|
|
|
|
def start_disk_scanner(cfg: dict) -> asyncio.Task:
|
|
"""磁盘配额后台扫描(§7.5 #4 应用层 gate)── 不依赖 docker backend,host
|
|
backend 也跑(/v1/files/upload 也走配额 gate)。yaml `quotas.disk_scan_interval_seconds`
|
|
默 900s = 15min;limit_bytes ≤ 0 视为不限,scan 仍跑(用量统计有用),check 短路放行。
|
|
"""
|
|
from core.agent_builder import resolve_workspace
|
|
from core.storage.disk_quota import scan_all_users
|
|
disk_user_root = resolve_workspace(None, cfg) / "users"
|
|
quotas_cfg = cfg.get("quotas") or {}
|
|
interval = int(quotas_cfg.get("disk_scan_interval_seconds") or 900)
|
|
|
|
async def _disk_scanner() -> None:
|
|
loop = asyncio.get_running_loop()
|
|
# 启动时跑一次,后续按 interval。首次扫完 check 才能命中。
|
|
try:
|
|
n = await loop.run_in_executor(None, scan_all_users, disk_user_root)
|
|
if n:
|
|
print(f"[disk_scanner] initial scan: {n} user(s)")
|
|
except Exception as e:
|
|
print(f"[disk_scanner] initial scan error: {type(e).__name__}: {e}")
|
|
while True:
|
|
try:
|
|
await asyncio.sleep(interval)
|
|
n = await loop.run_in_executor(None, scan_all_users, disk_user_root)
|
|
if n:
|
|
print(f"[disk_scanner] scanned {n} user(s)")
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
print(f"[disk_scanner] error: {type(e).__name__}: {e}")
|
|
|
|
return asyncio.create_task(_disk_scanner(), name="disk-scanner")
|
|
|
|
|
|
def start_stats_logger(app, run_max_workers: int) -> asyncio.Task:
|
|
"""并发/线程池监控(§8.4):周期采样,只在有负载/刷新峰值时打,空闲不刷屏。
|
|
|
|
active_runs 来自 inflight(已提交未完成的 run,含排队中);逼近 max_workers 即
|
|
线程池排队,新 run 的 SSE 会卡着不吐 token。查看:journalctl -u zcbot | grep '\\[stats\\]'
|
|
"""
|
|
def _rss_peak_mb() -> Optional[float]:
|
|
if resource is None:
|
|
return None # Windows dev:降级,不打 rss
|
|
# Linux ru_maxrss 单位 KB,是峰值/high-water(单调不降 —— 看内存涨势够用)
|
|
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
|
|
|
|
async def _stats_logger() -> None:
|
|
peak = 0
|
|
while True:
|
|
try:
|
|
await asyncio.sleep(60)
|
|
active = len(app.state.inflight)
|
|
if active > peak:
|
|
peak = active
|
|
warn = " [WARN >= max_workers,已在排队]" if active >= run_max_workers else ""
|
|
print(f"[stats] new peak active_runs={active} "
|
|
f"max_workers={run_max_workers}{warn}")
|
|
if active > 0:
|
|
rss = _rss_peak_mb()
|
|
rss_s = f" rss_peak={rss:.0f}MB" if rss is not None else ""
|
|
print(f"[stats] active_runs={active} "
|
|
f"max_workers={run_max_workers} "
|
|
f"sse_subs={broker.total_subscribers()}{rss_s}")
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
print(f"[stats] error: {type(e).__name__}: {e}")
|
|
|
|
return asyncio.create_task(_stats_logger(), name="stats-logger")
|
|
|
|
|
|
def start_toolfail_scanner() -> Optional[asyncio.Task]:
|
|
"""工具失败聚集巡检(0.58.11)── 反 9dcae061 教训(mmdc 在生产挂 90 天
|
|
0 成功无人知,靠人工扫 DB 才发现):每天扫近 7 天 tool 错误消息,同签名
|
|
>=5 次且跨 >=2 task 判聚集,发 ZCBOT_DEVELOPER_EMAIL(复用 SMTP_*;未配
|
|
邮箱/SMTP 则只打日志)。签名进程内去重,且只发近 24h 仍活跃的聚集
|
|
(count_24h>0)—— 去重集是内存态,重启/蓝绿清零,高频部署期若不加活跃
|
|
过滤,每次部署都会把 7 天窗口内早已安静的存量聚集重发一遍(0.58.19 实改;
|
|
还在烧的重启后再提醒一次是刻意保留的)。ZCBOT_TOOLFAIL_SCAN_INTERVAL
|
|
秒,默 86400,<=0 整体关掉。
|
|
"""
|
|
dev_email = os.getenv("ZCBOT_DEVELOPER_EMAIL", "").strip()
|
|
interval = int(os.getenv("ZCBOT_TOOLFAIL_SCAN_INTERVAL", "").strip() or 86400)
|
|
if interval <= 0:
|
|
return None
|
|
|
|
async def _toolfail_scanner() -> None:
|
|
from core.toolfail import format_alert, scan_tool_failures
|
|
from tools.send_email import send_email_smtp, smtp_configured
|
|
loop = asyncio.get_running_loop()
|
|
alerted: set = set()
|
|
while True:
|
|
try:
|
|
clusters = await loop.run_in_executor(None, scan_tool_failures)
|
|
fresh = [c for c in clusters
|
|
if c.get("count_24h")
|
|
and (c["tool"], c["signature"]) not in alerted]
|
|
if fresh:
|
|
alerted.update((c["tool"], c["signature"]) for c in fresh)
|
|
for c in fresh:
|
|
print(f"[toolfail] {c['tool']}/{c['kind']} x{c['count']} "
|
|
f"tasks={c['task_count']}: {c['signature'][:80]}")
|
|
if dev_email and smtp_configured():
|
|
await loop.run_in_executor(
|
|
None, send_email_smtp, dev_email,
|
|
f"[zcbot] 工具失败聚集 {len(fresh)} 类",
|
|
format_alert(fresh, 7),
|
|
)
|
|
else:
|
|
print("[toolfail] ZCBOT_DEVELOPER_EMAIL/SMTP 未配,仅日志")
|
|
await asyncio.sleep(interval)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
print(f"[toolfail] error: {type(e).__name__}: {e}")
|
|
await asyncio.sleep(interval)
|
|
|
|
return asyncio.create_task(_toolfail_scanner(), name="toolfail-scanner")
|
|
|
|
|
|
def init_sandbox(app, cfg: dict) -> Optional[asyncio.Task]:
|
|
"""Sandbox pool(§7.5):仅当 ZCBOT_SANDBOX_BACKEND=docker 时启用。
|
|
|
|
启动钩子:① init_pool(创建 docker network + pool 实例)② shutdown_all 清
|
|
前驱孤儿(上次进程留下的 zcbot-sandbox-* 容器,内存 _last_active 为空,
|
|
全清重启)③ 后台 reaper task,每 60s 跑 reap_idle。返回 reaper task;
|
|
host backend 返 None。init 失败 fail-fast(Stage C 协议:hardening 缺失
|
|
视为部署未完成,不退化到 host —— 否则误以为有沙盒实则在裸跑)。
|
|
"""
|
|
backend = os.getenv("ZCBOT_SANDBOX_BACKEND", "host").lower()
|
|
if backend != "docker":
|
|
return None
|
|
from core.agent_builder import resolve_workspace
|
|
from core.paths import ROOT
|
|
from core.sandbox import init_pool
|
|
from core.sandbox.check import detect_fs_quota
|
|
user_root_base = resolve_workspace(None, cfg) / "users"
|
|
# §7.5 #4 fs quota 探测:不阻塞启动(应用层周期扫描已有),仅打 WARN
|
|
# 提醒外部用户开放前必须升级到 xfs prjquota / ext4 project / zfs。
|
|
try:
|
|
level, msg = detect_fs_quota(user_root_base.resolve())
|
|
print(f"[startup] {'[ok]' if level == 'ok' else '[warn]'} {msg}")
|
|
except Exception as e:
|
|
print(f"[startup] [warn] fs quota detect failed: {type(e).__name__}: {e}")
|
|
try:
|
|
# repo_root=ROOT 让 SandboxPool 把 <repo>/skills 只读 mount 进容器
|
|
# (fs 工具进容器后 read SKILL references 需要)
|
|
# sandbox_cfg=yaml `sandbox` 段(memory/cpus/pids_limit 可调)
|
|
pool = init_pool(
|
|
user_root_base, repo_root=ROOT,
|
|
sandbox_cfg=cfg.get("sandbox") or {},
|
|
)
|
|
removed = pool.shutdown_all()
|
|
if removed:
|
|
print(f"[startup] swept {len(removed)} stale sandbox container(s)")
|
|
|
|
async def _reaper() -> None:
|
|
loop = asyncio.get_running_loop()
|
|
while True:
|
|
try:
|
|
await asyncio.sleep(60)
|
|
removed = await loop.run_in_executor(None, pool.reap_idle)
|
|
if removed:
|
|
print(f"[reaper] reaped {len(removed)} idle sandbox container(s)")
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
print(f"[reaper] error: {type(e).__name__}: {e}")
|
|
|
|
app.state.sandbox_pool = pool
|
|
return asyncio.create_task(_reaper(), name="sandbox-reaper")
|
|
except Exception as e:
|
|
raise RuntimeError(
|
|
f"ZCBOT_SANDBOX_BACKEND=docker but sandbox init failed: {e}"
|
|
)
|
|
|
|
|
|
def start_proc_sweeper(cfg: dict) -> asyncio.Task:
|
|
"""bg proc 清扫(DESIGN §8.12):终态 proc 目录过 TTL 删除 + 已结束的
|
|
zcbot-proc-* 容器回收。host / docker 两种 backend 都要跑(host 模式只做
|
|
文件清扫,docker CLI 不在时 sweep 内部静默跳过容器部分)。每小时一次;
|
|
幂等,蓝绿双实例同时跑无害。启动后先跑一轮,把上个进程周期留下的
|
|
已结束容器/过期目录收掉。
|
|
"""
|
|
from core.agent_builder import resolve_workspace
|
|
from core.procs import sweep as _procs_sweep
|
|
users_base = resolve_workspace(None, cfg) / "users"
|
|
|
|
async def _proc_sweeper() -> None:
|
|
loop = asyncio.get_running_loop()
|
|
while True:
|
|
try:
|
|
stats = await loop.run_in_executor(
|
|
None, _procs_sweep, users_base
|
|
)
|
|
if stats["removed_dirs"] or stats["reaped_containers"]:
|
|
print(f"[proc-sweep] dirs={stats['removed_dirs']} "
|
|
f"containers={stats['reaped_containers']}")
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
print(f"[proc-sweep] error: {type(e).__name__}: {e}")
|
|
await asyncio.sleep(3600)
|
|
|
|
return asyncio.create_task(_proc_sweeper(), name="proc-sweeper")
|
|
|
|
|
|
async def drain_inflight(app, drain_timeout: int, cancel_grace: int) -> None:
|
|
"""优雅 drain:先拒新 run(draining 已置位),等在跑的 run 收尾,超时转协作式 cancel。
|
|
|
|
单实例形态下消除"restart 误杀 in-flight run 标 error"。新 POST /messages
|
|
期间返 503(客户端退避重试覆盖)。drain_timeout 内自然跑完 → idle 零 error;
|
|
超时的 broker.request_cancel → 下个 chunk 间隙退(标 idle);cancel_grace 后仍
|
|
没退的留给 systemd SIGKILL,下次启动 reaper 标 error(最坏退化 = 改前行为)。
|
|
★ systemd TimeoutStopSec 必须 > drain_timeout + cancel_grace + 余量(见 RUN.md)。
|
|
"""
|
|
inflight = app.state.inflight
|
|
if not inflight:
|
|
return
|
|
print(f"[shutdown] draining {len(inflight)} in-flight run(s), "
|
|
f"timeout={drain_timeout}s")
|
|
_, pending = await asyncio.wait(
|
|
list(inflight.keys()), timeout=drain_timeout
|
|
)
|
|
if pending:
|
|
print(f"[shutdown] {len(pending)} run(s) over drain timeout; "
|
|
f"signalling cooperative cancel")
|
|
for t in pending:
|
|
cid = inflight.get(t)
|
|
if cid is not None:
|
|
broker.request_cancel(cid)
|
|
_, still = await asyncio.wait(pending, timeout=cancel_grace)
|
|
if still:
|
|
print(f"[shutdown] {len(still)} run(s) still active after "
|
|
f"cancel grace; SIGKILL takes over, next start reaps them")
|