"""web 层维护性后台协程(从 app.py lifespan 析出,2026-07-23)。 磁盘配额扫描 / 线程池监控 / 工具失败巡检 / sandbox 池初始化+reaper / bg proc 清扫 / 孤儿 run 收割 / 优雅 drain。每个 start_* 返回 asyncio.Task (或 None=未启用),关停统一走 `cancel_and_wait`。 定时任务引擎另见 scheduler_runner.py,微信入站另见 wechat_runner.py。 """ from __future__ import annotations import asyncio import importlib import os from types import ModuleType from typing import Optional from sqlalchemy import or_, update from core.storage import session_scope from core.storage.models import Task from .broker import broker from .common import INSTANCE resource: ModuleType | None try: resource = importlib.import_module("resource") except ImportError: # pragma: no cover - Windows resource = None async def cancel_and_wait(task: Optional[asyncio.Task]) -> None: """取消一个后台 task 并吞掉收尾异常(None 直接跳过)。""" if task is None: return task.cancel() try: await task except (asyncio.CancelledError, Exception): pass def reap_stale_runs() -> None: """Stale-run reaper:上次进程 crash 留下的 "running" / "cancelling" 已无 BG 线程 继续,启动时标 error,让对应 task 重新可发消息(否则 gate 永挂)。 蓝绿双实例(RUN.md B 档):ZCBOT_INSTANCE 在时只收 run_owner=自己(上次本色 实例的孤儿)或 NULL(0020 前的遗留行)的,不动另一实例正在跑的 run;单实例 部署 INSTANCE="" → 全量收,行为不变。 """ with session_scope() as s: stmt = update(Task).where(Task.run_status.in_(("running", "cancelling"))) if INSTANCE: stmt = stmt.where( or_(Task.run_owner == INSTANCE, Task.run_owner.is_(None)) ) result = s.execute( stmt.values( run_status="error", run_error="server restarted before run finished", ) ) rowcount = int(getattr(result, "rowcount", 0) or 0) if rowcount: print(f"[startup] reaped {rowcount} stale active run(s)") def start_disk_scanner(cfg: dict) -> asyncio.Task: """磁盘配额后台扫描(§7.5 #4 应用层 gate)── 不依赖 docker backend,host backend 也跑(/v1/files/upload 也走配额 gate)。yaml `quotas.disk_scan_interval_seconds` 默 900s = 15min;limit_bytes ≤ 0 视为不限,scan 仍跑(用量统计有用),check 短路放行。 """ from core.agent_builder import resolve_workspace from core.storage.disk_quota import scan_all_users disk_user_root = resolve_workspace(None, cfg) / "users" quotas_cfg = cfg.get("quotas") or {} interval = int(quotas_cfg.get("disk_scan_interval_seconds") or 900) async def _disk_scanner() -> None: loop = asyncio.get_running_loop() # 启动时跑一次,后续按 interval。首次扫完 check 才能命中。 try: n = await loop.run_in_executor(None, scan_all_users, disk_user_root) if n: print(f"[disk_scanner] initial scan: {n} user(s)") except Exception as e: print(f"[disk_scanner] initial scan error: {type(e).__name__}: {e}") while True: try: await asyncio.sleep(interval) n = await loop.run_in_executor(None, scan_all_users, disk_user_root) if n: print(f"[disk_scanner] scanned {n} user(s)") except asyncio.CancelledError: raise except Exception as e: print(f"[disk_scanner] error: {type(e).__name__}: {e}") return asyncio.create_task(_disk_scanner(), name="disk-scanner") def start_stats_logger(app, run_max_workers: int) -> asyncio.Task: """并发/线程池监控(§8.4):周期采样,只在有负载/刷新峰值时打,空闲不刷屏。 active_runs 来自 inflight(已提交未完成的 run,含排队中);逼近 max_workers 即 线程池排队,新 run 的 SSE 会卡着不吐 token。查看:journalctl -u zcbot | grep '\\[stats\\]' """ def _rss_peak_mb() -> Optional[float]: if resource is None: return None # Windows dev:降级,不打 rss # Linux ru_maxrss 单位 KB,是峰值/high-water(单调不降 —— 看内存涨势够用) return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 async def _stats_logger() -> None: peak = 0 while True: try: await asyncio.sleep(60) active = len(app.state.inflight) if active > peak: peak = active warn = " [WARN >= max_workers,已在排队]" if active >= run_max_workers else "" print(f"[stats] new peak active_runs={active} " f"max_workers={run_max_workers}{warn}") if active > 0: rss = _rss_peak_mb() rss_s = f" rss_peak={rss:.0f}MB" if rss is not None else "" print(f"[stats] active_runs={active} " f"max_workers={run_max_workers} " f"sse_subs={broker.total_subscribers()}{rss_s}") except asyncio.CancelledError: raise except Exception as e: print(f"[stats] error: {type(e).__name__}: {e}") return asyncio.create_task(_stats_logger(), name="stats-logger") def start_toolfail_scanner() -> Optional[asyncio.Task]: """工具失败聚集巡检(0.58.11)── 反 9dcae061 教训(mmdc 在生产挂 90 天 0 成功无人知,靠人工扫 DB 才发现):每天扫近 7 天 tool 错误消息,同签名 >=5 次且跨 >=2 task 判聚集,发 ZCBOT_DEVELOPER_EMAIL(复用 SMTP_*;未配 邮箱/SMTP 则只打日志)。签名进程内去重,且只发近 24h 仍活跃的聚集 (count_24h>0)—— 去重集是内存态,重启/蓝绿清零,高频部署期若不加活跃 过滤,每次部署都会把 7 天窗口内早已安静的存量聚集重发一遍(0.58.19 实改; 还在烧的重启后再提醒一次是刻意保留的)。ZCBOT_TOOLFAIL_SCAN_INTERVAL 秒,默 86400,<=0 整体关掉。 """ dev_email = os.getenv("ZCBOT_DEVELOPER_EMAIL", "").strip() interval = int(os.getenv("ZCBOT_TOOLFAIL_SCAN_INTERVAL", "").strip() or 86400) if interval <= 0: return None async def _toolfail_scanner() -> None: from core.toolfail import format_alert, scan_tool_failures from tools.send_email import send_email_smtp, smtp_configured loop = asyncio.get_running_loop() alerted: set = set() while True: try: clusters = await loop.run_in_executor(None, scan_tool_failures) fresh = [c for c in clusters if c.get("count_24h") and (c["tool"], c["signature"]) not in alerted] if fresh: alerted.update((c["tool"], c["signature"]) for c in fresh) for c in fresh: print(f"[toolfail] {c['tool']}/{c['kind']} x{c['count']} " f"tasks={c['task_count']}: {c['signature'][:80]}") if dev_email and smtp_configured(): await loop.run_in_executor( None, send_email_smtp, dev_email, f"[zcbot] 工具失败聚集 {len(fresh)} 类", format_alert(fresh, 7), ) else: print("[toolfail] ZCBOT_DEVELOPER_EMAIL/SMTP 未配,仅日志") await asyncio.sleep(interval) except asyncio.CancelledError: raise except Exception as e: print(f"[toolfail] error: {type(e).__name__}: {e}") await asyncio.sleep(interval) return asyncio.create_task(_toolfail_scanner(), name="toolfail-scanner") def init_sandbox(app, cfg: dict) -> Optional[asyncio.Task]: """Sandbox pool(§7.5):仅当 ZCBOT_SANDBOX_BACKEND=docker 时启用。 启动钩子:① init_pool(创建 docker network + pool 实例)② shutdown_all 清 前驱孤儿(上次进程留下的 zcbot-sandbox-* 容器,内存 _last_active 为空, 全清重启)③ 后台 reaper task,每 60s 跑 reap_idle。返回 reaper task; host backend 返 None。init 失败 fail-fast(Stage C 协议:hardening 缺失 视为部署未完成,不退化到 host —— 否则误以为有沙盒实则在裸跑)。 """ backend = os.getenv("ZCBOT_SANDBOX_BACKEND", "host").lower() if backend != "docker": return None from core.agent_builder import resolve_workspace from core.paths import ROOT from core.sandbox import init_pool from core.sandbox.check import detect_fs_quota user_root_base = resolve_workspace(None, cfg) / "users" # §7.5 #4 fs quota 探测:不阻塞启动(应用层周期扫描已有),仅打 WARN # 提醒外部用户开放前必须升级到 xfs prjquota / ext4 project / zfs。 try: level, msg = detect_fs_quota(user_root_base.resolve()) print(f"[startup] {'[ok]' if level == 'ok' else '[warn]'} {msg}") except Exception as e: print(f"[startup] [warn] fs quota detect failed: {type(e).__name__}: {e}") try: # repo_root=ROOT 让 SandboxPool 把 /skills 只读 mount 进容器 # (fs 工具进容器后 read SKILL references 需要) # sandbox_cfg=yaml `sandbox` 段(memory/cpus/pids_limit 可调) pool = init_pool( user_root_base, repo_root=ROOT, sandbox_cfg=cfg.get("sandbox") or {}, ) removed = pool.shutdown_all() if removed: print(f"[startup] swept {len(removed)} stale sandbox container(s)") async def _reaper() -> None: loop = asyncio.get_running_loop() while True: try: await asyncio.sleep(60) removed = await loop.run_in_executor(None, pool.reap_idle) if removed: print(f"[reaper] reaped {len(removed)} idle sandbox container(s)") except asyncio.CancelledError: raise except Exception as e: print(f"[reaper] error: {type(e).__name__}: {e}") app.state.sandbox_pool = pool return asyncio.create_task(_reaper(), name="sandbox-reaper") except Exception as e: raise RuntimeError( f"ZCBOT_SANDBOX_BACKEND=docker but sandbox init failed: {e}" ) def start_proc_sweeper(cfg: dict) -> asyncio.Task: """bg proc 清扫(DESIGN §8.12):终态 proc 目录过 TTL 删除 + 已结束的 zcbot-proc-* 容器回收。host / docker 两种 backend 都要跑(host 模式只做 文件清扫,docker CLI 不在时 sweep 内部静默跳过容器部分)。每小时一次; 幂等,蓝绿双实例同时跑无害。启动后先跑一轮,把上个进程周期留下的 已结束容器/过期目录收掉。 """ from core.agent_builder import resolve_workspace from core.procs import sweep as _procs_sweep users_base = resolve_workspace(None, cfg) / "users" async def _proc_sweeper() -> None: loop = asyncio.get_running_loop() while True: try: stats = await loop.run_in_executor( None, _procs_sweep, users_base ) if stats["removed_dirs"] or stats["reaped_containers"]: print(f"[proc-sweep] dirs={stats['removed_dirs']} " f"containers={stats['reaped_containers']}") except asyncio.CancelledError: raise except Exception as e: print(f"[proc-sweep] error: {type(e).__name__}: {e}") await asyncio.sleep(3600) return asyncio.create_task(_proc_sweeper(), name="proc-sweeper") async def drain_inflight(app, drain_timeout: int, cancel_grace: int) -> None: """优雅 drain:先拒新 run(draining 已置位),等在跑的 run 收尾,超时转协作式 cancel。 单实例形态下消除"restart 误杀 in-flight run 标 error"。新 POST /messages 期间返 503(客户端退避重试覆盖)。drain_timeout 内自然跑完 → idle 零 error; 超时的 broker.request_cancel → 下个 chunk 间隙退(标 idle);cancel_grace 后仍 没退的留给 systemd SIGKILL,下次启动 reaper 标 error(最坏退化 = 改前行为)。 ★ systemd TimeoutStopSec 必须 > drain_timeout + cancel_grace + 余量(见 RUN.md)。 """ inflight = app.state.inflight if not inflight: return print(f"[shutdown] draining {len(inflight)} in-flight run(s), " f"timeout={drain_timeout}s") _, pending = await asyncio.wait( list(inflight.keys()), timeout=drain_timeout ) if pending: print(f"[shutdown] {len(pending)} run(s) over drain timeout; " f"signalling cooperative cancel") for t in pending: cid = inflight.get(t) if cid is not None: broker.request_cancel(cid) _, still = await asyncio.wait(pending, timeout=cancel_grace) if still: print(f"[shutdown] {len(still)} run(s) still active after " f"cancel grace; SIGKILL takes over, next start reaps them")