"""web 层维护性后台协程(从 app.py lifespan 析出,2026-07-23)。

磁盘配额扫描 / 线程池监控 / 工具失败巡检 / sandbox 池初始化+reaper /
bg proc 清扫 / 孤儿 run 收割 / 优雅 drain。每个 start_* 返回 asyncio.Task
(或 None=未启用),关停统一走 `cancel_and_wait`。
定时任务引擎另见 scheduler_runner.py,微信入站另见 wechat_runner.py。
"""
from __future__ import annotations

import asyncio
import importlib
import os
from types import ModuleType
from typing import Optional

from sqlalchemy import or_, update

from core.storage import session_scope
from core.storage.models import Task

from .broker import broker
from .common import INSTANCE

resource: ModuleType | None
try:
    resource = importlib.import_module("resource")
except ImportError:  # pragma: no cover - Windows
    resource = None


async def cancel_and_wait(task: Optional[asyncio.Task]) -> None:
    """取消一个后台 task 并吞掉收尾异常(None 直接跳过)。"""
    if task is None:
        return
    task.cancel()
    try:
        await task
    except (asyncio.CancelledError, Exception):
        pass


def reap_stale_runs() -> None:
    """Stale-run reaper:上次进程 crash 留下的 "running" / "cancelling" 已无 BG 线程
    继续,启动时标 error,让对应 task 重新可发消息(否则 gate 永挂)。
    蓝绿双实例(RUN.md B 档):ZCBOT_INSTANCE 在时只收 run_owner=自己(上次本色
    实例的孤儿)或 NULL(0020 前的遗留行)的,不动另一实例正在跑的 run;单实例
    部署 INSTANCE="" → 全量收,行为不变。
    """
    with session_scope() as s:
        stmt = update(Task).where(Task.run_status.in_(("running", "cancelling")))
        if INSTANCE:
            stmt = stmt.where(
                or_(Task.run_owner == INSTANCE, Task.run_owner.is_(None))
            )
        result = s.execute(
            stmt.values(
                run_status="error",
                run_error="server restarted before run finished",
            )
        )
        rowcount = int(getattr(result, "rowcount", 0) or 0)
        if rowcount:
            print(f"[startup] reaped {rowcount} stale active run(s)")


def start_disk_scanner(cfg: dict) -> asyncio.Task:
    """磁盘配额后台扫描(§7.5 #4 应用层 gate)── 不依赖 docker backend,host
    backend 也跑(/v1/files/upload 也走配额 gate)。yaml `quotas.disk_scan_interval_seconds`
    默 900s = 15min;limit_bytes ≤ 0 视为不限,scan 仍跑(用量统计有用),check 短路放行。
    """
    from core.agent_builder import resolve_workspace
    from core.storage.disk_quota import scan_all_users
    disk_user_root = resolve_workspace(None, cfg) / "users"
    quotas_cfg = cfg.get("quotas") or {}
    interval = int(quotas_cfg.get("disk_scan_interval_seconds") or 900)

    async def _disk_scanner() -> None:
        loop = asyncio.get_running_loop()
        # 启动时跑一次,后续按 interval。首次扫完 check 才能命中。
        try:
            n = await loop.run_in_executor(None, scan_all_users, disk_user_root)
            if n:
                print(f"[disk_scanner] initial scan: {n} user(s)")
        except Exception as e:
            print(f"[disk_scanner] initial scan error: {type(e).__name__}: {e}")
        while True:
            try:
                await asyncio.sleep(interval)
                n = await loop.run_in_executor(None, scan_all_users, disk_user_root)
                if n:
                    print(f"[disk_scanner] scanned {n} user(s)")
            except asyncio.CancelledError:
                raise
            except Exception as e:
                print(f"[disk_scanner] error: {type(e).__name__}: {e}")

    return asyncio.create_task(_disk_scanner(), name="disk-scanner")


def start_stats_logger(app, run_max_workers: int) -> asyncio.Task:
    """并发/线程池监控(§8.4):周期采样,只在有负载/刷新峰值时打,空闲不刷屏。

    active_runs 来自 inflight(已提交未完成的 run,含排队中);逼近 max_workers 即
    线程池排队,新 run 的 SSE 会卡着不吐 token。查看:journalctl -u zcbot | grep '\\[stats\\]'
    """
    def _rss_peak_mb() -> Optional[float]:
        if resource is None:
            return None  # Windows dev:降级,不打 rss
        # Linux ru_maxrss 单位 KB,是峰值/high-water(单调不降 —— 看内存涨势够用)
        return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024

    async def _stats_logger() -> None:
        peak = 0
        while True:
            try:
                await asyncio.sleep(60)
                active = len(app.state.inflight)
                if active > peak:
                    peak = active
                    warn = " [WARN >= max_workers,已在排队]" if active >= run_max_workers else ""
                    print(f"[stats] new peak active_runs={active} "
                          f"max_workers={run_max_workers}{warn}")
                if active > 0:
                    rss = _rss_peak_mb()
                    rss_s = f" rss_peak={rss:.0f}MB" if rss is not None else ""
                    print(f"[stats] active_runs={active} "
                          f"max_workers={run_max_workers} "
                          f"sse_subs={broker.total_subscribers()}{rss_s}")
            except asyncio.CancelledError:
                raise
            except Exception as e:
                print(f"[stats] error: {type(e).__name__}: {e}")

    return asyncio.create_task(_stats_logger(), name="stats-logger")


def start_toolfail_scanner() -> Optional[asyncio.Task]:
    """工具失败聚集巡检(0.58.11)── 反 9dcae061 教训(mmdc 在生产挂 90 天
    0 成功无人知,靠人工扫 DB 才发现):每天扫近 7 天 tool 错误消息,同签名
    >=5 次且跨 >=2 task 判聚集,发 ZCBOT_DEVELOPER_EMAIL(复用 SMTP_*;未配
    邮箱/SMTP 则只打日志)。签名进程内去重,且只发近 24h 仍活跃的聚集
    (count_24h>0)—— 去重集是内存态,重启/蓝绿清零,高频部署期若不加活跃
    过滤,每次部署都会把 7 天窗口内早已安静的存量聚集重发一遍(0.58.19 实改;
    还在烧的重启后再提醒一次是刻意保留的)。ZCBOT_TOOLFAIL_SCAN_INTERVAL
    秒,默 86400,<=0 整体关掉。
    """
    dev_email = os.getenv("ZCBOT_DEVELOPER_EMAIL", "").strip()
    interval = int(os.getenv("ZCBOT_TOOLFAIL_SCAN_INTERVAL", "").strip() or 86400)
    if interval <= 0:
        return None

    async def _toolfail_scanner() -> None:
        from core.toolfail import format_alert, scan_tool_failures
        from tools.send_email import send_email_smtp, smtp_configured
        loop = asyncio.get_running_loop()
        alerted: set = set()
        while True:
            try:
                clusters = await loop.run_in_executor(None, scan_tool_failures)
                fresh = [c for c in clusters
                         if c.get("count_24h")
                         and (c["tool"], c["signature"]) not in alerted]
                if fresh:
                    alerted.update((c["tool"], c["signature"]) for c in fresh)
                    for c in fresh:
                        print(f"[toolfail] {c['tool']}/{c['kind']} x{c['count']} "
                              f"tasks={c['task_count']}: {c['signature'][:80]}")
                    if dev_email and smtp_configured():
                        await loop.run_in_executor(
                            None, send_email_smtp, dev_email,
                            f"[zcbot] 工具失败聚集 {len(fresh)} 类",
                            format_alert(fresh, 7),
                        )
                    else:
                        print("[toolfail] ZCBOT_DEVELOPER_EMAIL/SMTP 未配,仅日志")
                await asyncio.sleep(interval)
            except asyncio.CancelledError:
                raise
            except Exception as e:
                print(f"[toolfail] error: {type(e).__name__}: {e}")
                await asyncio.sleep(interval)

    return asyncio.create_task(_toolfail_scanner(), name="toolfail-scanner")


def init_sandbox(app, cfg: dict) -> Optional[asyncio.Task]:
    """Sandbox pool(§7.5):仅当 ZCBOT_SANDBOX_BACKEND=docker 时启用。

    启动钩子:① init_pool(创建 docker network + pool 实例)② shutdown_all 清
    前驱孤儿(上次进程留下的 zcbot-sandbox-* 容器,内存 _last_active 为空,
    全清重启)③ 后台 reaper task,每 60s 跑 reap_idle。返回 reaper task;
    host backend 返 None。init 失败 fail-fast(Stage C 协议:hardening 缺失
    视为部署未完成,不退化到 host —— 否则误以为有沙盒实则在裸跑)。
    """
    backend = os.getenv("ZCBOT_SANDBOX_BACKEND", "host").lower()
    if backend != "docker":
        return None
    from core.agent_builder import resolve_workspace
    from core.paths import ROOT
    from core.sandbox import init_pool
    from core.sandbox.check import detect_fs_quota
    user_root_base = resolve_workspace(None, cfg) / "users"
    # §7.5 #4 fs quota 探测:不阻塞启动(应用层周期扫描已有),仅打 WARN
    # 提醒外部用户开放前必须升级到 xfs prjquota / ext4 project / zfs。
    try:
        level, msg = detect_fs_quota(user_root_base.resolve())
        print(f"[startup] {'[ok]' if level == 'ok' else '[warn]'} {msg}")
    except Exception as e:
        print(f"[startup] [warn] fs quota detect failed: {type(e).__name__}: {e}")
    try:
        # repo_root=ROOT 让 SandboxPool 把 <repo>/skills 只读 mount 进容器
        # (fs 工具进容器后 read SKILL references 需要)
        # sandbox_cfg=yaml `sandbox` 段(memory/cpus/pids_limit 可调)
        pool = init_pool(
            user_root_base, repo_root=ROOT,
            sandbox_cfg=cfg.get("sandbox") or {},
        )
        removed = pool.shutdown_all()
        if removed:
            print(f"[startup] swept {len(removed)} stale sandbox container(s)")

        async def _reaper() -> None:
            loop = asyncio.get_running_loop()
            while True:
                try:
                    await asyncio.sleep(60)
                    removed = await loop.run_in_executor(None, pool.reap_idle)
                    if removed:
                        print(f"[reaper] reaped {len(removed)} idle sandbox container(s)")
                except asyncio.CancelledError:
                    raise
                except Exception as e:
                    print(f"[reaper] error: {type(e).__name__}: {e}")

        app.state.sandbox_pool = pool
        return asyncio.create_task(_reaper(), name="sandbox-reaper")
    except Exception as e:
        raise RuntimeError(
            f"ZCBOT_SANDBOX_BACKEND=docker but sandbox init failed: {e}"
        )


def start_proc_sweeper(cfg: dict) -> asyncio.Task:
    """bg proc 清扫(DESIGN §8.12):终态 proc 目录过 TTL 删除 + 已结束的
    zcbot-proc-* 容器回收。host / docker 两种 backend 都要跑(host 模式只做
    文件清扫,docker CLI 不在时 sweep 内部静默跳过容器部分)。每小时一次;
    幂等,蓝绿双实例同时跑无害。启动后先跑一轮,把上个进程周期留下的
    已结束容器/过期目录收掉。
    """
    from core.agent_builder import resolve_workspace
    from core.procs import sweep as _procs_sweep
    users_base = resolve_workspace(None, cfg) / "users"

    async def _proc_sweeper() -> None:
        loop = asyncio.get_running_loop()
        while True:
            try:
                stats = await loop.run_in_executor(
                    None, _procs_sweep, users_base
                )
                if stats["removed_dirs"] or stats["reaped_containers"]:
                    print(f"[proc-sweep] dirs={stats['removed_dirs']} "
                          f"containers={stats['reaped_containers']}")
            except asyncio.CancelledError:
                raise
            except Exception as e:
                print(f"[proc-sweep] error: {type(e).__name__}: {e}")
            await asyncio.sleep(3600)

    return asyncio.create_task(_proc_sweeper(), name="proc-sweeper")


async def drain_inflight(app, drain_timeout: int, cancel_grace: int) -> None:
    """优雅 drain:先拒新 run(draining 已置位),等在跑的 run 收尾,超时转协作式 cancel。

    单实例形态下消除"restart 误杀 in-flight run 标 error"。新 POST /messages
    期间返 503(客户端退避重试覆盖)。drain_timeout 内自然跑完 → idle 零 error;
    超时的 broker.request_cancel → 下个 chunk 间隙退(标 idle);cancel_grace 后仍
    没退的留给 systemd SIGKILL,下次启动 reaper 标 error(最坏退化 = 改前行为)。
    ★ systemd TimeoutStopSec 必须 > drain_timeout + cancel_grace + 余量(见 RUN.md)。
    """
    inflight = app.state.inflight
    if not inflight:
        return
    print(f"[shutdown] draining {len(inflight)} in-flight run(s), "
          f"timeout={drain_timeout}s")
    _, pending = await asyncio.wait(
        list(inflight.keys()), timeout=drain_timeout
    )
    if pending:
        print(f"[shutdown] {len(pending)} run(s) over drain timeout; "
              f"signalling cooperative cancel")
        for t in pending:
            cid = inflight.get(t)
            if cid is not None:
                broker.request_cancel(cid)
        _, still = await asyncio.wait(pending, timeout=cancel_grace)
        if still:
            print(f"[shutdown] {len(still)} run(s) still active after "
                  f"cancel grace; SIGKILL takes over, next start reaps them")
