"""后台进程(bg proc):detach + 文件系统状态,不引队列组件(DESIGN §8.12)。 解决的问题:模型写的长脚本(数据处理/模拟计算)在工具内同步跑会被超时掐死、 占死 run、蓝绿切换时随实例进程陪葬。方案是把进程从 zcbot 进程树上摘下来: - host 模式:detach 启动 `core/proc_wrapper.py`(stdlib-only),wrapper 负责跑 子进程、限时、写退出码 —— zcbot 重启不影响它 - docker 模式:每个 bg proc 一个专用容器 `zcbot-proc-`(executor_docker 侧 实现),dockerd 托管,与 sandbox 容器的 idle reaper / shutdown_all 生命周期解耦 目录布局(dotfile,/v1/files API 天然隐藏,用户文件浏览器不可见): /.zcbot_procs/// proc.json # 元数据(kind/command/backend/pid 或 container/timeout/created_at) output.log # stdout+stderr 合流(结束时截到最后 10MB) exit_code # 结束后由 wrapper(host)/runner.sh(docker)写入 —— 存在即已结束 script.py # run_python inline code 落盘(可选) cmd.sh # docker 模式的用户命令(可选,绕开引号转义) 状态判定不靠常驻登记,全靠文件 + 系统探测(list/check 时现算): exit_code 存在 → finished;否则 host 查 wrapper pid 活着 / docker 查容器 running → running;都探不到 → lost(宿主重启把进程带走了,产物文件仍在)。 边界(DESIGN §8.12,防滑坡):只覆盖「单个本地长进程」。job 链/依赖/重试策略 不进这里 —— 编排的唯一归属是 agent loop 本身(模型在 check 后自己决定下一步)。 """ from __future__ import annotations import json import os import re import secrets import shutil import subprocess import sys import threading import time from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Tuple PROCS_SUBDIR = ".zcbot_procs" # bg 进程默认/最大运行时长(秒);前台 60/120s 的默认值不放大 —— 它是逼模型 # 在"快命令"与"长任务"之间做选择的杠杆 DEFAULT_TIMEOUT_S = 7200 MAX_TIMEOUT_S = 86400 # 每用户并发 bg 进程上限(跨 task 合计);防模型失控起一堆 MAX_RUNNING_PER_USER = int(os.getenv("ZCBOT_MAX_BG_PROCS", "3")) # 终态 proc 目录保留时长,sweep 超期删除 FINISHED_TTL_S = 7 * 86400 _ID_RE = re.compile(r"^[A-Za-z0-9_-]+$") def new_proc_id() -> str: return "p" + secrets.token_hex(4) def procs_root(anchor: Path) -> Path: """anchor = user_root(常规)或 base_dir(CLI --working-dir 无 user_root 兜底)。""" return Path(anchor) / PROCS_SUBDIR def proc_dir(anchor: Path, task_id: str, proc_id: str) -> Optional[Path]: """proc_id 来自模型输入,先校验再拼路径(防 ../ 越界)。非法返 None。""" if not (_ID_RE.match(str(task_id) or "") and _ID_RE.match(proc_id or "")): return None return procs_root(anchor) / str(task_id) / proc_id def write_meta(d: Path, meta: Dict[str, Any]) -> None: """原子写(tmp + replace):wrapper 与 check_process 并发读写不撕裂。""" tmp = d / "proc.json.tmp" tmp.write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") os.replace(tmp, d / "proc.json") def read_meta(d: Path) -> Optional[Dict[str, Any]]: try: return json.loads((d / "proc.json").read_text(encoding="utf-8")) except (OSError, ValueError): return None def clamp_timeout(timeout: Optional[int]) -> int: try: t = int(timeout) if timeout else DEFAULT_TIMEOUT_S except (TypeError, ValueError): t = DEFAULT_TIMEOUT_S return max(60, min(t, MAX_TIMEOUT_S)) def count_running(anchor: Path) -> int: """该用户(anchor=user_root)当前 running 状态的 bg 进程数,跨 task 合计。""" root = procs_root(anchor) if not root.is_dir(): return 0 n = 0 for tdir in root.iterdir(): if not tdir.is_dir(): continue for d in tdir.iterdir(): meta = read_meta(d) if d.is_dir() else None if meta and status_of(meta, d)[0] == "running": n += 1 return n # ───────────── 状态探测 ───────────── def _pid_alive(pid: int) -> bool: if pid <= 0: return False if os.name == "posix": try: os.kill(pid, 0) return True except ProcessLookupError: return False except PermissionError: return True except OSError: return False # Windows:OpenProcess 探测(PID 复用误报概率低,可接受) import ctypes PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 h = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid) if not h: return False try: code = ctypes.c_ulong() ok = ctypes.windll.kernel32.GetExitCodeProcess(h, ctypes.byref(code)) STILL_ACTIVE = 259 return bool(ok) and code.value == STILL_ACTIVE finally: ctypes.windll.kernel32.CloseHandle(h) def _container_running(name: str) -> bool: try: r = subprocess.run( ["docker", "inspect", "--type=container", "--format={{.State.Running}}", name], capture_output=True, text=True, timeout=15, ) return r.returncode == 0 and r.stdout.strip() == "true" except (OSError, subprocess.TimeoutExpired): return False def status_of(meta: Dict[str, Any], d: Path) -> Tuple[str, Optional[int]]: """→ ("running"|"finished"|"lost", exit_code|None)。exit_code 文件是唯一终态锚。""" try: raw = (d / "exit_code").read_text(encoding="utf-8").strip() return "finished", int(raw) except (OSError, ValueError): pass if meta.get("backend") == "docker": if _container_running(str(meta.get("container") or "")): return "running", None return "lost", None pid = int(meta.get("wrapper_pid") or 0) if _pid_alive(pid): return "running", None return "lost", None def tail_log(d: Path, max_bytes: int = 8_000) -> str: p = d / "output.log" try: size = p.stat().st_size with open(p, "rb") as f: if size > max_bytes: f.seek(size - max_bytes) data = f.read() text = data.decode("utf-8", errors="replace") if size > max_bytes: text = f"[...前 {size - max_bytes} 字节省略...]\n" + text return text except OSError: return "(暂无输出)" def list_procs(anchor: Path, task_id: str) -> List[Dict[str, Any]]: """该 task 的所有 bg proc,带现算 status/exit_code,按创建时间排。""" tdir = procs_root(anchor) / str(task_id) if not tdir.is_dir(): return [] out = [] for d in sorted(tdir.iterdir()): if not d.is_dir(): continue meta = read_meta(d) if not meta: continue st, ec = status_of(meta, d) meta["_status"], meta["_exit_code"], meta["_dir"] = st, ec, d out.append(meta) out.sort(key=lambda m: m.get("created_ts") or 0) return out def list_all_procs(anchor: Path) -> List[Dict[str, Any]]: """该用户(anchor=user_root)全部 task 的 bg proc(web `/v1/procs` 用:跨 task 通知——用户切到别的 task 也能收到完成提示)。""" root = procs_root(anchor) if not root.is_dir(): return [] out: List[Dict[str, Any]] = [] for tdir in sorted(root.iterdir()): if tdir.is_dir(): out.extend(list_procs(anchor, tdir.name)) return out def finished_at(d: Path) -> Optional[float]: """终态时刻 = exit_code 文件 mtime(结束时才写,天然就是完成时间戳)。""" try: return (d / "exit_code").stat().st_mtime except OSError: return None # ───────────── host 模式启动 / 终止 ───────────── def launch_host( anchor: Path, task_id: str, *, kind: str, # "shell" | "python" command_display: str, # 给 LLM/用户看的命令描述 argv: Optional[List[str]] = None, # 二选一:argv(python) shell_cmd: Optional[str] = None, # 或 shell 命令串 cwd: Path, timeout_s: int, env: Optional[Dict[str, str]] = None, inline_code: Optional[str] = None, # run_python inline → 落 script.py,argv 引用它 ) -> Tuple[str, Path]: """detach 启动 wrapper,立即返回 (proc_id, proc_dir)。 wrapper 是独立脚本(core/proc_wrapper.py,stdlib-only),用当前解释器直跑 (非 -m,不依赖 PYTHONPATH),zcbot 进程退出/重启不影响它。 """ proc_id = new_proc_id() d = procs_root(anchor) / str(task_id) / proc_id d.mkdir(parents=True, exist_ok=True) if inline_code is not None: script = d / "script.py" script.write_text(inline_code, encoding="utf-8") argv = [sys.executable, str(script)] meta: Dict[str, Any] = { "proc_id": proc_id, "task_id": str(task_id), "kind": kind, "backend": "host", "command": command_display[:500], "cwd": str(cwd), "timeout_s": timeout_s, "created_at": datetime.now().isoformat(timespec="seconds"), "created_ts": time.time(), } if argv: meta["argv"] = argv else: meta["shell_cmd"] = shell_cmd write_meta(d, meta) wrapper = Path(__file__).parent / "proc_wrapper.py" spawn_kw: Dict[str, Any] = dict( stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env, ) if os.name == "posix": spawn_kw["start_new_session"] = True else: spawn_kw["creationflags"] = ( subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW ) p = subprocess.Popen([sys.executable, str(wrapper), str(d)], **spawn_kw) meta["wrapper_pid"] = p.pid write_meta(d, meta) # 后台线程 wait 掉 wrapper,避免 POSIX 僵尸挂在 web 进程下 threading.Thread(target=p.wait, daemon=True).start() return proc_id, d def kill_proc(meta: Dict[str, Any], d: Path) -> str: """强杀。docker → rm -f 容器;host → taskkill /T(win)/ killpg(posix)。 杀完补写 exit_code=137,让状态收敛到 finished(killed)。""" st, _ = status_of(meta, d) if st == "finished": return "进程已结束,无需终止" if meta.get("backend") == "docker": name = str(meta.get("container") or "") if name: subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) else: wrapper_pid = int(meta.get("wrapper_pid") or 0) child_pid = int(meta.get("child_pid") or 0) if os.name == "posix": import signal for pid in (child_pid, wrapper_pid): if pid > 0: try: os.killpg(pid, signal.SIGKILL) except (ProcessLookupError, PermissionError, OSError): try: os.kill(pid, signal.SIGKILL) except OSError: pass else: for pid in (wrapper_pid, child_pid): if pid > 0: subprocess.run( ["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True, timeout=30, ) try: (d / "exit_code").write_text("137", encoding="utf-8") except OSError: pass meta["killed"] = True write_meta(d, meta) return "已终止" # ───────────── 清扫(web lifespan 周期调用) ───────────── def sweep(user_root_base: Path, ttl_s: int = FINISHED_TTL_S) -> Dict[str, int]: """两件事:① 终态超 TTL 的 proc 目录删掉 ② 已结束(exit_code 在)但容器还挂着 的 docker proc 容器 rm -f(runner 写完退出码后容器空转 sleep,等这里回收)。 另兜底:label=zcbot.product=proc 的孤儿容器(目录已删/exited)一并清。 幂等,蓝绿双实例同时跑无害(rm -f / rmtree 都幂等)。""" removed_dirs = reaped_containers = 0 now = time.time() base = Path(user_root_base) if base.is_dir(): for uroot in base.iterdir(): # 单条目异常(权限/竞态删除)不打断整轮清扫 try: root = uroot / PROCS_SUBDIR if not root.is_dir(): continue for tdir in list(root.iterdir()): if not tdir.is_dir(): continue for d in list(tdir.iterdir()): if not d.is_dir(): continue meta = read_meta(d) if not meta: continue st, _ = status_of(meta, d) if st != "running" and meta.get("backend") == "docker": name = str(meta.get("container") or "") if name and _container_exists(name): subprocess.run( ["docker", "rm", "-f", name], capture_output=True, timeout=30, ) reaped_containers += 1 age = now - float(meta.get("created_ts") or now) if st != "running" and age > ttl_s: shutil.rmtree(d, ignore_errors=True) removed_dirs += 1 try: tdir.rmdir() # 空了顺手收掉,非空抛 OSError 忽略 except OSError: pass except OSError: continue # 孤儿容器兜底(proc 目录已被删,或容器已 exited) try: r = subprocess.run( ["docker", "ps", "-aq", "--filter", "label=zcbot.product=proc", "--filter", "status=exited"], capture_output=True, text=True, timeout=30, ) ids = r.stdout.split() if r.returncode == 0 else [] if ids: subprocess.run(["docker", "rm", "-f", *ids], capture_output=True, timeout=60) reaped_containers += len(ids) except (OSError, subprocess.TimeoutExpired): pass # docker 不在(host 模式部署)→ 只做文件清扫 return {"removed_dirs": removed_dirs, "reaped_containers": reaped_containers} def _container_exists(name: str) -> bool: try: r = subprocess.run( ["docker", "inspect", "--type=container", name], capture_output=True, timeout=15, ) return r.returncode == 0 except (OSError, subprocess.TimeoutExpired): return False # ───────────── LLM 面向的描述格式化(check_process / 启动返回共用) ───────────── def format_proc_line(meta: Dict[str, Any]) -> str: st = meta.get("_status", "?") ec = meta.get("_exit_code") elapsed = "" ts = meta.get("created_ts") if ts: elapsed = f" · 已运行 {_fmt_elapsed(time.time() - float(ts))}" if st == "running" else "" tail = f"(exit {ec})" if st == "finished" else "" return ( f"- {meta.get('proc_id')} [{st}{tail}] {meta.get('kind')}: " f"{meta.get('command', '')[:120]}{elapsed}" ) def _fmt_elapsed(s: float) -> str: s = int(s) if s < 60: return f"{s}s" if s < 3600: return f"{s // 60}m{s % 60:02d}s" return f"{s // 3600}h{(s % 3600) // 60:02d}m"