206 lines
10 KiB
Bash
Executable File
206 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# zcbot 蓝绿无感部署(RUN.md §无感更新 B 档)。
|
|
#
|
|
# 拓扑:zcbot@blue(127.0.0.1:8766)/ zcbot@green(127.0.0.1:8767)双 systemd 实例,
|
|
# nginx 对外监听原单实例端口 8765(客户端 / platform 嵌入无感),upstream
|
|
# (/etc/nginx/zcbot_upstream.conf,单行 server 指令)在两实例间切流量。
|
|
#
|
|
# 干什么(顺序写死):
|
|
# 1) update.sh --prepare-only 拉码 / pip / db migration / docker build(复用单实例脚本)
|
|
# 2) systemctl restart zcbot@<空闲色> 空闲实例起新代码
|
|
# 3) curl 空闲实例 /healthz 验活 + 核对 instance 字段
|
|
# 4) 改写 nginx upstream → reload 新 REST/SSE 全走新实例;旧 SSE 留旧实例不断
|
|
# 5) 等旧实例在跑 run 自然清空 轮询 DB run_owner=旧色 的活跃行,归零才停
|
|
# (上限 DRAIN_WAIT_SECONDS 默 600;期间旧 SSE
|
|
# 继续直播不断流 —— 这步是"无感"的关键)
|
|
# 6) systemctl stop zcbot@<旧live色> 此时通常已无在跑 run,秒停;超上限的残留 run
|
|
# 走 SIGTERM drain → 协作式 cancel(标 idle 可重发)
|
|
# 7) DB 收尾 旧色 SIGKILL 兜底残留的 running 行标 error
|
|
# 8) enable 新 / disable 旧 重启机器只拉起当前 live 色
|
|
#
|
|
# 效果:部署全程无 503 窗口(对比单实例 update.sh 的几十秒 drain 窗口),在跑对话的
|
|
# SSE 直播也不断(对比直接 stop:uvicorn timeout_graceful_shutdown=5 会 5s 掐长连接)。
|
|
#
|
|
# 怎么跑:sudo bash /opt/zcbot/deploy/update_bluegreen.sh
|
|
# --skip-build 透传给 update.sh(跳过 sandbox 镜像重建)
|
|
# --switch blue|green 只切 nginx 流量到指定色(不拉码不重启;半途恢复 / 手动调流用)
|
|
#
|
|
# 一次性 bootstrap(装模板 unit / nginx 配置 / 停旧单实例服务)见 RUN.md §蓝绿部署。
|
|
# 回滚注意:代码目录两实例共享,旧实例停了再起跑的也是新代码 —— 回滚 = git revert 后
|
|
# 重跑本脚本,不是把流量切回旧色。
|
|
|
|
set -euo pipefail
|
|
|
|
ORIG_ARGS=("$@")
|
|
SKIP_BUILD_ARG=()
|
|
SWITCH_TO=""
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--skip-build) SKIP_BUILD_ARG=(--skip-build); shift ;;
|
|
--switch) SWITCH_TO="${2:-}"; shift 2 ;;
|
|
-h|--help) echo "用法:sudo $0 [--skip-build] [--switch blue|green]"; exit 0 ;;
|
|
*) echo "[deploy] ERROR: 未知参数:$1" >&2; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
|
|
APP_DIR="${APP_DIR:-$(dirname "$SCRIPT_DIR")}"
|
|
APP_USER="${APP_USER:-$(stat -c %U "$APP_DIR")}"
|
|
PY="$APP_DIR/.venv/bin/python"
|
|
UPSTREAM_FILE="${UPSTREAM_FILE:-/etc/nginx/zcbot_upstream.conf}"
|
|
# 切流量后等旧实例在跑 run 自然收尾的上限(超了就 stop,残留 run 走协作式 cancel)
|
|
DRAIN_WAIT_SECONDS="${DRAIN_WAIT_SECONDS:-600}"
|
|
|
|
declare -A PORT=([blue]=8766 [green]=8767)
|
|
|
|
log() { echo "[deploy] $*"; }
|
|
fail() { echo "[deploy] ERROR: $*" >&2; exit 1; }
|
|
asuser() { sudo -u "$APP_USER" "$@"; }
|
|
|
|
# ── 0) 前置检查 ────────────────────────────────────────────────
|
|
[ "$(id -u)" -eq 0 ] || fail "需 root(systemctl / 写 nginx 配置),请用:sudo $0"
|
|
[ -d "$APP_DIR/.git" ] || fail "$APP_DIR 不是 git 仓库"
|
|
[ -x "$PY" ] || fail "找不到 venv python:$PY"
|
|
[ -f "$APP_DIR/.env" ] || fail "缺 $APP_DIR/.env"
|
|
command -v nginx >/dev/null || fail "nginx 未安装(蓝绿依赖 nginx 反代,见 RUN.md §蓝绿部署 bootstrap)"
|
|
[ -f "$UPSTREAM_FILE" ] || fail "缺 $UPSTREAM_FILE(nginx upstream 未初始化,见 RUN.md §蓝绿部署 bootstrap)"
|
|
systemctl cat zcbot@blue >/dev/null 2>&1 || fail "模板 unit 未安装(sudo cp deploy/zcbot@.service /etc/systemd/system/ && daemon-reload)"
|
|
if systemctl is-active --quiet zcbot 2>/dev/null; then
|
|
fail "旧单实例服务 zcbot.service 还在跑(占 8765,与 nginx 对外监听冲突)—— 先走 RUN.md §蓝绿部署 bootstrap 停掉它"
|
|
fi
|
|
DB_URL="$(grep -E '^[[:space:]]*ZCBOT_DB_URL=' "$APP_DIR/.env" | tail -n1 | cut -d= -f2-)"
|
|
[ -n "$DB_URL" ] || fail ".env 里没读到 ZCBOT_DB_URL"
|
|
|
|
# ── 当前 live 色:从 nginx upstream 文件反推(单一事实源,不另立状态文件)──
|
|
detect_live() {
|
|
if grep -q ":${PORT[blue]}" "$UPSTREAM_FILE"; then echo blue
|
|
elif grep -q ":${PORT[green]}" "$UPSTREAM_FILE"; then echo green
|
|
else fail "$UPSTREAM_FILE 内容不认识(应含 127.0.0.1:${PORT[blue]} 或 :${PORT[green]}):$(cat "$UPSTREAM_FILE")"
|
|
fi
|
|
}
|
|
|
|
wait_health() { # $1=color;实例 /healthz 返 ok 且 instance 字段对得上
|
|
local color="$1" url="http://127.0.0.1:${PORT[$color]}/healthz" body=""
|
|
for _ in $(seq 1 30); do
|
|
body="$(curl -fsS --max-time 2 "$url" 2>/dev/null || true)"
|
|
if echo "$body" | grep -q '"ok"'; then
|
|
echo "$body" | grep -q "\"instance\":\"$color\"" \
|
|
|| fail "$url 返回的 instance 不是 $color(实例 env 没生效?):$body"
|
|
log "zcbot@$color 健康:$body"
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "[deploy] ----- journalctl -u zcbot@$color -n 50 -----" >&2
|
|
journalctl -u "zcbot@$color" -n 50 --no-pager >&2 || true
|
|
fail "$url 30s 内没返回 ok(看上面日志)"
|
|
}
|
|
|
|
switch_nginx() { # $1=color;改 upstream + 校验 + reload,校验失败回滚原文件
|
|
local color="$1" backup
|
|
backup="$(cat "$UPSTREAM_FILE")"
|
|
echo "server 127.0.0.1:${PORT[$color]};" > "$UPSTREAM_FILE"
|
|
if ! nginx -t 2>&1; then
|
|
echo "$backup" > "$UPSTREAM_FILE"
|
|
fail "nginx -t 校验失败,upstream 已回滚"
|
|
fi
|
|
systemctl reload nginx 2>/dev/null || nginx -s reload
|
|
log "nginx upstream → zcbot@$color(127.0.0.1:${PORT[$color]})"
|
|
}
|
|
|
|
# ── --switch:只调流量,不部署 ─────────────────────────────────
|
|
if [ -n "$SWITCH_TO" ]; then
|
|
[[ "$SWITCH_TO" == "blue" || "$SWITCH_TO" == "green" ]] || fail "--switch 只接受 blue|green"
|
|
systemctl is-active --quiet "zcbot@$SWITCH_TO" || fail "zcbot@$SWITCH_TO 没在跑,先 systemctl start"
|
|
wait_health "$SWITCH_TO"
|
|
switch_nginx "$SWITCH_TO"
|
|
log "OK:流量已切到 zcbot@$SWITCH_TO(另一实例未动,自行决定 stop/disable)"
|
|
exit 0
|
|
fi
|
|
|
|
LIVE="$(detect_live)"
|
|
if [ "$LIVE" = "blue" ]; then IDLE="green"; else IDLE="blue"; fi
|
|
log "当前 live=$LIVE,本次部署到 idle=$IDLE"
|
|
|
|
# ── 1) 准备:拉码 / 依赖 / migration / sandbox 镜像(复用 update.sh)──
|
|
# migration 先于切换跑 —— 项目纪律要求 schema 改动平滑兼容存量代码,旧实例(旧代码)
|
|
# 在新 schema 上继续 drain 是安全的。
|
|
OLD_HEAD="$(asuser git -C "$APP_DIR" rev-parse HEAD)"
|
|
bash "$APP_DIR/deploy/update.sh" --prepare-only "${SKIP_BUILD_ARG[@]}"
|
|
NEW_HEAD="$(asuser git -C "$APP_DIR" rev-parse HEAD)"
|
|
|
|
# 本脚本自身被 pull 更新 → 用新版本重跑(同 update.sh 的自更新逻辑;里面的
|
|
# prepare 已幂等,重跑秒过)。ZCBOT_UPDATE_BG_REEXEC 防死循环。
|
|
if [ "$OLD_HEAD" != "$NEW_HEAD" ] && [ "${ZCBOT_UPDATE_BG_REEXEC:-0}" != "1" ] \
|
|
&& ! asuser git -C "$APP_DIR" diff --quiet "$OLD_HEAD" "$NEW_HEAD" -- deploy/update_bluegreen.sh; then
|
|
log "update_bluegreen.sh 自身有更新 —— 用新版本重跑..."
|
|
exec env ZCBOT_UPDATE_BG_REEXEC=1 bash "$APP_DIR/deploy/update_bluegreen.sh" "${ORIG_ARGS[@]}"
|
|
fi
|
|
|
|
# ── 2) 空闲实例起新代码 ────────────────────────────────────────
|
|
log "systemctl restart zcbot@$IDLE ..."
|
|
systemctl restart "zcbot@$IDLE"
|
|
|
|
# ── 3) 验活(healthz + instance 字段)──────────────────────────
|
|
wait_health "$IDLE"
|
|
|
|
# ── 4) nginx 切流量:新请求全走 $IDLE,旧 SSE 留 $LIVE 上不断 ──
|
|
switch_nginx "$IDLE"
|
|
|
|
# ── 5) 等旧实例在跑的 run 自然清空(SSE 直播不断的关键)──────────
|
|
# 新流量已全走新实例;旧实例上的 run 继续跑、SSE 继续推(nginx reload 不影响已建立
|
|
# 连接)。直接 stop 的话 uvicorn timeout_graceful_shutdown=5 会 5s 掐掉长连接 ——
|
|
# run 能跑完但用户看到断流,所以先等 DB 里 run_owner=旧色 的活跃行归零再停。
|
|
# 超 DRAIN_WAIT_SECONDS 的转 stop 的 SIGTERM drain(协作式 cancel,标 idle 可重发)。
|
|
count_live_runs() {
|
|
asuser env ZCBOT_DB_URL="$DB_URL" "$PY" - "$LIVE" <<'PYEOF'
|
|
import os, sys
|
|
from sqlalchemy import create_engine, text
|
|
eng = create_engine(os.environ["ZCBOT_DB_URL"])
|
|
with eng.connect() as c:
|
|
print(c.execute(text(
|
|
"SELECT count(*) FROM tasks "
|
|
"WHERE run_status IN ('running','cancelling') AND run_owner=:o"
|
|
), {"o": sys.argv[1]}).scalar())
|
|
PYEOF
|
|
}
|
|
log "等 zcbot@$LIVE 在跑的 run 自然收尾(上限 ${DRAIN_WAIT_SECONDS}s)..."
|
|
waited=0
|
|
while [ "$waited" -lt "$DRAIN_WAIT_SECONDS" ]; do
|
|
n="$(count_live_runs)"
|
|
if [ "$n" = "0" ]; then
|
|
log "zcbot@$LIVE 已无在跑 run"
|
|
break
|
|
fi
|
|
log " $LIVE 还有 $n 个 run 在跑,${waited}s/${DRAIN_WAIT_SECONDS}s ..."
|
|
sleep 10
|
|
waited=$((waited + 10))
|
|
done
|
|
|
|
# ── 6) 停旧实例(通常已空,秒停;残留的走 SIGTERM drain)────────
|
|
log "systemctl stop zcbot@$LIVE ..."
|
|
systemctl stop "zcbot@$LIVE"
|
|
|
|
# ── 7) DB 收尾:旧色被 SIGKILL 兜底砍掉的 run(没来得及收尾的)标 error ──
|
|
# 正常路径这里是 0 行;非 0 说明有 run 超上限被硬停,对应 task 重新可发消息。
|
|
asuser env ZCBOT_DB_URL="$DB_URL" "$PY" - "$LIVE" <<'PYEOF'
|
|
import os, sys
|
|
from sqlalchemy import create_engine, text
|
|
color = sys.argv[1]
|
|
eng = create_engine(os.environ["ZCBOT_DB_URL"])
|
|
with eng.begin() as c:
|
|
r = c.execute(text(
|
|
"UPDATE tasks SET run_status='error', "
|
|
"run_error='instance stopped before run finished' "
|
|
"WHERE run_status IN ('running','cancelling') AND run_owner=:o"
|
|
), {"o": color})
|
|
print(f"[deploy] reaped {r.rowcount} leftover run(s) owned by {color}")
|
|
PYEOF
|
|
|
|
# ── 8) 开机自启只留新 live 色 ──────────────────────────────────
|
|
systemctl enable "zcbot@$IDLE" >/dev/null 2>&1 || true
|
|
systemctl disable "zcbot@$LIVE" >/dev/null 2>&1 || true
|
|
|
|
log "OK:蓝绿切换完成($NEW_HEAD),live=$IDLE,旧实例 $LIVE 已停"
|
|
log "看实时日志:journalctl -u zcbot@$IDLE -f"
|