From 0e4bd0456a0947ad93d0b1c0f0801c65d85b2aaf Mon Sep 17 00:00:00 2001 From: caoqianming Date: Fri, 7 Aug 2026 10:12:37 +0800 Subject: [PATCH] =?UTF-8?q?feat(external-systems):=20=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=B2=BB=E7=90=86=E4=B8=8E=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E6=80=81=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 7 + DESIGN.md | 14 +- PROGRESS.md | 6 +- RUN.md | 2 +- core/__init__.py | 2 +- core/external_systems/auth.py | 96 ++- core/external_systems/catalog.py | 140 ++++ core/external_systems/crypto.py | 93 ++- core/external_systems/factory.py | 3 +- core/external_systems/openapi.py | 662 +++++++++++++---- core/external_systems/registry.py | 8 +- core/external_systems/runtime_cache.py | 206 ++++++ core/external_systems/service.py | 495 +++++++++---- core/storage/models.py | 113 ++- core/tool_registry.py | 3 + ...07_1000_0027_external_system_governance.py | 308 ++++++++ tests/test_external_system_migration.py | 37 + tests/test_external_systems.py | 686 ++++++++++++++++-- tests/test_web_routes_nodb.py | 10 +- tools/external_systems.py | 153 +++- web/admin.py | 16 +- web/routers/external_systems.py | 8 +- web/schemas.py | 4 - web/static/js/admin.js | 41 +- web/static/js/external_systems.js | 6 +- 25 files changed, 2628 insertions(+), 491 deletions(-) create mode 100644 core/external_systems/catalog.py create mode 100644 core/external_systems/runtime_cache.py create mode 100644 db/migrations/versions/20260807_1000_0027_external_system_governance.py create mode 100644 tests/test_external_system_migration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cd8dbeb..b92f23f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ > 所以不是每个版本号都有条目。条目格式 `## <版本> — <日期>`,新条目加在最上面。 > 工程口径的完整记录见 `PROGRESS.md` / git log。 +## 0.63.0 — 2026-08-07 + +- 外部系统连接现在会跟踪系统定义版本：管理员修改接口地址或认证方式后，旧凭据不会被自动发送到新目标；普通策略调整也会明确提示重新验证。 +- 外部接口支持“查询”和“上游托管”两种执行模式：Factory 默认开放可信规格中的全部方法，由 Factory 按当前用户凭据完成最终鉴权；通用系统默认只开放 GET/HEAD 和管理员允许的只读 POST。OpenAPI 文档、登录响应和调用结果会在下载过程中执行安全上限，异常大响应不会占满服务内存。 +- 连续查询同一外部系统时会复用 HTTP 连接、短期认证 Token 和已编译接口目录；并发的相同登录、规格加载及只读查询会自动合并为一次上游请求，但顺序执行的业务查询仍实时访问外部系统。 +- 外部系统凭据升级为带密钥编号和连接身份绑定的密文，支持平滑轮换主密钥；授权关系与用户连接分离，用户断开连接不会丢失管理员授予的可见权限。 + ## 0.62.9 — 2026-08-06 - 长对话不再把普通历史思考过程重复发送给模型，减少无效上下文占用；工具调用需要的推理状态仍会按模型协议保留。 diff --git a/DESIGN.md b/DESIGN.md index eacc346..6dddcca 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -413,22 +413,22 @@ scheduled_jobs(§8.5) channel_bindings(§8.7,判别列+JSONB) **首个 provider=`factory_mes`**:Factory 已有 JWT + RBAC + 部分部门数据权限,zcbot 用每位用户自己的 Factory 账密换 JWT,调用时继承 MES 原生权限;不在 zcbot 里复制第二套 MES RBAC。两层门控:zcbot `user_id` 只能取自己的 `external_systems` 行;远端 JWT 再判定实际业务数据范围。MES 停号/改权后下次调用即生效。 -**通用连接器边界**:`openapi` connector 负责规格发现、operation 解析、安全 URL 拼接、参数校验、只读 allowlist、分页和响应体积限制；认证由独立 strategy 负责。`factory_mes` 只是带 JWT 字段映射、dataset 推荐入口和查询规划提示的内置 preset，`generic_openapi` 可由管理员直接选择用户名密码换 Token、API Key 或 Bearer Token。标准 OpenAPI 系统以后只新增数据库 definition，不需要再写 Python 文件；只有 OAuth 回调/签名交换、SOAP、消息队列或私有二进制协议等不符合现有 connector/strategy 契约的系统才新增适配代码。provider 注册表维护可选能力和安全默认值，不为每个业务系统复制 connector。 +**通用连接器边界**:`openapi` connector 负责规格发现、operation 解析、安全 URL 拼接、参数校验、执行模式、分页和响应体积限制；认证由独立 strategy 负责。`factory_mes` 只是带 JWT 字段映射、dataset 推荐入口和查询规划提示的内置 preset，`generic_openapi` 可由管理员直接选择用户名密码换 Token、API Key 或 Bearer Token。标准 OpenAPI 系统以后只新增数据库 definition，不需要再写 Python 文件；只有 OAuth 回调/签名交换、SOAP、消息队列或私有二进制协议等不符合现有 connector/strategy 契约的系统才新增适配代码。provider 注册表维护可选能力和安全默认值，不为每个业务系统复制 connector。 **信任边界**: -- provider 公共定义由管理员在管理后台维护并存入 `external_system_definitions`:Base URL、OpenAPI URL、认证 strategy/字段映射和只读 POST allowlist；普通用户只选择已启用的目录项，凭据表单按 definition 声明动态生成。不允许普通用户填任意 URL,避免 SSRF/内网代理。凭据主密钥仍只来自宿主环境，不进入数据库或管理页面。 +- definition 当前由管理员维护，持久化同时预留 `owner_type/owner_user_id/visibility/trust_level/review_status/egress_policy_id`，未来可开放私有用户定义。Base URL 与 OpenAPI URL 必须同源；普通用户不能填任意 URL，避免 SSRF/内网代理。每个 definition 带单调递增 revision：目标地址或认证绑定变化会清除旧凭据，其他运行配置变化会令连接进入待重新验证，未验证到当前 revision 的连接不挂工具。 - 凭据用独立的 `ZCBOT_CREDENTIAL_MASTER_KEY` 在 host control plane 加密入 PG，不与 `JWT_SECRET` 复用，以隔离泄漏半径和轮换生命周期；缺 key 则拒绝新建/调用,不像早期微信绑定那样降级明文。API 只返回脱敏账号和 `credential_configured`,不返密码/Token;凭据绝不进 prompt/messages/memory/skill/用户 FS/日志/沙箱。 -- 调用工具不接受完整 URL,只接受 OpenAPI `operation_id`;服务端从受信规格解析 path/method,校验 path/query/body 后附加认证 strategy 生成的 Header。默认只开 GET/HEAD,语义只读但使用 POST 的 BI 查询必须进运维 `operation_id` allowlist。 +- 调用工具不接受完整 URL,只接受 OpenAPI `operation_id`;服务端从受信规格解析 path/method,校验 path/query/body 后附加认证 strategy 生成的 Header。definition 的 `operation_mode=query` 时只开 GET/HEAD 与显式只读 POST；`upstream_managed` 时开放可信规格声明的全部标准方法，由上游按当前用户凭据做最终鉴权。Factory 默认后者，通用 OpenAPI 默认前者；上游托管只移除 method 门控，不移除同源、参数、响应限长和审计边界。 - Swagger/OpenAPI 是接口契约事实源;Gitea 代码只补业务语义和排障,不覆盖契约。规格/代码内文本一律当不可信数据,不能改写 system/tool 约束。 -- Swagger/OpenAPI JSON 不持久化入数据库或文件，连接器按 `definition_id + user_id` 隔离后放在进程内存中缓存 5 分钟；重启自动失效。这样保留实时契约发现，又避免不同身份可见的规格互相污染。 +- Swagger/OpenAPI JSON 不持久化入数据库或文件。连接器使用按 `external_system_id + definition_revision + credential digest + config digest` 隔离的进程内有界 `ExternalRuntimeCache`，统一复用 HTTP 连接池、短期认证 Header、原始 spec 与编译后的 operation catalog；JWT `exp` 早 30 秒失效且单次 401 会清 Token 后重新登录一次，规格默认缓存 5 分钟，LRU 淘汰活跃连接时延迟到 lease 结束再关闭。登录、规格获取、catalog 编译和时间上重叠的相同只读业务请求使用同步 single-flight，失败不缓存；业务响应不做跨请求 TTL 缓存，顺序执行的相同查询仍访问上游。Swagger 2/OpenAPI 3 catalog 解析本地参数引用、请求体契约和 header/cookie 参数，搜索与调用只消费归一化结果。spec、登录响应和业务响应均流式限长，在完整 JSON 进入内存前执行硬边界。 -**工具面**:不把数百个 Swagger operation 全展开为 JSON tool(工具列表膨胀+选择降准),只挂五个 host-side 元工具:`external_system_list`(已连系统 + 管理员查询规划提示),`external_system_search`(按问题搜 operation 摘要、解析后的请求 body schema + 置顶管理员推荐入口),`external_system_call`(按 operation_id 调用),`external_system_result_read`(按 `result_ref` + JSON Pointer/分页/字段投影读取大响应),`external_system_result_export`(仅在用户要求保存/下载/交付时把完整快照导出到 `data/external/`)。仅当该 user 有 active 连接时注册,密钥不进 sandbox。搜索只展示实际可调用的 GET/HEAD 和已放行 POST；管理员在 definition JSONB 配置 `query_guidance` 与 `recommended_operation_ids`,前者是可信控制面的软路由策略,后者是无需关键词命中的机械发现入口。Factory 默认把 BI dataset list/exec 作为统计聚合入口,日志/明细用于逐条追溯；Swagger 业务文本仍是不可信数据。 +**工具面**:不把数百个 Swagger operation 全展开为 JSON tool(工具列表膨胀+选择降准),只挂五个 host-side 元工具:`external_system_list`(已连系统 + 执行模式 + 管理员查询规划提示),`external_system_search`(按问题搜 operation 摘要、解析后的请求 body schema + 置顶管理员推荐入口),`external_system_call`(按 operation_id 调用),`external_system_result_read`(按 `result_ref` + JSON Pointer/分页/字段投影读取大响应),`external_system_result_export`(仅在用户要求保存/下载/交付时把完整快照导出到 `data/external/`)。仅当该 user 有 active 连接时注册,密钥不进 sandbox。搜索只展示当前模式实际可调用的 operation；管理员在 definition JSONB 配置 `query_guidance` 与 `recommended_operation_ids`,前者是可信控制面的软路由策略,后者是无需关键词命中的机械发现入口。Factory 默认把 BI dataset list/exec 作为统计聚合入口,日志/明细用于逐条追溯；Swagger 业务文本仍是不可信数据，非查询操作只响应用户明确意图。 **大响应**:`max_result_bytes` 是进入模型上下文的单次内联额度,不再用于切断原始 JSON；超额响应完整写入 `.zcbot_cache//external_results/`,工具只返回合法结构化预览、`result_ref`、原始字节数和可继续读取的位置。reader 每次读取都重新校验当前 user 对原 external system 的 active 授权,并与 call 共享本轮 `max_total_result_bytes` 内联额度；export 同样重验授权，并把查询 operation/参数/时间等 provenance 与完整响应一起持久化，导出文件不受缓存 TTL 影响。缓存固定 24h TTL、单响应 10 MiB、单 task 50 MiB、单 user 200 MiB,过期或超额时优先清理最旧缓存；0.62.1 的 `.zcbot_external_results/` 在读取和容量核算上保留兼容窗口。超过响应安全上限的远端结果直接拒绝并要求缩小范围,不产生半截 JSON。这里把“上游响应安全边界”“完整结果保存”“模型上下文额度”“用户明确留存”拆成四层,既不丢数据,也不靠无限提高上下文额度解决大结果问题。 **明细扫描边界**:单次响应保留安全上限与模型内联额度,每次 agent run 另按外部系统累计内联返回量；Factory connector 将 `page_size` 限在管理员上限,拒绝 `page=0` / `pageoff` 关闭分页。三者防模型通过连续翻日志自行做昂贵聚合,但不改变 Factory 对其他客户端的分页契约。达到边界后工具正向引导回 dataset/聚合接口、`result_ref` 分段读取或缩小查询范围。 -**状态与 UI（两表）**:`external_system_definitions` 保存管理员维护的可信系统目录、查询规划提示、推荐入口和 `access_mode=all|selected`；这些新增项复用既有 `config` JSONB,无 schema/migration。提示词在 admin 表单里复用通用 dialog 的多行编辑器,不把长文常驻铺在页面。`external_systems` 同时承载指定用户授权和用户密文连接，`pending` 表示已授权但未配置凭据，`active` 才挂工具。管理员撤销指定用户会删除其连接和密文凭据；用户自行断开只清凭据、保留管理员授权。管理后台可新增、编辑、停用目录项，已有用户连接的目录项禁止直接删除。左栏「外部系统」面板只能选择当前用户可见目录、测试连接、替换凭据和断开,不能查看密码。稳定问法沉淀到用户私有 skill 时只写 provider/operation_id/参数规则,永远使用当前提问者的连接执行,共享 skill 不等于共享权限。 +**状态与 UI（三实体）**:`external_system_definitions` 保存可信目录、revision、治理元数据、查询提示、`query|upstream_managed` 执行模式和查询模式下只读 POST 的显式 `operation_id -> read|export` policy；`external_system_grants` 只保存 selected 可见授权；`external_systems` 只保存用户连接、AAD 绑定密文、verified revision 和 `active|invalid|needs_reverify|needs_credentials` 状态。管理员撤权删除独立 grant 并同步删除该用户连接；用户自行断开只删除 connection，grant 保留。凭据使用带 key id 的 AES-GCM envelope，AAD 绑定 user、definition 和字段，旧 Fernet 密文只保留滚动读取入口；调用审计仅保存身份、operation、耗时、状态和响应字节，不保存凭据、请求体或完整响应。管理后台当前仍是唯一 definition 创建入口，未来用户私有定义复用同一模型进入 draft/review 流程。 **不选**:①zcbot 直连 Factory DB(绕过现有 RBAC/审计,只读仍可越权/拖垮主库);②固定几个查询模板(把 agent 降成菜单,无法利用 Factory 已有广泛 API);③直接复用 Factory `ichat` 自由 SQL 原型(字符串安全判断不构成边界,且使用默认 DB 凭据);④自动把相似问题生成并上线新代码工具(候选配方可自动生成,可执行能力仍需工具门控/人审)。 @@ -440,7 +440,7 @@ scheduled_jobs(§8.5) channel_bindings(§8.7,判别列+JSONB) **P0 决策候选——统一 Attention Inbox**:`ask_user` 继续服务「2-4 个方向选择、结束本轮等下一条用户消息」的轻交互；新增 attention item 服务「某个在途动作暂停后从原 tool_call 恢复」。PG 是唯一事实源,最小状态机 `pending -> resolved|expired|cancelled`,以 `(task_id, tool_call_id)` 唯一保证重连/重启不重复提问,resolve 用条件更新实现 first-responder-wins。item 至少记录 kind(approval/question/notification)、脱敏请求摘要、resolution、来源渠道和时间；Web SSE、企业微信/个人微信只是同一 item 的展示/响应 transport,不各存一份状态。删除 task、取消 run、授权被管理员撤回时确定性关闭关联 pending item；恢复前重跑权限判断,防等待期间策略或用户权限已经变化。**边界**:不能把待审批工具参数作为普通 user 文本让模型重新解释,批准的是被冻结且可校验的具体动作；凭据和完整敏感正文不入 item。 -**P0 决策候选——外部动作审计**:新增独立 `action_audit_events`(不挤进回答费用口径的 `usage_events`,也不拿 toolfail 代替),记录 user/task/run、provider/tool/operation、风险级别、决策依据、approval item/rule、执行状态、目标资源标识、脱敏 args/result preview 与时间。审计回答「谁在什么任务中、凭哪条授权、对哪个对象做了什么、结果如何」；token/password/secret、邮件/消息正文、浏览器输入、外部响应全文机械脱敏或不落库。只读查询可按采样/高价值 operation 记,外部写与拒绝/审批必须全记。该项是 §8.14 从查询扩到写操作前的 hard prerequisite。 +**P0 决策候选——统一外部动作审计**:新增独立 `action_audit_events`(不挤进回答费用口径的 `usage_events`,也不拿 toolfail 代替),记录 user/task/run、provider/tool/operation、风险级别、决策依据、approval item/rule、执行状态、目标资源标识、脱敏 args/result preview 与时间。审计回答「谁在什么任务中、凭哪条授权、对哪个对象做了什么、结果如何」；token/password/secret、邮件/消息正文、浏览器输入、外部响应全文机械脱敏或不落库。§8.14 的上游托管 OpenAPI 已用 `external_system_audits` 全量记录 operation、结果、耗时和响应大小，但不保存请求载荷；未来把写能力扩到消息、邮件、浏览器等多 provider 或加入精确目标审批时，再抽象为本表，避免现在为了单一 connector 过早统一。 **P1——定时任务的精确目标长期授权**:无人值守任务不能靠“整个工具永远允许”。借鉴 OpenWorker 的 task-scoped standing rule,授权归具体 scheduled job,形态为 `provider + operation_id/tool + normalized_target`;删除/停用 job 或管理员撤权即失效。`normalized_target` 的组成字段由可信 provider definition 声明(如 recipient/channel_id/plant_id/dataset_id),模型不能自行挑字段,不支持通配符；创建任务时 consent card 同时展示将读取的数据与将写入的精确目标。shell、任意文件删除及无法提取稳定目标的动作不授长期许可,每次仍 ask。现有定时查询与确定性 notify 不受影响；只有未来开放外部写时才启用该契约。 diff --git a/PROGRESS.md b/PROGRESS.md index 49b420d..2513451 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,7 +2,7 @@ > 配合 `DESIGN.md`。本文件只记 phase 状态、决策偏差、文件量、下一步。每条 1-2 句:做了啥 + 关键判断;细节查 `git log` / `git diff` / `DESIGN §7.9`。 -最后更新:2026-08-06(上下文 reasoning 回传治理与任务标题兜底,bump 0.62.9) +最后更新:2026-08-07(外部系统治理底座重构,bump 0.63.0) --- @@ -21,6 +21,10 @@ ## 已完成关键能力 +### 2026-08-07 + +- **08-07 / 0.63.0 / 外部系统治理底座重构**:外部系统持久化拆为 definition、grant、connection 三实体，0027 migration 搬运 selected 授权并去除连接表 provider/connector/config 重复列；definition 增加 revision、owner/visibility、trust/review 与 egress policy 预留，配置变化按目标/认证绑定差异将连接置为 `needs_reverify` 或清密文进入 `needs_credentials`。执行边界拆为 `query|upstream_managed`：通用系统默认 GET/HEAD + 显式只读 POST，Factory 默认开放可信规格全部标准 method 并委托 Factory 按用户凭据鉴权；Swagger 2/OpenAPI 3 先编译统一 catalog，补本地参数 `$ref`、header/cookie、数组序列化与请求体基础校验。运行态缓存统一复用 HTTP 连接、短期 Token、spec 和 catalog，登录/规格/catalog/并发相同只读查询使用 single-flight，明确不做顺序业务查询结果缓存；spec、登录和业务响应均流式硬限长。凭据升级为带 key id、user/definition/field AAD 的 AES-GCM envelope并保留旧 Fernet 滚动读取，新增无敏感载荷调用审计；完整 507 项 unittest 全绿(17 skip)，0027 PostgreSQL DDL 定向编译、Alembic 单 head、外部模块 mypy、Ruff 致命规则和 JavaScript 语法检查通过，未配置或连接生产 DB。 + ### 2026-08-06 - **08-06 / 0.62.9 / reasoning 回传治理 + 上下文环口径统一**:原始 assistant reasoning 继续完整落库供展示/导出，provider-bound 请求按模型档案选择性剥离；DeepSeek V4 仅保留工具调用 reasoning，普通跨轮与模型切换不再携带私有状态。任务详情、SSE、压缩和折叠统一使用清洗后视图，顶部环悬停补充 reasoning 剥离数、当前压缩工具消息数与累计整理次数，超可靠容量封顶显示 `100%+` 并保留真实百分比。自动标题模型调用失败时改用首条消息首行本地兜底并一次性消费 pending；新增设计院工程图纸知识库与智能设计辅助系统调研文档。相关 85 项上下文/前端/循环 unittest 全绿、1 项测试库门控安全跳过，标题专项、Python 编译、JavaScript 语法及 diff 检查通过；无 schema、migration、依赖或运行方式变化，未连接生产 DB。 diff --git a/RUN.md b/RUN.md index 25e4e6c..9682c68 100644 --- a/RUN.md +++ b/RUN.md @@ -150,7 +150,7 @@ - **未绑定成员发消息 → 回绑定指引**(不再静默):聊天优先布局下新员工第一动作就是打字,回调对未绑定成员的 text/图片/文件消息每条回一句"先去控制台绑定"(事件不回)。未绑定成员点菜单「工作台」则落在绑定提示页(不自动建号)。 - **channel 长会话上下文(微信/企业微信通用,0019)**:常驻会话不再无限膨胀。① **自动分段**——入站时距上次消息超过 `config.json` 的 `channel.session_gap_hours`(默 **6** 小时,设 `<=0` 关闭)→ 软重置:只把「最后一条 user 消息起」喂模型(保留上一轮做续聊锚点),之前的历史仍全留 DB,网页端照旧翻完整记录;② **手动新话题**——用户在微信/企业微信里直接发「新话题 / 新会话 / `/new` / 清空上下文」→ 硬重置,彻底从零(回执提示已归档)。两者都**不删任何消息**,只移动「喂给模型的窗口起点」`tasks.context_base_idx`。网页端「清空对话」(`POST /v1/tasks/{id}/clear`)仍整清并把 base 归 0。需 `main.py db upgrade head` 带上 `0019`。 - **PG**:`ZCBOT_DB_URL` 必填。本地 docker compose / 远端 dev / 生产任选;未设置时启动清晰报错,不引导 docker(§7.4)。 -- **OpenAPI 外部系统**:① `.env` 只配置独立的 `ZCBOT_CREDENTIAL_MASTER_KEY`；② 首次启用执行 `main.py db upgrade head`；③ admin 进入管理后台「外部系统」，选择 Factory MES preset 或通用 OpenAPI，配置可信 Base URL、Swagger URL、认证方式、只读 POST operationId、推荐查询入口和查询规划提示，再选择“全部用户”或指定用户。通用类型支持“用户名密码换取 Token”“API Key”“Bearer Token”；用户名、密码、Token 和 Header 字段映射由管理员维护。④ 普通用户点击左栏 **「外部」**，页面按定义动态显示所需凭据。工具下一轮对话开始挂载；推荐入口会在接口搜索中置顶。Factory preset 默认把统计聚合路由到 `bi_dataset_list` → `bi_dataset_exec`，生产日志只用于逐条追溯；单轮累计返回量、`page_size<=200` 及禁止关闭分页共同约束明细扫描。配置复用既有 JSONB，无新 migration。Swagger JSON 只在进程内按系统定义和用户缓存 5 分钟，不写文件/数据库。普通用户和模型仍不能传任意 URL。 +- **OpenAPI 外部系统**:① `.env` 配置独立的 `ZCBOT_CREDENTIAL_MASTER_KEY`，可选 `ZCBOT_CREDENTIAL_KEY_ID` 标识当前密钥；轮换时把旧 key 以 JSON 对象放入 `ZCBOT_CREDENTIAL_PREVIOUS_KEYS`，待用户凭据完成重写后再移除。② 执行 `main.py db upgrade head`，0027 会把既有 selected 授权迁入独立 grants，不连接或清理业务库数据。③ admin 进入管理后台「外部系统」，选择 Factory MES preset 或通用 OpenAPI，配置同源的可信 Base URL / Swagger URL、认证方式、执行模式、推荐入口和查询规划提示，再选择“全部用户”或指定用户。Factory 默认“上游托管”：可信规格声明的全部标准 HTTP method 均可调用，由 Factory 按当前用户凭据最终鉴权；通用系统默认“查询模式”：GET/HEAD 默认可查，POST 只有加入只读清单才开放。④ 普通用户点击左栏 **「外部」**，页面按定义动态显示所需凭据；定义目标或认证变化后必须重新填写凭据，其他策略变化需重新测试连接。通用类型支持“用户名密码换取 Token”“API Key”“Bearer Token”；Swagger JSON 只在进程内按定义和用户有界缓存 5 分钟，spec、登录和业务响应都在流式下载时限长，普通用户和模型不能传任意 URL。 - **测试库(可选,`ZCBOT_TEST_DB_URL`)**:DB 级单测(`tests/test_usage_report.py` / `tests/test_scheduler.py` / `tests/test_web_routes_db.py`)**只认这个显式变量、绝不回退 `.env` 的 `ZCBOT_DB_URL`**——后者可能经隧道指向生产库,测试插入的到点 job 会被生产实例调度守护真跑一次(2026-07-23 实锤)。未设则这几组自动 skip。一键起库(docker,端口 5433 避开本地 5432): ```bash docker run -d --name zcbot-test-pg -e POSTGRES_PASSWORD=zcbot_test \ diff --git a/core/__init__.py b/core/__init__.py index b45a5e3..27157ef 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,3 +1,3 @@ # zcbot 版本号单一事实源:web/app.py 的 FastAPI version、/healthz 返回、前端展示都引这里。 # 改版本只动这一行。 -__version__ = "0.62.9" +__version__ = "0.63.0" diff --git a/core/external_systems/auth.py b/core/external_systems/auth.py index 4d59ea5..b2fe89e 100644 --- a/core/external_systems/auth.py +++ b/core/external_systems/auth.py @@ -2,8 +2,10 @@ 认证只消费管理员保存的可信配置和用户加密保存的字段，不允许模型指定认证地址或请求头。 """ + from __future__ import annotations +import json from dataclasses import dataclass from typing import Any, Protocol from urllib.parse import urljoin @@ -45,7 +47,9 @@ class AuthStrategy(Protocol): def _required(credentials: dict[str, str], fields: tuple[CredentialField, ...]) -> None: - missing = [field.label for field in fields if not credentials.get(field.name, "").strip()] + missing = [ + field.label for field in fields if not credentials.get(field.name, "").strip() + ] if missing: raise ExternalAuthError("请填写" + "、".join(missing)) @@ -59,10 +63,14 @@ def _nested_value(payload: Any, path: str) -> Any: return current -def _auth_header(config: dict[str, Any], token: str, *, default_name: str, default_template: str) -> dict[str, str]: +def _auth_header( + config: dict[str, Any], token: str, *, default_name: str, default_template: str +) -> dict[str, str]: name = str(config.get("auth_header_name") or default_name).strip() template = str(config.get("auth_header_template") or default_template) - if any(char in name for char in "\r\n:") or any(char in template for char in "\r\n"): + if any(char in name for char in "\r\n:") or any( + char in template for char in "\r\n" + ): raise ExternalAuthError("认证 Header 配置非法") if "{token}" not in template: raise ExternalAuthError("认证 Header 模板必须包含 {token}") @@ -70,12 +78,19 @@ def _auth_header(config: dict[str, Any], token: str, *, default_name: str, defau class PasswordJwtAuth: - credential_fields = ( + credential_fields: tuple[CredentialField, ...] = ( CredentialField("username", "用户名", secret=False, autocomplete="username"), CredentialField("password", "密码", autocomplete="current-password"), ) - def headers(self, *, client, base_url, credentials, config) -> dict[str, str]: + def headers( + self, + *, + client: httpx.Client, + base_url: str, + credentials: dict[str, str], + config: dict[str, Any], + ) -> dict[str, str]: _required(credentials, self.credential_fields) login_path = str(config.get("login_path") or "/api/auth/token/").strip() if not login_path.startswith("/") or "://" in login_path: @@ -84,45 +99,84 @@ class PasswordJwtAuth: password_field = str(config.get("password_field") or "password").strip() token_field = str(config.get("token_field") or "access").strip() try: - response = client.post( + with client.stream( + "POST", urljoin(base_url + "/", login_path.lstrip("/")), json={ username_field: credentials["username"], password_field: credentials["password"], }, - ) + ) as response: + chunks: list[bytes] = [] + total = 0 + for chunk in response.iter_bytes(): + total += len(chunk) + if total > 65536: + raise ExternalAuthError("外部系统登录响应超过安全上限") + chunks.append(chunk) + status_code = response.status_code + content = b"".join(chunks) except httpx.HTTPError as exc: - raise ExternalAuthError(f"外部系统登录连接失败: {type(exc).__name__}") from exc - if response.status_code >= 400: - raise ExternalAuthError(f"外部系统登录失败(HTTP {response.status_code})") + raise ExternalAuthError( + f"外部系统登录连接失败: {type(exc).__name__}" + ) from exc + if status_code >= 400: + raise ExternalAuthError(f"外部系统登录失败(HTTP {status_code})") try: - token = _nested_value(response.json(), token_field) - except ValueError: + token = _nested_value(json.loads(content.decode("utf-8-sig")), token_field) + except (UnicodeDecodeError, ValueError): token = None if not isinstance(token, str) or not token: raise ExternalAuthError(f"外部系统登录响应缺少 {token_field}") return _auth_header( - config, token, default_name="Authorization", default_template="Bearer {token}" + config, + token, + default_name="Authorization", + default_template="Bearer {token}", ) class ApiKeyAuth: - credential_fields = (CredentialField("api_key", "API Key"),) + credential_fields: tuple[CredentialField, ...] = ( + CredentialField("api_key", "API Key"), + ) - def headers(self, *, client, base_url, credentials, config) -> dict[str, str]: + def headers( + self, + *, + client: httpx.Client, + base_url: str, + credentials: dict[str, str], + config: dict[str, Any], + ) -> dict[str, str]: _required(credentials, self.credential_fields) return _auth_header( - config, credentials["api_key"], default_name="X-API-Key", default_template="{token}" + config, + credentials["api_key"], + default_name="X-API-Key", + default_template="{token}", ) class BearerTokenAuth: - credential_fields = (CredentialField("token", "Bearer Token"),) + credential_fields: tuple[CredentialField, ...] = ( + CredentialField("token", "Bearer Token"), + ) - def headers(self, *, client, base_url, credentials, config) -> dict[str, str]: + def headers( + self, + *, + client: httpx.Client, + base_url: str, + credentials: dict[str, str], + config: dict[str, Any], + ) -> dict[str, str]: _required(credentials, self.credential_fields) return _auth_header( - config, credentials["token"], default_name="Authorization", default_template="Bearer {token}" + config, + credentials["token"], + default_name="Authorization", + default_template="Bearer {token}", ) @@ -150,7 +204,9 @@ def auth_catalog() -> list[dict[str, Any]]: { "auth_type": key, "title": titles[key], - "credential_fields": [field.as_dict() for field in strategy.credential_fields], + "credential_fields": [ + field.as_dict() for field in strategy.credential_fields + ], } for key, strategy in _AUTH_STRATEGIES.items() ] diff --git a/core/external_systems/catalog.py b/core/external_systems/catalog.py new file mode 100644 index 0000000..857e5f4 --- /dev/null +++ b/core/external_systems/catalog.py @@ -0,0 +1,140 @@ +"""把 Swagger 2 / OpenAPI 3 编译为连接器使用的统一 operation catalog。""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +HTTP_METHODS = ( + "get", + "head", + "post", + "put", + "patch", + "delete", + "options", + "trace", +) + + +def resolve_local_object(spec: dict[str, Any], value: Any) -> Any: + """解析单个本地 JSON Pointer 引用；远端引用保留原值并由上层拒绝。""" + if not isinstance(value, dict): + return value + ref = value.get("$ref") + if not isinstance(ref, str) or not ref.startswith("#/"): + return value + current: Any = spec + for raw in ref[2:].split("/"): + part = raw.replace("~1", "/").replace("~0", "~") + if not isinstance(current, dict) or part not in current: + return value + current = current[part] + return current if isinstance(current, dict) else value + + +def operation_id(method: str, path: str, operation: dict[str, Any]) -> str: + explicit = operation.get("operationId") + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + safe_path = re.sub(r"[^a-zA-Z0-9]+", "_", path).strip("_") + return f"{method}_{safe_path}" + + +@dataclass(frozen=True) +class OperationCatalog: + operations: tuple[dict[str, Any], ...] + + def find(self, operation_id_value: str) -> tuple[dict[str, Any], ...]: + return tuple( + operation + for operation in self.operations + if operation["operation_id"] == operation_id_value + ) + + +def compile_operation_catalog(spec: dict[str, Any]) -> OperationCatalog: + results: list[dict[str, Any]] = [] + for path, path_item in (spec.get("paths") or {}).items(): + if not isinstance(path, str) or not isinstance(path_item, dict): + continue + common = path_item.get("parameters") or [] + for method in HTTP_METHODS: + operation = path_item.get(method) + if not isinstance(operation, dict): + continue + params = [ + resolve_local_object(spec, param) + for param in list(common) + list(operation.get("parameters") or []) + ] + results.append( + { + "operation_id": operation_id(method, path, operation), + "method": method.upper(), + "path": path, + "summary": operation.get("summary") or "", + "description": operation.get("description") or "", + "tags": operation.get("tags") or [], + "parameters": params, + "request_body": resolve_local_object( + spec, operation.get("requestBody") + ), + } + ) + return OperationCatalog(tuple(results)) + + +def validate_json_value( + spec: dict[str, Any], + value: Any, + schema: Any, + *, + path: str = "body", + depth: int = 0, +) -> None: + """验证调用前最关键的 JSON Schema 子集，复杂语义仍由上游最终判定。""" + if depth > 20 or not isinstance(schema, dict): + return + schema = resolve_local_object(spec, schema) + expected = schema.get("type") + type_ok = { + "object": isinstance(value, dict), + "array": isinstance(value, list), + "string": isinstance(value, str), + "integer": isinstance(value, int) and not isinstance(value, bool), + "number": isinstance(value, (int, float)) and not isinstance(value, bool), + "boolean": isinstance(value, bool), + "null": value is None, + } + if expected in type_ok and not type_ok[expected]: + raise ValueError(f"{path} 应为 {expected}") + if "enum" in schema and value not in schema.get("enum", []): + raise ValueError(f"{path} 不在允许值范围内") + if isinstance(value, dict): + required = schema.get("required") or [] + missing = [str(name) for name in required if name not in value] + if missing: + raise ValueError(f"{path} 缺少必填字段: " + ", ".join(missing)) + properties = schema.get("properties") or {} + if isinstance(properties, dict): + for name, child in value.items(): + if name in properties: + validate_json_value( + spec, + child, + properties[name], + path=f"{path}.{name}", + depth=depth + 1, + ) + elif schema.get("additionalProperties") is False: + raise ValueError(f"{path} 包含未定义字段: {name}") + if isinstance(value, list) and isinstance(schema.get("items"), dict): + for index, item in enumerate(value): + validate_json_value( + spec, + item, + schema["items"], + path=f"{path}[{index}]", + depth=depth + 1, + ) diff --git a/core/external_systems/crypto.py b/core/external_systems/crypto.py index c4afa5c..a6bca7d 100644 --- a/core/external_systems/crypto.py +++ b/core/external_systems/crypto.py @@ -2,45 +2,110 @@ 与早期微信绑定不同，这里没有明文降级：未配置 master key 时拒绝创建和调用。 """ + from __future__ import annotations import base64 import hashlib +import json import os +import secrets +from cryptography.exceptions import InvalidTag from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives.ciphers.aead import AESGCM -_PREFIX = "v1:" +_LEGACY_PREFIX = "v1:" +_PREFIX = "v2:" _ENV = "ZCBOT_CREDENTIAL_MASTER_KEY" +_KEY_ID_ENV = "ZCBOT_CREDENTIAL_KEY_ID" +_PREVIOUS_KEYS_ENV = "ZCBOT_CREDENTIAL_PREVIOUS_KEYS" def configured() -> bool: return len(os.getenv(_ENV, "").strip()) >= 32 -def _fernet() -> Fernet: - raw = os.getenv(_ENV, "").strip() - if not raw: - raise RuntimeError(f"{_ENV} 未配置，不能保存或使用外部系统凭据") - if len(raw) < 32: +def _keyring() -> tuple[str, dict[str, str]]: + current = os.getenv(_ENV, "").strip() + if len(current) < 32: raise RuntimeError(f"{_ENV} 至少需要 32 个字符") - digest = hashlib.sha256(raw.encode("utf-8")).digest() - return Fernet(base64.urlsafe_b64encode(digest)) + current_id = os.getenv(_KEY_ID_ENV, "primary").strip() or "primary" + if ":" in current_id or len(current_id) > 64: + raise RuntimeError(f"{_KEY_ID_ENV} 格式无效") + keys = {current_id: current} + raw_previous = os.getenv(_PREVIOUS_KEYS_ENV, "").strip() + if raw_previous: + try: + previous = json.loads(raw_previous) + except ValueError as exc: + raise RuntimeError(f"{_PREVIOUS_KEYS_ENV} 必须是 JSON 对象") from exc + if not isinstance(previous, dict): + raise RuntimeError(f"{_PREVIOUS_KEYS_ENV} 必须是 JSON 对象") + for key_id, secret in previous.items(): + key_id = str(key_id).strip() + secret = str(secret).strip() + if not key_id or ":" in key_id or len(key_id) > 64 or len(secret) < 32: + raise RuntimeError(f"{_PREVIOUS_KEYS_ENV} 包含无效密钥") + keys.setdefault(key_id, secret) + return current_id, keys -def encrypt_secret(value: str) -> str: +def _aes_key(secret: str) -> bytes: + return hashlib.sha256(secret.encode("utf-8")).digest() + + +def encrypt_secret(value: str, *, aad: str = "") -> str: if not isinstance(value, str) or not value: raise ValueError("credential value must be a non-empty string") - return _PREFIX + _fernet().encrypt(value.encode("utf-8")).decode("ascii") + key_id, keys = _keyring() + nonce = secrets.token_bytes(12) + ciphertext = AESGCM(_aes_key(keys[key_id])).encrypt( + nonce, + value.encode("utf-8"), + aad.encode("utf-8"), + ) + encoded = base64.urlsafe_b64encode(nonce + ciphertext).decode("ascii") + return f"{_PREFIX}{key_id}:{encoded}" -def decrypt_secret(value: str) -> str: - if not isinstance(value, str) or not value.startswith(_PREFIX): +def decrypt_secret(value: str, *, aad: str = "") -> str: + if not isinstance(value, str): raise RuntimeError("外部系统凭据格式无效") + if value.startswith(_PREFIX): + try: + _, key_id, encoded = value.split(":", 2) + _, keys = _keyring() + secret = keys[key_id] + payload = base64.urlsafe_b64decode(encoded.encode("ascii")) + plaintext = AESGCM(_aes_key(secret)).decrypt( + payload[:12], + payload[12:], + aad.encode("utf-8"), + ) + return plaintext.decode("utf-8") + except (KeyError, ValueError, InvalidTag, UnicodeDecodeError) as exc: + raise RuntimeError( + "外部系统凭据无法解密，密钥或绑定上下文可能已变化" + ) from exc + if not value.startswith(_LEGACY_PREFIX): + raise RuntimeError("外部系统凭据格式无效") + # 0027 前的 Fernet 密文没有 key id/AAD；仅用于滚动迁移时读取。 try: - return _fernet().decrypt(value[len(_PREFIX):].encode("ascii")).decode("utf-8") - except InvalidToken as exc: + _, keys = _keyring() + for secret in keys.values(): + digest = hashlib.sha256(secret.encode("utf-8")).digest() + try: + return ( + Fernet(base64.urlsafe_b64encode(digest)) + .decrypt(value[len(_LEGACY_PREFIX) :].encode("ascii")) + .decode("utf-8") + ) + except InvalidToken: + continue + except (RuntimeError, UnicodeDecodeError) as exc: raise RuntimeError("外部系统凭据无法解密，master key 可能已变化") from exc + raise RuntimeError("外部系统凭据无法解密，master key 可能已变化") def mask_username(username: str) -> str: diff --git a/core/external_systems/factory.py b/core/external_systems/factory.py index a122fad..8228291 100644 --- a/core/external_systems/factory.py +++ b/core/external_systems/factory.py @@ -3,11 +3,12 @@ 新代码使用 :mod:`core.external_systems.openapi`；保留原类名，避免已有测试和内部引用 在通用化过程中发生无意义破坏。 """ + from __future__ import annotations from typing import Any -from .openapi import OpenApiClient, OpenApiConfig, OpenApiError, _SPEC_CACHE +from .openapi import OpenApiClient, OpenApiConfig, OpenApiError from .registry import merged_config diff --git a/core/external_systems/openapi.py b/core/external_systems/openapi.py index 73b5176..0b7eff5 100644 --- a/core/external_systems/openapi.py +++ b/core/external_systems/openapi.py @@ -3,42 +3,68 @@ 目标地址全部来自管理员维护的可信系统目录；模型和普通用户只能传 operation_id 与结构化参数，不能传 URL。 """ + from __future__ import annotations -import json +import base64 +import binascii +import copy import hashlib +import json import re import time +from contextlib import contextmanager from dataclasses import dataclass, field -from threading import Lock -from typing import Any, Optional +from typing import Any, Iterator, Optional from urllib.parse import quote, urljoin, urlparse import httpx from .auth import ExternalAuthError, get_auth_strategy +from .catalog import ( + compile_operation_catalog, + operation_id, + resolve_local_object, + validate_json_value, +) from .results import MAX_STORED_RESULT_BYTES +from .runtime_cache import RUNTIME_CACHE class OpenApiError(RuntimeError): pass -_HTTP_METHODS = ("get", "head", "post", "put", "patch", "delete") -_SPEC_CACHE: dict[str, tuple[float, dict[str, Any]]] = {} -_SPEC_LOCK = Lock() +_AUTH_CACHE_TTL_SECONDS = 300 +_SPEC_CACHE_TTL_SECONDS = 300 _SCHEMA_MAX_DEPTH = 5 _SCHEMA_MAX_PROPERTIES = 50 _SCHEMA_MAX_ENUM_ITEMS = 30 _SCHEMA_MAX_NODES = 100 _SCHEMA_TEXT_MAX_CHARS = 1000 _ERROR_DETAIL_MAX_CHARS = 2000 +_MAX_SPEC_BYTES = 5 * 1024 * 1024 _SENSITIVE_KEY_RE = re.compile( r"(?:password|passwd|secret|token|api[_-]?key|authorization|cookie|credential)", re.IGNORECASE, ) +class _SpecCacheView: + """保留测试和诊断入口；实际数据由统一运行态缓存持有。""" + + @staticmethod + def clear() -> None: + RUNTIME_CACHE.clear() + + @staticmethod + def __len__() -> int: + return RUNTIME_CACHE.spec_count() + + +_SPEC_CACHE = _SpecCacheView() + + def _bool_value(value: Any, default: bool) -> bool: raw = str(value if value is not None else "").strip().lower() if not raw: @@ -61,7 +87,7 @@ class OpenApiConfig: base_url: str openapi_url: str login_path: str - allowed_post_operations: frozenset[str] + operation_policies: dict[str, str] timeout_seconds: float max_result_bytes: int max_total_result_bytes: int @@ -69,53 +95,77 @@ class OpenApiConfig: verify_tls: bool query_guidance: str recommended_operation_ids: tuple[str, ...] + operation_mode: str = "query" auth_type: str = "password_jwt" - auth_config: dict[str, Any] = field(default_factory=lambda: { - "login_path": "/api/auth/token/", - "username_field": "username", - "password_field": "password", - "token_field": "access", - "auth_header_name": "Authorization", - "auth_header_template": "Bearer {token}", - }) + auth_config: dict[str, Any] = field( + default_factory=lambda: { + "login_path": "/api/auth/token/", + "username_field": "username", + "password_field": "password", + "token_field": "access", + "auth_header_name": "Authorization", + "auth_header_template": "Bearer {token}", + } + ) @classmethod def from_mapping(cls, data: dict[str, Any]) -> "OpenApiConfig": """从管理员保存的可信目录配置构建运行态配置。""" base = _validated_http_url(str(data.get("base_url") or ""), "base_url") - spec = _validated_http_url( - str(data.get("openapi_url") or ""), "openapi_url" - ) + spec = _validated_http_url(str(data.get("openapi_url") or ""), "openapi_url") + base_origin = urlparse(base) + spec_origin = urlparse(spec) + if (base_origin.scheme, base_origin.netloc) != ( + spec_origin.scheme, + spec_origin.netloc, + ): + raise OpenApiError("openapi_url 必须与 base_url 同源") login_path = str(data.get("login_path") or "/api/auth/token/").strip() if not login_path.startswith("/") or "://" in login_path: raise OpenApiError("login_path 必须是站内绝对路径") - raw_allowed = data.get("allowed_post_operations") or [] - if isinstance(raw_allowed, str): - raw_allowed = raw_allowed.split(",") - if not isinstance(raw_allowed, (list, tuple, set)): - raise OpenApiError("allowed_post_operations 必须是字符串数组") - allowed = frozenset(str(item).strip() for item in raw_allowed if str(item).strip()) + raw_policies = data.get("operation_policies") or {} + if not isinstance(raw_policies, dict): + raise OpenApiError("operation_policies 必须是 operationId 到策略的对象") + policies = { + str(operation_id).strip(): str(policy).strip().lower() + for operation_id, policy in raw_policies.items() + if str(operation_id).strip() + } + if len(policies) > 500 or any(len(key) > 200 for key in policies): + raise OpenApiError( + "operation_policies 最多 500 项且 operationId 不超过 200 字符" + ) + invalid_policies = sorted(set(policies.values()) - {"read", "export"}) + if invalid_policies: + raise OpenApiError( + "不支持的 operation policy: " + ", ".join(invalid_policies) + ) + operation_mode = str(data.get("operation_mode") or "query").strip().lower() + if operation_mode not in {"query", "upstream_managed"}: + raise OpenApiError("operation_mode 必须是 query 或 upstream_managed") guidance = str(data.get("query_guidance") or "").strip() if len(guidance) > 4000: raise OpenApiError("query_guidance 不能超过 4000 字符") - raw_recommended = data.get( - "recommended_operation_ids", [] - ) + raw_recommended = data.get("recommended_operation_ids", []) if isinstance(raw_recommended, str): raw_recommended = raw_recommended.split(",") if not isinstance(raw_recommended, (list, tuple, set)): raise OpenApiError("recommended_operation_ids 必须是字符串数组") - recommended = tuple(dict.fromkeys( - str(item).strip() for item in raw_recommended if str(item).strip() - )) + recommended = tuple( + dict.fromkeys( + str(item).strip() for item in raw_recommended if str(item).strip() + ) + ) if len(recommended) > 30 or any(len(item) > 200 for item in recommended): - raise OpenApiError("recommended_operation_ids 最多 30 项且每项不超过 200 字符") + raise OpenApiError( + "recommended_operation_ids 最多 30 项且每项不超过 200 字符" + ) max_result = max(4096, min(int(data.get("max_result_bytes", 65536)), 1048576)) return cls( base_url=base, openapi_url=spec, login_path=login_path, - allowed_post_operations=allowed, + operation_policies=policies, timeout_seconds=max(1.0, min(float(data.get("timeout_seconds", 15)), 60.0)), max_result_bytes=max_result, max_total_result_bytes=max( @@ -126,12 +176,17 @@ class OpenApiConfig: verify_tls=_bool_value(data.get("verify_tls"), True), query_guidance=guidance, recommended_operation_ids=recommended, + operation_mode=operation_mode, auth_type=str(data.get("auth_type") or "password_jwt").strip(), auth_config={ key: data[key] for key in ( - "login_path", "username_field", "password_field", "token_field", - "auth_header_name", "auth_header_template", + "login_path", + "username_field", + "password_field", + "token_field", + "auth_header_name", + "auth_header_template", ) if key in data }, @@ -148,9 +203,18 @@ class OpenApiClient: ): self.credentials = credentials self.cfg = cfg - identity = cache_namespace or json.dumps(credentials, sort_keys=True, ensure_ascii=False) + identity = json.dumps( + { + "namespace": cache_namespace, + "credentials": credentials, + "config": cfg.__dict__, + }, + sort_keys=True, + ensure_ascii=False, + default=str, + ) digest = hashlib.sha256(identity.encode("utf-8")).hexdigest() - self._spec_cache_key = f"{cfg.openapi_url}:{digest}" + self._runtime_identity = digest def _client(self) -> httpx.Client: return httpx.Client( @@ -159,75 +223,198 @@ class OpenApiClient: follow_redirects=False, ) - def authenticate(self) -> dict[str, str]: - try: - with self._client() as client: - return get_auth_strategy(self.cfg.auth_type).headers( + @contextmanager + def _runtime_client(self) -> Iterator[httpx.Client]: + with RUNTIME_CACHE.client(self._runtime_identity, self._client) as client: + yield client + + @staticmethod + def _auth_cache_ttl(headers: dict[str, str]) -> float: + authorization = next( + ( + value + for name, value in headers.items() + if name.lower() == "authorization" + ), + "", + ) + token = authorization.split(" ", 1)[-1].strip() + parts = token.split(".") + if len(parts) == 3: + try: + padding = "=" * (-len(parts[1]) % 4) + payload = json.loads( + base64.urlsafe_b64decode(parts[1] + padding).decode("utf-8") + ) + expires_at = float(payload.get("exp")) + return max( + 0.0, + min(_AUTH_CACHE_TTL_SECONDS, expires_at - time.time() - 30), + ) + except (binascii.Error, TypeError, ValueError, UnicodeDecodeError): + pass + return _AUTH_CACHE_TTL_SECONDS + + def authenticate( + self, *, client: httpx.Client | None = None, force: bool = False + ) -> dict[str, str]: + if client is None: + with self._runtime_client() as runtime_client: + return self.authenticate(client=runtime_client, force=force) + if force: + RUNTIME_CACHE.invalidate_auth(self._runtime_identity) + else: + cached = RUNTIME_CACHE.get_auth(self._runtime_identity) + if cached is not None: + return cached + + def load() -> dict[str, str]: + if not force: + cached = RUNTIME_CACHE.get_auth(self._runtime_identity) + if cached is not None: + return cached + try: + headers = get_auth_strategy(self.cfg.auth_type).headers( client=client, base_url=self.cfg.base_url, credentials=self.credentials, config=self.cfg.auth_config, ) - except ExternalAuthError as exc: - raise OpenApiError(str(exc)) from exc + except ExternalAuthError as exc: + raise OpenApiError(str(exc)) from exc + RUNTIME_CACHE.set_auth( + self._runtime_identity, + headers, + ttl_seconds=self._auth_cache_ttl(headers), + ) + return dict(headers) - def _fetch_spec(self, headers: dict[str, str]) -> dict[str, Any]: - now = time.monotonic() - with _SPEC_LOCK: - hit = _SPEC_CACHE.get(self._spec_cache_key) - if hit and now - hit[0] < 300: - return hit[1] - try: - with self._client() as client: - response = client.get( + return RUNTIME_CACHE.singleflight("auth", self._runtime_identity, load) + + def _refresh_auth( + self, client: httpx.Client, *, failed_generation: int + ) -> dict[str, str]: + def refresh() -> dict[str, str]: + cached, current_generation = RUNTIME_CACHE.auth_state( + self._runtime_identity + ) + if cached is not None and current_generation != failed_generation: + return cached + return self.authenticate(client=client, force=True) + + return RUNTIME_CACHE.singleflight( + "auth-refresh", self._runtime_identity, refresh + ) + + @staticmethod + def _limited_request( + client: httpx.Client, + method: str, + url: str, + *, + limit: int, + label: str, + **kwargs: Any, + ) -> tuple[int, dict[str, str], bytes]: + """流式读取远端响应，在解析 JSON 前执行硬字节上限。""" + with client.stream(method, url, **kwargs) as response: + raw_length = response.headers.get("content-length") + if raw_length: + try: + if int(raw_length) > limit: + raise OpenApiError(f"{label}超过安全下载上限({limit} bytes)") + except ValueError: + pass + chunks: list[bytes] = [] + total = 0 + for chunk in response.iter_bytes(): + total += len(chunk) + if total > limit: + raise OpenApiError(f"{label}超过安全下载上限({limit} bytes)") + chunks.append(chunk) + return response.status_code, dict(response.headers), b"".join(chunks) + + def _fetch_spec( + self, + headers: dict[str, str], + *, + client: httpx.Client | None = None, + ) -> dict[str, Any]: + if client is None: + with self._runtime_client() as runtime_client: + return self._fetch_spec(headers, client=runtime_client) + cached = RUNTIME_CACHE.get_spec(self._runtime_identity) + if cached is not None: + return cached + + def load() -> dict[str, Any]: + cached = RUNTIME_CACHE.get_spec(self._runtime_identity) + if cached is not None: + return cached + request_client = client + try: + _, auth_generation = RUNTIME_CACHE.auth_state(self._runtime_identity) + status_code, _, content = self._limited_request( + request_client, + "GET", self.cfg.openapi_url, headers=headers, + limit=_MAX_SPEC_BYTES, + label="OpenAPI 文档", ) - except httpx.HTTPError as exc: - raise OpenApiError(f"OpenAPI 获取失败: {type(exc).__name__}") from exc - if response.status_code >= 400: - raise OpenApiError(f"OpenAPI 获取失败(HTTP {response.status_code})") - try: - spec = response.json() - except ValueError as exc: - raise OpenApiError("OpenAPI 文档不是有效 JSON") from exc - if not isinstance(spec, dict) or not isinstance(spec.get("paths"), dict): - raise OpenApiError("OpenAPI 文档缺少 paths") - with _SPEC_LOCK: - _SPEC_CACHE[self._spec_cache_key] = (now, spec) - return spec + if status_code == 401: + refreshed_headers = self._refresh_auth( + request_client, failed_generation=auth_generation + ) + status_code, _, content = self._limited_request( + request_client, + "GET", + self.cfg.openapi_url, + headers=refreshed_headers, + limit=_MAX_SPEC_BYTES, + label="OpenAPI 文档", + ) + except httpx.HTTPError as exc: + raise OpenApiError(f"OpenAPI 获取失败: {type(exc).__name__}") from exc + if status_code >= 400: + raise OpenApiError(f"OpenAPI 获取失败(HTTP {status_code})") + try: + spec = json.loads(content.decode("utf-8-sig")) + except (UnicodeDecodeError, ValueError) as exc: + raise OpenApiError("OpenAPI 文档不是有效 JSON") from exc + if not isinstance(spec, dict) or not isinstance(spec.get("paths"), dict): + raise OpenApiError("OpenAPI 文档缺少 paths") + RUNTIME_CACHE.set_spec( + self._runtime_identity, spec, ttl_seconds=_SPEC_CACHE_TTL_SECONDS + ) + return spec + + return RUNTIME_CACHE.singleflight("spec", self._runtime_identity, load) @staticmethod def _operation_id(method: str, path: str, operation: dict[str, Any]) -> str: - explicit = operation.get("operationId") - if isinstance(explicit, str) and explicit.strip(): - return explicit.strip() - safe_path = re.sub(r"[^a-zA-Z0-9]+", "_", path).strip("_") - return f"{method}_{safe_path}" + return operation_id(method, path, operation) - @classmethod - def _operations(cls, spec: dict[str, Any]) -> list[dict[str, Any]]: - results: list[dict[str, Any]] = [] - for path, path_item in (spec.get("paths") or {}).items(): - if not isinstance(path_item, dict): - continue - common = path_item.get("parameters") or [] - for method in _HTTP_METHODS: - operation = path_item.get(method) - if not isinstance(operation, dict): - continue - params = list(common) + list(operation.get("parameters") or []) - results.append({ - "operation_id": cls._operation_id(method, path, operation), - "method": method.upper(), - "path": path, - "summary": operation.get("summary") or "", - "description": operation.get("description") or "", - "tags": operation.get("tags") or [], - "parameters": params, - "request_body": operation.get("requestBody"), - }) - return results + @staticmethod + def _resolve_object(spec: dict[str, Any], value: Any) -> Any: + return resolve_local_object(spec, value) + + def _operations(self, spec: dict[str, Any]) -> list[dict[str, Any]]: + cached = RUNTIME_CACHE.get_catalog(self._runtime_identity, spec) + if cached is None: + + def compile_catalog(): + current = RUNTIME_CACHE.get_catalog(self._runtime_identity, spec) + if current is not None: + return current + catalog = compile_operation_catalog(spec) + RUNTIME_CACHE.set_catalog(self._runtime_identity, spec, catalog) + return catalog + + cached = RUNTIME_CACHE.singleflight( + "catalog", self._runtime_identity, compile_catalog + ) + return list(cached.operations) @classmethod def _compact_schema( @@ -425,7 +612,12 @@ class OpenApiClient: if not isinstance(raw_base_path, str) or not raw_base_path.startswith("/"): raise OpenApiError("Swagger basePath 必须是站内绝对路径") parsed = urlparse(raw_base_path) - if parsed.netloc or parsed.query or parsed.fragment or "://" in raw_base_path: + if ( + parsed.netloc + or parsed.query + or parsed.fragment + or "://" in raw_base_path + ): raise OpenApiError("Swagger basePath 非法") return parsed.path.rstrip("/") @@ -462,7 +654,7 @@ class OpenApiClient: path == prefix or path.startswith(prefix + "/") ) if configured_has_prefix and operation_has_prefix: - path = path[len(prefix):] or "/" + path = path[len(prefix) :] or "/" elif prefix and not configured_has_prefix and not operation_has_prefix: path = prefix + "/" + path.lstrip("/") combined_path = "/".join( @@ -477,36 +669,57 @@ class OpenApiClient: return url def test_connection(self) -> dict[str, Any]: - headers = self.authenticate() - spec = self._fetch_spec(headers) - return {"operation_count": len(self._operations(spec))} + RUNTIME_CACHE.invalidate_auth(self._runtime_identity) + RUNTIME_CACHE.invalidate_spec(self._runtime_identity) + with self._runtime_client() as client: + headers = self.authenticate(client=client) + spec = self._fetch_spec(headers, client=client) + return {"operation_count": len(self._operations(spec))} + + def _operation_allowed(self, operation: dict[str, Any]) -> bool: + if self.cfg.operation_mode == "upstream_managed": + return True + method = operation["method"].lower() + return method in {"get", "head"} or ( + method == "post" + and self.cfg.operation_policies.get(operation["operation_id"]) + in {"read", "export"} + ) def search(self, query: str, limit: int = 12) -> list[dict[str, Any]]: query = (query or "").strip().lower() if not query: raise OpenApiError("query 不能为空") - headers = self.authenticate() - spec = self._fetch_spec(headers) - terms = list(dict.fromkeys( - [query] + [x for x in re.split(r"[\s,，。/]+", query) if len(x) >= 2] - )) + with self._runtime_client() as client: + headers = self.authenticate(client=client) + spec = self._fetch_spec(headers, client=client) + terms = list( + dict.fromkeys( + [query] + [x for x in re.split(r"[\s,，。/]+", query) if len(x) >= 2] + ) + ) recommended_order = { operation_id: index for index, operation_id in enumerate(self.cfg.recommended_operation_ids) } scored: list[tuple[int, int, dict[str, Any]]] = [] for op in self._operations(spec): - method = op["method"].lower() - if method not in {"get", "head"} and not ( - method == "post" - and op["operation_id"] in self.cfg.allowed_post_operations - ): + if not self._operation_allowed(op): continue - hay = " ".join([ - op["operation_id"], op["path"], op["summary"], op["description"], - " ".join(str(x) for x in op["tags"]), - ]).lower() - score = sum(5 if term == query and term in hay else 1 for term in terms if term in hay) + hay = " ".join( + [ + op["operation_id"], + op["path"], + op["summary"], + op["description"], + " ".join(str(x) for x in op["tags"]), + ] + ).lower() + score = sum( + 5 if term == query and term in hay else 1 + for term in terms + if term in hay + ) recommended = op["operation_id"] in recommended_order if score or recommended: compact = dict(op) @@ -518,18 +731,21 @@ class OpenApiClient: "type": p.get("type") or (p.get("schema") or {}).get("type"), "description": p.get("description") or "", } - for p in op["parameters"] if isinstance(p, dict) and "$ref" not in p + for p in op["parameters"] + if isinstance(p, dict) and "$ref" not in p ] body_contract = self._body_contract(spec, op) if body_contract is not None: compact["body"] = body_contract compact.pop("request_body", None) compact["recommended"] = recommended - scored.append(( - 0 if recommended else 1, - recommended_order.get(op["operation_id"], -score), - compact, - )) + scored.append( + ( + 0 if recommended else 1, + recommended_order.get(op["operation_id"], -score), + compact, + ) + ) scored.sort(key=lambda item: (item[0], item[1], item[2]["operation_id"])) return [item[2] for item in scored[: max(1, min(int(limit), 30))]] @@ -539,23 +755,36 @@ class OpenApiClient: arguments: Optional[dict[str, Any]] = None, body: Any = None, ) -> dict[str, Any]: - headers = self.authenticate() - spec = self._fetch_spec(headers) - matches = [op for op in self._operations(spec) if op["operation_id"] == operation_id] + with self._runtime_client() as client: + return self._call_with_client(client, operation_id, arguments, body) + + def _call_with_client( + self, + client: httpx.Client, + operation_id: str, + arguments: Optional[dict[str, Any]], + body: Any, + ) -> dict[str, Any]: + headers = self.authenticate(client=client) + spec = self._fetch_spec(headers, client=client) + headers = self.authenticate(client=client) + matches = [ + op for op in self._operations(spec) if op["operation_id"] == operation_id + ] if len(matches) != 1: raise OpenApiError("operation_id 不存在或不唯一，请先搜索接口") op = matches[0] if not op["path"].startswith("/") or "://" in op["path"]: raise OpenApiError("OpenAPI operation path 非法") method = op["method"].lower() - if method not in {"get", "head"} and not ( - method == "post" and operation_id in self.cfg.allowed_post_operations - ): + if not self._operation_allowed(op): raise OpenApiError(f"operation {operation_id} 未列入只读调用范围") supplied = dict(arguments or {}) path = op["path"] query: dict[str, Any] = {} + parameter_headers: dict[str, str] = {} + cookies: dict[str, str] = {} request_body = body for param in op["parameters"]: if not isinstance(param, dict) or "$ref" in param: @@ -566,12 +795,26 @@ class OpenApiClient: continue # Swagger 2 的 body 参数既可按搜索结果中的参数名放在 arguments， # 也可使用元工具独立的 body 字段；两者只取一个。 - present = name in supplied or (location == "body" and request_body is not None) + present = name in supplied or ( + location == "body" and request_body is not None + ) if param.get("required") and not present: raise OpenApiError(f"缺少必填参数: {name}") if name not in supplied: continue value = supplied.pop(name) + parameter_schema = param.get("schema") or { + key: param[key] for key in ("type", "enum", "items") if key in param + } + try: + validate_json_value( + spec, + value, + parameter_schema, + path=f"arguments.{name}", + ) + except ValueError as exc: + raise OpenApiError(str(exc)) from exc if location == "path": path = path.replace("{" + name + "}", quote(str(value), safe="")) elif location == "query": @@ -588,46 +831,155 @@ class OpenApiClient: raise OpenApiError( "外部系统查询不允许关闭分页，请使用 dataset 或分页查看明细" ) - query[name] = value + if isinstance(value, list): + raw_collection_format = param.get("collectionFormat") + collection_format = ( + raw_collection_format + if isinstance(raw_collection_format, str) + else "" + ) + raw_style = param.get("style") + style = raw_style if isinstance(raw_style, str) else "" + explode = param.get("explode", True) + separators = { + "csv": ",", + "ssv": " ", + "tsv": "\t", + "pipes": "|", + "spaceDelimited": " ", + "pipeDelimited": "|", + } + separator = separators.get(collection_format) or separators.get( + style + ) + if separator: + query[name] = separator.join(str(item) for item in value) + elif style == "form" and explode is False: + query[name] = ",".join(str(item) for item in value) + else: + query[name] = value + else: + query[name] = value + elif location == "header": + if _SENSITIVE_KEY_RE.search(name): + raise OpenApiError(f"接口参数不允许覆盖敏感 Header: {name}") + parameter_headers[name] = str(value) + elif location == "cookie": + if _SENSITIVE_KEY_RE.search(name): + raise OpenApiError(f"接口参数不允许覆盖敏感 Cookie: {name}") + cookies[name] = str(value) elif location == "body" and request_body is None: request_body = value + elif location not in {"body"}: + raise OpenApiError(f"暂不支持参数位置: {location}") if supplied: raise OpenApiError("存在接口定义之外的参数: " + ", ".join(sorted(supplied))) if "{" in path or "}" in path: raise OpenApiError("路径参数未完整提供") + body_contract = self._body_contract(spec, op) + if request_body is not None and body_contract is not None: + if "json" not in str(body_contract.get("content_type") or "").lower(): + raise OpenApiError("当前连接器仅支持 JSON 请求体") + try: + validate_json_value(spec, request_body, body_contract.get("schema")) + except ValueError as exc: + raise OpenApiError(str(exc)) from exc + elif body_contract and body_contract.get("required"): + raise OpenApiError("缺少必填请求体") url = self._operation_url(spec, path) - try: - with self._client() as client: - response = client.request( + + def execute_request() -> dict[str, Any]: + request_headers = {**headers, **parameter_headers} + try: + _, auth_generation = RUNTIME_CACHE.auth_state(self._runtime_identity) + status_code, response_headers, content = self._limited_request( + client, method.upper(), url, params=query, - json=request_body if method == "post" else None, - headers=headers, + json=request_body if request_body is not None else None, + headers=request_headers, + cookies=cookies, + limit=MAX_STORED_RESULT_BYTES, + label="外部系统响应", ) - except httpx.HTTPError as exc: - raise OpenApiError(f"外部系统接口调用失败: {type(exc).__name__}") from exc - if response.status_code >= 400: - detail = self._error_detail(response) - suffix = f": {detail}" if detail else "" - raise OpenApiError(f"外部系统接口返回 HTTP {response.status_code}{suffix}") - content_type = response.headers.get("content-type", "") - try: - payload: Any = response.json() if "json" in content_type else response.text - except ValueError: - payload = response.text - encoded = json.dumps(payload, ensure_ascii=False, default=str) - response_bytes = len(encoded.encode("utf-8")) - if response_bytes > MAX_STORED_RESULT_BYTES: - raise OpenApiError( - f"外部系统响应超过安全下载上限({MAX_STORED_RESULT_BYTES} bytes)，" - "请缩小查询范围或使用远端分页/聚合接口" + if status_code == 401 and self.cfg.auth_type == "password_jwt": + refreshed = self._refresh_auth( + client, failed_generation=auth_generation + ) + status_code, response_headers, content = self._limited_request( + client, + method.upper(), + url, + params=query, + json=request_body if request_body is not None else None, + headers={**refreshed, **parameter_headers}, + cookies=cookies, + limit=MAX_STORED_RESULT_BYTES, + label="外部系统响应", + ) + except httpx.HTTPError as exc: + raise OpenApiError( + f"外部系统接口调用失败: {type(exc).__name__}" + ) from exc + response = httpx.Response( + status_code, + headers=response_headers, + content=content, ) - return { - "operation_id": operation_id, - "status_code": response.status_code, - "truncated": False, - "response_bytes": response_bytes, - "data": payload, - } + if status_code >= 400: + detail = self._error_detail(response) + suffix = f": {detail}" if detail else "" + raise OpenApiError(f"外部系统接口返回 HTTP {status_code}{suffix}") + content_type = response_headers.get("content-type", "") + try: + payload: Any = ( + json.loads(content.decode("utf-8-sig")) + if "json" in content_type + else content.decode("utf-8", errors="replace") + ) + except ValueError: + payload = content.decode("utf-8", errors="replace") + encoded = json.dumps(payload, ensure_ascii=False, default=str) + response_bytes = len(encoded.encode("utf-8")) + if response_bytes > MAX_STORED_RESULT_BYTES: + raise OpenApiError( + f"外部系统响应超过安全下载上限({MAX_STORED_RESULT_BYTES} bytes)，" + "请缩小查询范围或使用远端分页/聚合接口" + ) + return { + "operation_id": operation_id, + "status_code": status_code, + "truncated": False, + "response_bytes": response_bytes, + "data": payload, + } + + query_like = method in {"get", "head"} or ( + method == "post" + and self.cfg.operation_policies.get(operation_id) in {"read", "export"} + ) + if not query_like: + return execute_request() + request_fingerprint = hashlib.sha256( + json.dumps( + { + "method": method, + "url": url, + "query": query, + "headers": parameter_headers, + "cookies": cookies, + "body": request_body, + }, + sort_keys=True, + ensure_ascii=False, + default=str, + ).encode("utf-8") + ).hexdigest() + result = RUNTIME_CACHE.singleflight( + "query", + f"{self._runtime_identity}:{request_fingerprint}", + execute_request, + ) + return copy.deepcopy(result) diff --git a/core/external_systems/registry.py b/core/external_systems/registry.py index bf6221d..39f626a 100644 --- a/core/external_systems/registry.py +++ b/core/external_systems/registry.py @@ -2,6 +2,7 @@ 标准 OpenAPI 系统通过数据库配置接入；只有非 OpenAPI 协议才需要新增 connector 文件。 """ + from __future__ import annotations from dataclasses import dataclass @@ -9,7 +10,6 @@ from typing import Any from .auth import ExternalAuthError, get_auth_strategy - FACTORY_QUERY_GUIDANCE = ( "产量、良率、缺陷、库存、绩效、趋势和按日/月汇总等统计聚合查询，" "统一先调用 BI dataset list，再执行匹配的数据集。日志和业务明细列表用于" @@ -45,6 +45,10 @@ _PROVIDERS = { "auth_header_template": "Bearer {token}", "query_guidance": FACTORY_QUERY_GUIDANCE, "recommended_operation_ids": list(FACTORY_RECOMMENDED_OPERATIONS), + "operation_mode": "upstream_managed", + "operation_policies": { + "bi_dataset_exec": "read", + }, }, ), "generic_openapi": ProviderSpec( @@ -62,6 +66,8 @@ _PROVIDERS = { "auth_header_template": "Bearer {token}", "query_guidance": "", "recommended_operation_ids": [], + "operation_mode": "query", + "operation_policies": {}, }, ), } diff --git a/core/external_systems/runtime_cache.py b/core/external_systems/runtime_cache.py new file mode 100644 index 0000000..2efa5ba --- /dev/null +++ b/core/external_systems/runtime_cache.py @@ -0,0 +1,206 @@ +"""外部系统进程内运行态缓存与同步 single-flight。""" + +from __future__ import annotations + +import atexit +import time +from collections import OrderedDict +from concurrent.futures import Future +from contextlib import contextmanager +from dataclasses import dataclass +from threading import RLock +from typing import Any, Callable, Iterator, TypeVar + + +T = TypeVar("T") + + +@dataclass +class _RuntimeEntry: + client: Any = None + active_client_leases: int = 0 + evicted: bool = False + auth_headers: dict[str, str] | None = None + auth_expires_at: float = 0.0 + auth_generation: int = 0 + spec: dict[str, Any] | None = None + spec_expires_at: float = 0.0 + catalog: Any = None + catalog_spec: dict[str, Any] | None = None + + +class ExternalRuntimeCache: + """按连接身份隔离的有界 LRU；凭据、Token 和规格均只驻留当前进程。""" + + def __init__(self, *, max_entries: int = 256): + self.max_entries = max_entries + self._entries: OrderedDict[str, _RuntimeEntry] = OrderedDict() + self._inflight: dict[tuple[str, str], Future[Any]] = {} + self._lock = RLock() + + @staticmethod + def _close_client(client: Any) -> None: + close = getattr(client, "close", None) + if callable(close): + try: + close() + except Exception: + pass + + def _entry_locked(self, identity: str) -> _RuntimeEntry: + entry = self._entries.get(identity) + if entry is None: + entry = _RuntimeEntry() + self._entries[identity] = entry + self._entries.move_to_end(identity) + while len(self._entries) > self.max_entries: + _, evicted = self._entries.popitem(last=False) + evicted.evicted = True + if evicted.active_client_leases == 0: + self._close_client(evicted.client) + return entry + + @contextmanager + def client(self, identity: str, factory: Callable[[], T]) -> Iterator[T]: + with self._lock: + entry = self._entry_locked(identity) + if entry.client is None: + entry.client = factory() + entry.active_client_leases += 1 + client = entry.client + try: + yield client + finally: + close_client = None + with self._lock: + entry.active_client_leases -= 1 + if entry.evicted and entry.active_client_leases == 0: + close_client = entry.client + entry.client = None + if close_client is not None: + self._close_client(close_client) + + def get_auth(self, identity: str) -> dict[str, str] | None: + now = time.monotonic() + with self._lock: + entry = self._entries.get(identity) + if entry is None or entry.auth_expires_at <= now: + if entry is not None: + entry.auth_headers = None + entry.auth_expires_at = 0.0 + return None + self._entries.move_to_end(identity) + return dict(entry.auth_headers or {}) + + def set_auth( + self, identity: str, headers: dict[str, str], *, ttl_seconds: float + ) -> None: + with self._lock: + entry = self._entry_locked(identity) + entry.auth_headers = dict(headers) + entry.auth_expires_at = time.monotonic() + max(0.0, ttl_seconds) + entry.auth_generation += 1 + + def auth_state(self, identity: str) -> tuple[dict[str, str] | None, int]: + headers = self.get_auth(identity) + with self._lock: + entry = self._entries.get(identity) + return headers, entry.auth_generation if entry is not None else 0 + + def invalidate_auth(self, identity: str) -> None: + with self._lock: + entry = self._entries.get(identity) + if entry is not None: + entry.auth_headers = None + entry.auth_expires_at = 0.0 + + def get_spec(self, identity: str) -> dict[str, Any] | None: + now = time.monotonic() + with self._lock: + entry = self._entries.get(identity) + if entry is None or entry.spec_expires_at <= now: + if entry is not None: + entry.spec = None + entry.spec_expires_at = 0.0 + entry.catalog = None + entry.catalog_spec = None + return None + self._entries.move_to_end(identity) + return entry.spec + + def set_spec( + self, identity: str, spec: dict[str, Any], *, ttl_seconds: float + ) -> None: + with self._lock: + entry = self._entry_locked(identity) + entry.spec = spec + entry.spec_expires_at = time.monotonic() + max(0.0, ttl_seconds) + entry.catalog = None + entry.catalog_spec = None + + def invalidate_spec(self, identity: str) -> None: + with self._lock: + entry = self._entries.get(identity) + if entry is not None: + entry.spec = None + entry.spec_expires_at = 0.0 + entry.catalog = None + entry.catalog_spec = None + + def get_catalog(self, identity: str, spec: dict[str, Any]) -> Any: + with self._lock: + entry = self._entries.get(identity) + if entry is None or entry.catalog_spec is not spec: + return None + self._entries.move_to_end(identity) + return entry.catalog + + def set_catalog(self, identity: str, spec: dict[str, Any], catalog: Any) -> None: + with self._lock: + entry = self._entry_locked(identity) + entry.catalog_spec = spec + entry.catalog = catalog + + def singleflight(self, namespace: str, key: str, compute: Callable[[], T]) -> T: + flight_key = (namespace, key) + with self._lock: + future = self._inflight.get(flight_key) + leader = future is None + if leader: + future = Future() + self._inflight[flight_key] = future + assert future is not None + if not leader: + return future.result() + try: + result = compute() + except BaseException as exc: + future.set_exception(exc) + raise + else: + future.set_result(result) + return result + finally: + with self._lock: + if self._inflight.get(flight_key) is future: + self._inflight.pop(flight_key, None) + + def clear(self) -> None: + with self._lock: + entries = list(self._entries.values()) + self._entries.clear() + for entry in entries: + self._close_client(entry.client) + + def spec_count(self) -> int: + now = time.monotonic() + with self._lock: + return sum( + 1 + for entry in self._entries.values() + if entry.spec is not None and entry.spec_expires_at > now + ) + + +RUNTIME_CACHE = ExternalRuntimeCache() +atexit.register(RUNTIME_CACHE.clear) diff --git a/core/external_systems/service.py b/core/external_systems/service.py index 9945313..5f81fdd 100644 --- a/core/external_systems/service.py +++ b/core/external_systems/service.py @@ -1,7 +1,9 @@ """外部系统目录、用户可见授权和密文连接的持久化服务层。""" + from __future__ import annotations from datetime import datetime, timezone +from collections.abc import Sequence from typing import Any, Optional from urllib.parse import urlparse from uuid import UUID @@ -10,7 +12,13 @@ from sqlalchemy import delete, exists, or_, select from sqlalchemy.exc import IntegrityError from core.storage import session_scope -from core.storage.models import ExternalSystem, ExternalSystemDefinition, User +from core.storage.models import ( + ExternalSystem, + ExternalSystemAudit, + ExternalSystemDefinition, + ExternalSystemGrant, + User, +) from .crypto import configured as crypto_configured from .crypto import decrypt_secret, encrypt_secret, mask_username @@ -35,7 +43,8 @@ def _normalized_config(provider: str, data: dict[str, Any]) -> dict[str, Any]: "base_url": cfg.base_url, "openapi_url": cfg.openapi_url, "login_path": cfg.login_path, - "allowed_post_operations": sorted(cfg.allowed_post_operations), + "operation_mode": cfg.operation_mode, + "operation_policies": dict(sorted(cfg.operation_policies.items())), "timeout_seconds": cfg.timeout_seconds, "max_result_bytes": cfg.max_result_bytes, "max_total_result_bytes": cfg.max_total_result_bytes, @@ -48,7 +57,9 @@ def _normalized_config(provider: str, data: dict[str, Any]) -> dict[str, Any]: } -def _definition_view(row: ExternalSystemDefinition, *, include_config: bool) -> dict[str, Any]: +def _definition_view( + row: ExternalSystemDefinition, *, include_config: bool +) -> dict[str, Any]: config = row.config or {} result = { "definition_id": str(row.definition_id), @@ -57,7 +68,13 @@ def _definition_view(row: ExternalSystemDefinition, *, include_config: bool) -> "connector": get_provider(row.provider).connector, "name": row.name, "enabled": row.enabled, - "access_mode": row.access_mode, + "revision": row.revision, + "owner_type": row.owner_type, + "owner_user_id": str(row.owner_user_id) if row.owner_user_id else None, + "visibility": row.visibility, + "trust_level": row.trust_level, + "review_status": row.review_status, + "egress_policy_id": row.egress_policy_id, "host": urlparse(str(config.get("base_url") or "")).hostname or "", "created_at": row.created_at.isoformat() if row.created_at else None, "updated_at": row.updated_at.isoformat() if row.updated_at else None, @@ -68,20 +85,37 @@ def _definition_view(row: ExternalSystemDefinition, *, include_config: bool) -> return result -def _validate_access_mode(access_mode: str) -> str: - mode = (access_mode or "selected").strip().lower() - if mode not in {"all", "selected"}: - raise ExternalSystemError("access_mode 必须是 all 或 selected") +def _validate_visibility(visibility: str) -> str: + mode = (visibility or "selected").strip().lower() + if mode not in {"organization", "selected", "private"}: + raise ExternalSystemError("visibility 必须是 organization、selected 或 private") return mode +def _invalidate_connections_for_revision( + connections: Sequence[ExternalSystem], *, credential_binding_changed: bool +) -> None: + for connection in connections: + connection.last_verified_at = None + connection.last_error = "系统定义已更新，请重新验证连接" + if credential_binding_changed: + connection.credentials = {} + connection.credential_hint = "***" + connection.status = "needs_credentials" + else: + connection.status = "needs_reverify" + + def _selected_user_ids(s: Any, definition_id: UUID) -> list[str]: return [ - str(uid) for uid in s.execute( - select(ExternalSystem.user_id) - .where(ExternalSystem.definition_id == definition_id) - .order_by(ExternalSystem.user_id) - ).scalars().all() + str(uid) + for uid in s.execute( + select(ExternalSystemGrant.user_id) + .where(ExternalSystemGrant.definition_id == definition_id) + .order_by(ExternalSystemGrant.user_id) + ) + .scalars() + .all() ] @@ -89,57 +123,90 @@ def _sync_selected_users( s: Any, definition: ExternalSystemDefinition, selected_user_ids: list[UUID], + *, + granted_by: UUID | None, ) -> None: wanted = set(selected_user_ids) if wanted: - existing_users = set(s.execute( - select(User.user_id).where(User.user_id.in_(wanted)) - ).scalars().all()) + existing_users = set( + s.execute(select(User.user_id).where(User.user_id.in_(wanted))) + .scalars() + .all() + ) missing = wanted - existing_users if missing: - raise ExternalSystemError("包含不存在的用户: " + ", ".join(sorted(map(str, missing)))) - current_rows = s.execute( - select(ExternalSystem).where( - ExternalSystem.definition_id == definition.definition_id + raise ExternalSystemError( + "包含不存在的用户: " + ", ".join(sorted(map(str, missing))) + ) + current_rows = ( + s.execute( + select(ExternalSystemGrant).where( + ExternalSystemGrant.definition_id == definition.definition_id + ) ) - ).scalars().all() + .scalars() + .all() + ) current = {row.user_id: row for row in current_rows} for uid, row in current.items(): if uid not in wanted: - s.delete(row) # 撤权同时删除该用户的密文凭据 + s.delete(row) + connection = s.execute( + select(ExternalSystem).where( + ExternalSystem.definition_id == definition.definition_id, + ExternalSystem.user_id == uid, + ) + ).scalar_one_or_none() + if connection is not None: + s.delete(connection) # 撤权同时删除该用户的密文凭据 + connections = ( + s.execute( + select(ExternalSystem).where( + ExternalSystem.definition_id == definition.definition_id + ) + ) + .scalars() + .all() + ) + for connection in connections: + if connection.user_id not in wanted: + s.delete(connection) for uid in wanted - set(current): - s.add(ExternalSystem( - user_id=uid, - definition_id=definition.definition_id, - provider=definition.provider, - connector=get_provider(definition.provider).connector, - name=definition.name, - credentials={}, - config={}, - status="pending", - )) + s.add( + ExternalSystemGrant( + user_id=uid, + definition_id=definition.definition_id, + granted_by=granted_by, + ) + ) def provider_catalog(user_id: UUID) -> list[dict[str, Any]]: try: with session_scope() as s: - rows = s.execute( - select(ExternalSystemDefinition) - .where( - ExternalSystemDefinition.enabled.is_(True), - or_( - ExternalSystemDefinition.access_mode == "all", - exists( - select(ExternalSystem.external_system_id).where( - ExternalSystem.definition_id - == ExternalSystemDefinition.definition_id, - ExternalSystem.user_id == user_id, - ) + rows = ( + s.execute( + select(ExternalSystemDefinition) + .where( + ExternalSystemDefinition.enabled.is_(True), + or_( + ExternalSystemDefinition.visibility == "organization", + ExternalSystemDefinition.owner_user_id == user_id, + exists( + select(ExternalSystemGrant.user_id).where( + ExternalSystemGrant.definition_id + == ExternalSystemDefinition.definition_id, + ExternalSystemGrant.user_id == user_id, + ) + ), ), - ), + ExternalSystemDefinition.review_status == "active", + ) + .order_by(ExternalSystemDefinition.name) ) - .order_by(ExternalSystemDefinition.name) - ).scalars().all() + .scalars() + .all() + ) definitions_by_provider: dict[str, list[dict[str, Any]]] = {} for row in rows: definitions_by_provider.setdefault(row.provider, []).append( @@ -156,7 +223,9 @@ def provider_catalog(user_id: UUID) -> list[dict[str, Any]]: "default_auth_type": spec.default_auth_type, "allowed_auth_types": list(spec.allowed_auth_types), "configured": bool(definitions_by_provider.get(spec.provider) and key_ok), - "reason": "" if key_ok else "ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符", + "reason": "" + if key_ok + else "ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符", "definitions": definitions_by_provider.get(spec.provider, []), } for spec in provider_specs() @@ -165,9 +234,13 @@ def provider_catalog(user_id: UUID) -> list[dict[str, Any]]: def list_external_system_definitions() -> list[dict[str, Any]]: with session_scope() as s: - rows = s.execute( - select(ExternalSystemDefinition).order_by(ExternalSystemDefinition.name) - ).scalars().all() + rows = ( + s.execute( + select(ExternalSystemDefinition).order_by(ExternalSystemDefinition.name) + ) + .scalars() + .all() + ) results = [] for row in rows: item = _definition_view(row, include_config=True) @@ -183,7 +256,7 @@ def create_external_system_definition( name: str, config: dict[str, Any], enabled: bool = True, - access_mode: str = "selected", + visibility: str = "selected", selected_user_ids: Optional[list[UUID]] = None, ) -> dict[str, Any]: provider = (provider or "").strip() @@ -199,15 +272,20 @@ def create_external_system_definition( name=name, config=_normalized_config(provider, config), enabled=bool(enabled), - access_mode=_validate_access_mode(access_mode), + visibility=_validate_visibility(visibility), + owner_type="platform", + trust_level="managed", + review_status="active", created_by=admin_user_id, ) try: with session_scope() as s: s.add(row) s.flush() - if row.access_mode == "selected": - _sync_selected_users(s, row, selected_user_ids or []) + if row.visibility == "selected": + _sync_selected_users( + s, row, selected_user_ids or [], granted_by=admin_user_id + ) s.flush() result = _definition_view(row, include_config=True) result["selected_user_ids"] = _selected_user_ids(s, row.definition_id) @@ -222,7 +300,7 @@ def update_external_system_definition( name: str, config: dict[str, Any], enabled: bool, - access_mode: str, + visibility: str, selected_user_ids: Optional[list[UUID]] = None, ) -> dict[str, Any]: name = (name or "").strip() @@ -237,12 +315,59 @@ def update_external_system_definition( ).scalar_one_or_none() if row is None: raise ExternalSystemError("external system definition not found") + old_config = row.config or {} + new_config = _normalized_config(row.provider, config) + config_changed = old_config != new_config + credential_binding_keys = { + "base_url", + "openapi_url", + "login_path", + "auth_type", + "username_field", + "password_field", + "token_field", + "auth_header_name", + "auth_header_template", + } + binding_changed = any( + old_config.get(key) != new_config.get(key) + for key in credential_binding_keys + ) row.name = name - row.config = _normalized_config(row.provider, config) + row.config = new_config row.enabled = bool(enabled) - row.access_mode = _validate_access_mode(access_mode) - if row.access_mode == "selected": - _sync_selected_users(s, row, selected_user_ids or []) + row.visibility = _validate_visibility(visibility) + if config_changed: + row.revision += 1 + connections = ( + s.execute( + select(ExternalSystem).where( + ExternalSystem.definition_id == definition_id + ) + ) + .scalars() + .all() + ) + _invalidate_connections_for_revision( + connections, + credential_binding_changed=binding_changed, + ) + if row.visibility == "selected": + _sync_selected_users( + s, row, selected_user_ids or [], granted_by=row.created_by + ) + else: + grants = ( + s.execute( + select(ExternalSystemGrant).where( + ExternalSystemGrant.definition_id == definition_id + ) + ) + .scalars() + .all() + ) + for grant in grants: + s.delete(grant) s.flush() result = _definition_view(row, include_config=True) result["selected_user_ids"] = _selected_user_ids(s, row.definition_id) @@ -259,12 +384,14 @@ def delete_external_system_definition(definition_id: UUID) -> bool: ExternalSystemDefinition.definition_id == definition_id ) ) - return bool(result.rowcount) + return bool(getattr(result, "rowcount", 0)) except IntegrityError as exc: raise ExternalSystemError("该系统已有用户连接，请先停用而不是删除") from exc -def get_definition(definition_id: UUID, *, enabled_only: bool = False) -> ExternalSystemDefinition: +def get_definition( + definition_id: UUID, *, enabled_only: bool = False +) -> ExternalSystemDefinition: with session_scope() as s: stmt = select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id @@ -278,18 +405,22 @@ def get_definition(definition_id: UUID, *, enabled_only: bool = False) -> Extern return row -def get_definition_for_user(user_id: UUID, definition_id: UUID) -> ExternalSystemDefinition: +def get_definition_for_user( + user_id: UUID, definition_id: UUID +) -> ExternalSystemDefinition: with session_scope() as s: row = s.execute( select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id, ExternalSystemDefinition.enabled.is_(True), + ExternalSystemDefinition.review_status == "active", or_( - ExternalSystemDefinition.access_mode == "all", + ExternalSystemDefinition.visibility == "organization", + ExternalSystemDefinition.owner_user_id == user_id, exists( - select(ExternalSystem.external_system_id).where( - ExternalSystem.definition_id == definition_id, - ExternalSystem.user_id == user_id, + select(ExternalSystemGrant.user_id).where( + ExternalSystemGrant.definition_id == definition_id, + ExternalSystemGrant.user_id == user_id, ) ), ), @@ -310,7 +441,9 @@ def _client( ) -> OpenApiClient: spec = get_provider(provider) if spec.connector != "openapi": - raise ExternalSystemError(f"unsupported external system connector: {spec.connector}") + raise ExternalSystemError( + f"unsupported external system connector: {spec.connector}" + ) return OpenApiClient( credentials, _runtime_config(provider, config), @@ -333,40 +466,57 @@ def _credential_values( def _credentials( - provider: str, config: dict[str, Any], credentials: dict[str, str] + provider: str, + config: dict[str, Any], + credentials: dict[str, str], + *, + user_id: UUID, + definition_id: UUID, ) -> dict[str, str]: normalized = _credential_values(provider, config, credentials) try: - return {name: encrypt_secret(value) for name, value in normalized.items()} + return { + name: encrypt_secret( + value, + aad=f"{user_id}:{definition_id}:{name}", + ) + for name, value in normalized.items() + } except (RuntimeError, ValueError) as exc: raise ExternalSystemError(str(exc)) from exc def credentials_for(row: ExternalSystem) -> dict[str, str]: try: - return {name: decrypt_secret(value) for name, value in row.credentials.items()} + return { + name: decrypt_secret( + value, + aad=f"{row.user_id}:{row.definition_id}:{name}", + ) + for name, value in row.credentials.items() + } except (AttributeError, RuntimeError) as exc: raise ExternalSystemError(str(exc)) from exc def client_for_external_system(row: ExternalSystem) -> OpenApiClient: definition = get_definition_for_user(row.user_id, row.definition_id) + if row.status != "active" or row.verified_revision != definition.revision: + raise ExternalSystemError("外部系统连接需要重新验证") return _client( definition.provider, credentials_for(row), definition.config or {}, - cache_namespace=f"{definition.definition_id}:{row.user_id}", + cache_namespace=( + f"connection:{row.external_system_id}:revision:{row.verified_revision}" + ), ) def _view(row: ExternalSystem, definition: ExternalSystemDefinition) -> dict[str, Any]: try: - credentials = credentials_for(row) - identity = credentials.get("username") or next(iter(credentials.values())) - masked = mask_username(identity) if credentials.get("username") else "***" - credential_ok = True - except (ExternalSystemError, StopIteration): - masked = "***" + credential_ok = bool(row.credentials) + except (AttributeError, TypeError): credential_ok = False runtime_config = _runtime_config(definition.provider, definition.config or {}) return { @@ -374,15 +524,23 @@ def _view(row: ExternalSystem, definition: ExternalSystemDefinition) -> dict[str "definition_id": str(row.definition_id), "system_name": definition.name, "provider": definition.provider, - "connector": row.connector, + "connector": get_provider(definition.provider).connector, "name": row.name, "status": row.status if definition.enabled else "disabled", - "username_masked": masked, + "username_masked": row.credential_hint or "***", "credential_configured": credential_ok, - "credential_fields": credential_fields(definition.provider, definition.config or {}), + "credential_fields": credential_fields( + definition.provider, definition.config or {} + ), "query_guidance": runtime_config.query_guidance, "recommended_operation_ids": list(runtime_config.recommended_operation_ids), - "last_verified_at": row.last_verified_at.isoformat() if row.last_verified_at else None, + "operation_mode": runtime_config.operation_mode, + "last_verified_at": row.last_verified_at.isoformat() + if row.last_verified_at + else None, + "definition_revision": definition.revision, + "verified_revision": row.verified_revision, + "last_error": row.last_error or "", "created_at": row.created_at.isoformat() if row.created_at else None, "updated_at": row.updated_at.isoformat() if row.updated_at else None, } @@ -397,13 +555,14 @@ def list_external_systems(user_id: UUID) -> list[dict[str, Any]]: ExternalSystemDefinition.definition_id == ExternalSystem.definition_id, ) .where(ExternalSystem.user_id == user_id) - .where(ExternalSystem.status != "pending") .order_by(ExternalSystem.created_at) ).all() return [_view(row, definition) for row, definition in rows] -def get_external_system(user_id: UUID, system_id: UUID, *, active_only: bool = False) -> ExternalSystem: +def get_external_system( + user_id: UUID, system_id: UUID, *, active_only: bool = False +) -> ExternalSystem: with session_scope() as s: stmt = select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, @@ -423,9 +582,7 @@ def create_external_system( *, definition_id: UUID, name: str, - credentials: Optional[dict[str, str]] = None, - username: str = "", - password: str = "", + credentials: dict[str, str], ) -> dict[str, Any]: if not crypto_configured(): raise ExternalSystemError("ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符") @@ -436,14 +593,17 @@ def create_external_system( plain = _credential_values( definition.provider, definition.config or {}, - credentials or {"username": username, "password": password}, + credentials, ) try: - probe = _client( + _client( definition.provider, plain, definition.config, - cache_namespace=f"{definition.definition_id}:{user_id}", + cache_namespace=( + f"probe:{definition.definition_id}:revision:{definition.revision}:" + f"user:{user_id}" + ), ).test_connection() except OpenApiError as exc: raise ExternalSystemError(str(exc)) from exc @@ -455,20 +615,27 @@ def create_external_system( ExternalSystem.definition_id == definition.definition_id, ) ).scalar_one_or_none() - if row is not None and row.status != "pending": + if row is not None: raise ExternalSystemError("该外部系统已连接，请使用更新凭据") - if row is None: - row = ExternalSystem( - user_id=user_id, - definition_id=definition.definition_id, - provider=definition.provider, - connector=get_provider(definition.provider).connector, - ) - s.add(row) + row = ExternalSystem( + user_id=user_id, + definition_id=definition.definition_id, + ) + s.add(row) row.name = name - row.credentials = _credentials(definition.provider, definition.config or {}, plain) - row.config = {"operation_count": probe.get("operation_count", 0)} + row.credentials = _credentials( + definition.provider, + definition.config or {}, + plain, + user_id=user_id, + definition_id=definition.definition_id, + ) + row.credential_hint = ( + mask_username(plain["username"]) if plain.get("username") else "***" + ) row.status = "active" + row.verified_revision = definition.revision + row.last_error = None row.last_verified_at = datetime.now(timezone.utc) s.flush() return _view(row, definition) @@ -480,23 +647,23 @@ def update_external_system_credentials( user_id: UUID, system_id: UUID, *, - credentials: Optional[dict[str, str]] = None, - username: str = "", - password: str = "", + credentials: dict[str, str], ) -> dict[str, Any]: row = get_external_system(user_id, system_id) definition = get_definition(row.definition_id, enabled_only=True) plain = _credential_values( definition.provider, definition.config or {}, - credentials or {"username": username, "password": password}, + credentials, ) try: - probe = _client( + _client( definition.provider, plain, definition.config, - cache_namespace=f"{definition.definition_id}:{user_id}", + cache_namespace=( + f"probe:{system_id}:revision:{definition.revision}:user:{user_id}" + ), ).test_connection() except OpenApiError as exc: raise ExternalSystemError(str(exc)) from exc @@ -507,9 +674,19 @@ def update_external_system_credentials( ExternalSystem.user_id == user_id, ) ).scalar_one() - current.credentials = _credentials(definition.provider, definition.config or {}, plain) - current.config = {**(current.config or {}), "operation_count": probe.get("operation_count", 0)} + current.credentials = _credentials( + definition.provider, + definition.config or {}, + plain, + user_id=user_id, + definition_id=definition.definition_id, + ) + current.credential_hint = ( + mask_username(plain["username"]) if plain.get("username") else "***" + ) current.status = "active" + current.verified_revision = definition.revision + current.last_error = None current.last_verified_at = datetime.now(timezone.utc) s.flush() return _view(current, definition) @@ -519,7 +696,15 @@ def test_external_system(user_id: UUID, system_id: UUID) -> dict[str, Any]: row = get_external_system(user_id, system_id) ok, error, probe = False, "", {} try: - probe = client_for_external_system(row).test_connection() + definition = get_definition_for_user(user_id, row.definition_id) + probe = _client( + definition.provider, + credentials_for(row), + definition.config or {}, + cache_namespace=( + f"connection:{row.external_system_id}:revision:{definition.revision}" + ), + ).test_connection() ok = True except (ExternalSystemError, OpenApiError) as exc: error = str(exc) @@ -531,9 +716,15 @@ def test_external_system(user_id: UUID, system_id: UUID) -> dict[str, Any]: ) ).scalar_one() current.status = "active" if ok else "invalid" + current.last_error = None if ok else error[:2000] if ok: current.last_verified_at = datetime.now(timezone.utc) - current.config = {**(current.config or {}), **probe} + current_definition = s.execute( + select(ExternalSystemDefinition).where( + ExternalSystemDefinition.definition_id == current.definition_id + ) + ).scalar_one() + current.verified_revision = current_definition.revision return {"ok": ok, "error": error if not ok else "", **probe} @@ -547,19 +738,7 @@ def delete_external_system(user_id: UUID, system_id: UUID) -> bool: ).scalar_one_or_none() if row is None: return False - definition = s.execute( - select(ExternalSystemDefinition).where( - ExternalSystemDefinition.definition_id == row.definition_id - ) - ).scalar_one() - if definition.access_mode == "selected": - # 用户断开只清凭据，保留管理员授予的可见权。 - row.credentials = {} - row.config = {} - row.status = "pending" - row.last_verified_at = None - else: - s.delete(row) + s.delete(row) # grants 独立存在，断开连接不撤销可见授权。 return True @@ -568,18 +747,60 @@ def external_system_tools_available(user_id: UUID) -> bool: return False try: with session_scope() as s: - return s.execute( - select(ExternalSystem.external_system_id) - .join( - ExternalSystemDefinition, - ExternalSystemDefinition.definition_id == ExternalSystem.definition_id, - ) - .where( - ExternalSystem.user_id == user_id, - ExternalSystem.status == "active", - ExternalSystemDefinition.enabled.is_(True), - ) - .limit(1) - ).scalar_one_or_none() is not None + return ( + s.execute( + select(ExternalSystem.external_system_id) + .join( + ExternalSystemDefinition, + ExternalSystemDefinition.definition_id + == ExternalSystem.definition_id, + ) + .where( + ExternalSystem.user_id == user_id, + ExternalSystem.status == "active", + ExternalSystemDefinition.enabled.is_(True), + ExternalSystemDefinition.review_status == "active", + ExternalSystem.verified_revision + == ExternalSystemDefinition.revision, + ) + .limit(1) + ).scalar_one_or_none() + is not None + ) except Exception: return False + + +def record_external_system_audit( + *, + user_id: UUID, + task_id: UUID | None, + external_system_id: UUID | None, + definition_id: UUID | None, + definition_revision: int, + event: str, + operation_id: str | None, + outcome: str, + duration_ms: int, + status_code: int | None = None, + response_bytes: int | None = None, + detail: dict[str, Any] | None = None, +) -> None: + """写入无敏感载荷的调用审计；调用方决定失败时是否降级。""" + with session_scope() as s: + s.add( + ExternalSystemAudit( + user_id=user_id, + task_id=task_id, + external_system_id=external_system_id, + definition_id=definition_id, + definition_revision=max(0, int(definition_revision)), + event=event, + operation_id=operation_id, + outcome=outcome, + status_code=status_code, + duration_ms=max(0, int(duration_ms)), + response_bytes=response_bytes, + detail=detail or {}, + ) + ) diff --git a/core/storage/models.py b/core/storage/models.py index 1df8ae2..c95f39a 100644 --- a/core/storage/models.py +++ b/core/storage/models.py @@ -25,11 +25,13 @@ from sqlalchemy import ( Boolean, DateTime, ForeignKey, + Index, Integer, Numeric, Text, UniqueConstraint, func, + text, ) from sqlalchemy.dialects.postgresql import JSONB, UUID as PG_UUID from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, validates @@ -347,7 +349,21 @@ class ExternalSystemDefinition(Base): __tablename__ = "external_system_definitions" __table_args__ = ( - UniqueConstraint("provider", "name", name="uq_external_system_definition_provider_name"), + Index( + "uq_external_system_definition_platform_name", + "provider", + "name", + unique=True, + postgresql_where=text("owner_type = 'platform'"), + ), + Index( + "uq_external_system_definition_user_name", + "owner_user_id", + "provider", + "name", + unique=True, + postgresql_where=text("owner_type = 'user'"), + ), ) definition_id: Mapped[UUID] = mapped_column( @@ -356,9 +372,25 @@ class ExternalSystemDefinition(Base): provider: Mapped[str] = mapped_column(Text, nullable=False) name: Mapped[str] = mapped_column(Text, nullable=False) config: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict) - access_mode: Mapped[str] = mapped_column( + revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1") + owner_type: Mapped[str] = mapped_column( + Text, nullable=False, default="platform", server_default="platform" + ) + owner_user_id: Mapped[Optional[UUID]] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("users.user_id", ondelete="SET NULL"), + nullable=True, + ) + visibility: Mapped[str] = mapped_column( Text, nullable=False, default="selected", server_default="selected" ) + trust_level: Mapped[str] = mapped_column( + Text, nullable=False, default="managed", server_default="managed" + ) + review_status: Mapped[str] = mapped_column( + Text, nullable=False, default="active", server_default="active" + ) + egress_policy_id: Mapped[Optional[str]] = mapped_column(Text, nullable=True) enabled: Mapped[bool] = mapped_column( Boolean, nullable=False, default=True, server_default="true" ) @@ -375,11 +407,36 @@ class ExternalSystemDefinition(Base): ) +class ExternalSystemGrant(Base): + """平台定义对用户的可见与连接授权；不承载连接或凭据状态。""" + + __tablename__ = "external_system_grants" + + definition_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("external_system_definitions.definition_id", ondelete="CASCADE"), + primary_key=True, + ) + user_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("users.user_id", ondelete="CASCADE"), + primary_key=True, + ) + granted_by: Mapped[Optional[UUID]] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("users.user_id", ondelete="SET NULL"), + nullable=True, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + class ExternalSystem(Base): """用户配置的外部业务系统连接(DESIGN §8.14)。 - credentials 只保存 host-side 加密后的字段；API、prompt、工具参数和用户文件 - 均不得出现明文。provider/connector 由平台定义，用户不能提交任意目标 URL。 + 授权关系独立保存在 external_system_grants。连接只保存用户显示名、密文凭据和 + 相对 definition revision 的验证状态，provider/connector/config 均从定义派生。 """ __tablename__ = "external_systems" @@ -402,16 +459,18 @@ class ExternalSystem(Base): ForeignKey("external_system_definitions.definition_id", ondelete="RESTRICT"), nullable=False, ) - provider: Mapped[str] = mapped_column(Text, nullable=False) - connector: Mapped[str] = mapped_column( - Text, nullable=False, default="openapi", server_default="openapi" - ) name: Mapped[str] = mapped_column(Text, nullable=False) credentials: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict) - config: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict) + credential_hint: Mapped[str] = mapped_column( + Text, nullable=False, default="***", server_default="***" + ) status: Mapped[str] = mapped_column( Text, nullable=False, default="active", server_default="active" ) + verified_revision: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + last_error: Mapped[Optional[str]] = mapped_column(Text, nullable=True) last_verified_at: Mapped[Optional[datetime]] = mapped_column( DateTime(timezone=True), nullable=True ) @@ -422,3 +481,39 @@ class ExternalSystem(Base): DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False ) + +class ExternalSystemAudit(Base): + """外部系统调用的结构化审计；不保存凭据、请求体或完整业务响应。""" + + __tablename__ = "external_system_audits" + + audit_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), primary_key=True, default=uuid4 + ) + user_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), ForeignKey("users.user_id", ondelete="CASCADE"), nullable=False + ) + task_id: Mapped[Optional[UUID]] = mapped_column( + PG_UUID(as_uuid=True), ForeignKey("tasks.task_id", ondelete="SET NULL"), nullable=True + ) + external_system_id: Mapped[Optional[UUID]] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("external_systems.external_system_id", ondelete="SET NULL"), + nullable=True, + ) + definition_id: Mapped[Optional[UUID]] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("external_system_definitions.definition_id", ondelete="SET NULL"), + nullable=True, + ) + definition_revision: Mapped[int] = mapped_column(Integer, nullable=False) + event: Mapped[str] = mapped_column(Text, nullable=False) + operation_id: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + outcome: Mapped[str] = mapped_column(Text, nullable=False) + status_code: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + duration_ms: Mapped[int] = mapped_column(Integer, nullable=False) + response_bytes: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True) + detail: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) diff --git a/core/tool_registry.py b/core/tool_registry.py index 7eecb96..c5386a2 100644 --- a/core/tool_registry.py +++ b/core/tool_registry.py @@ -156,6 +156,8 @@ def build_tools(ctx: ToolContext) -> dict[str, Any]: ] def _external_systems() -> list: + from core.external_systems.service import record_external_system_audit + result_budget: dict[str, int] = {} return [ ExternalSystemListTool(ctx.uid, **base), @@ -164,6 +166,7 @@ def build_tools(ctx: ToolContext) -> dict[str, Any]: ctx.uid, task_id=ctx.task_id, result_budget=result_budget, + audit_recorder=record_external_system_audit, **wd_base, ), ExternalSystemResultReadTool( diff --git a/db/migrations/versions/20260807_1000_0027_external_system_governance.py b/db/migrations/versions/20260807_1000_0027_external_system_governance.py new file mode 100644 index 0000000..29f4a99 --- /dev/null +++ b/db/migrations/versions/20260807_1000_0027_external_system_governance.py @@ -0,0 +1,308 @@ +"""Split external-system grants from connections and add definition governance. + +Revision ID: 0027 +Revises: 0026 +Create Date: 2026-08-07 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0027" +down_revision: Union[str, None] = "0026" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "external_system_definitions", + sa.Column("revision", sa.Integer(), server_default="1", nullable=False), + ) + op.add_column( + "external_system_definitions", + sa.Column("owner_type", sa.Text(), server_default="platform", nullable=False), + ) + op.add_column( + "external_system_definitions", + sa.Column("owner_user_id", postgresql.UUID(as_uuid=True), nullable=True), + ) + op.add_column( + "external_system_definitions", + sa.Column("visibility", sa.Text(), server_default="selected", nullable=False), + ) + op.add_column( + "external_system_definitions", + sa.Column("trust_level", sa.Text(), server_default="managed", nullable=False), + ) + op.add_column( + "external_system_definitions", + sa.Column("review_status", sa.Text(), server_default="active", nullable=False), + ) + op.add_column( + "external_system_definitions", + sa.Column("egress_policy_id", sa.Text(), nullable=True), + ) + op.create_foreign_key( + "fk_external_system_definitions_owner_user", + "external_system_definitions", + "users", + ["owner_user_id"], + ["user_id"], + ondelete="SET NULL", + ) + op.drop_constraint( + "uq_external_system_definition_provider_name", + "external_system_definitions", + type_="unique", + ) + op.create_index( + "uq_external_system_definition_platform_name", + "external_system_definitions", + ["provider", "name"], + unique=True, + postgresql_where=sa.text("owner_type = 'platform'"), + ) + op.create_index( + "uq_external_system_definition_user_name", + "external_system_definitions", + ["owner_user_id", "provider", "name"], + unique=True, + postgresql_where=sa.text("owner_type = 'user'"), + ) + op.execute("UPDATE external_system_definitions SET visibility = access_mode") + op.execute( + """ + UPDATE external_system_definitions d + SET config = (d.config - 'allowed_post_operations') || jsonb_build_object( + 'operation_mode', + CASE WHEN d.provider = 'factory_mes' THEN 'upstream_managed' ELSE 'query' END, + 'operation_policies', + COALESCE( + ( + SELECT jsonb_object_agg(operation_id, 'read') + FROM ( + SELECT jsonb_array_elements_text( + COALESCE(d.config->'allowed_post_operations', '[]'::jsonb) + ) AS operation_id + UNION + SELECT operation_id + FROM (VALUES ('bi_dataset_exec')) defaults(operation_id) + WHERE d.provider = 'factory_mes' + ) policies + ), + '{}'::jsonb + ) + ) + """ + ) + + op.create_table( + "external_system_grants", + sa.Column("definition_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("granted_by", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["definition_id"], + ["external_system_definitions.definition_id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint(["user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["granted_by"], ["users.user_id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("definition_id", "user_id"), + ) + op.execute( + """ + INSERT INTO external_system_grants (definition_id, user_id, granted_by) + SELECT DISTINCT es.definition_id, es.user_id, d.created_by + FROM external_systems es + JOIN external_system_definitions d ON d.definition_id = es.definition_id + WHERE d.access_mode = 'selected' + """ + ) + + op.add_column( + "external_systems", + sa.Column("credential_hint", sa.Text(), server_default="***", nullable=False), + ) + op.add_column( + "external_systems", + sa.Column( + "verified_revision", sa.Integer(), server_default="0", nullable=False + ), + ) + op.add_column("external_systems", sa.Column("last_error", sa.Text(), nullable=True)) + op.execute( + """ + UPDATE external_systems es + SET verified_revision = d.revision + FROM external_system_definitions d + WHERE d.definition_id = es.definition_id + AND es.status IN ('active', 'invalid') + AND es.credentials <> '{}'::jsonb + """ + ) + op.execute("DELETE FROM external_systems WHERE status = 'pending'") + op.drop_column("external_systems", "config") + op.drop_column("external_systems", "connector") + op.drop_column("external_systems", "provider") + op.drop_column("external_system_definitions", "access_mode") + op.create_table( + "external_system_audits", + sa.Column("audit_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("external_system_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("definition_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("definition_revision", sa.Integer(), nullable=False), + sa.Column("event", sa.Text(), nullable=False), + sa.Column("operation_id", sa.Text(), nullable=True), + sa.Column("outcome", sa.Text(), nullable=False), + sa.Column("status_code", sa.Integer(), nullable=True), + sa.Column("duration_ms", sa.Integer(), nullable=False), + sa.Column("response_bytes", sa.BigInteger(), nullable=True), + sa.Column( + "detail", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint(["user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["task_id"], ["tasks.task_id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint( + ["external_system_id"], + ["external_systems.external_system_id"], + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["definition_id"], + ["external_system_definitions.definition_id"], + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("audit_id"), + ) + op.create_index( + "ix_external_system_audits_user_created", + "external_system_audits", + ["user_id", "created_at"], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_external_system_audits_user_created", + table_name="external_system_audits", + ) + op.drop_table("external_system_audits") + op.add_column( + "external_system_definitions", + sa.Column("access_mode", sa.Text(), server_default="selected", nullable=False), + ) + op.execute("UPDATE external_system_definitions SET access_mode = visibility") + op.add_column( + "external_systems", + sa.Column( + "provider", sa.Text(), server_default="generic_openapi", nullable=False + ), + ) + op.add_column( + "external_systems", + sa.Column("connector", sa.Text(), server_default="openapi", nullable=False), + ) + op.add_column( + "external_systems", + sa.Column( + "config", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + ) + op.execute( + """ + UPDATE external_systems es + SET provider = d.provider + FROM external_system_definitions d + WHERE d.definition_id = es.definition_id + """ + ) + op.execute( + """ + INSERT INTO external_systems ( + external_system_id, user_id, definition_id, provider, connector, name, + credentials, config, status + ) + SELECT md5(g.definition_id::text || ':' || g.user_id::text)::uuid, + g.user_id, g.definition_id, d.provider, 'openapi', + d.name, '{}'::jsonb, '{}'::jsonb, 'pending' + FROM external_system_grants g + JOIN external_system_definitions d ON d.definition_id = g.definition_id + WHERE NOT EXISTS ( + SELECT 1 FROM external_systems es + WHERE es.user_id = g.user_id AND es.definition_id = g.definition_id + ) + """ + ) + op.execute( + """ + UPDATE external_system_definitions d + SET config = (d.config - 'operation_policies' - 'operation_mode') || jsonb_build_object( + 'allowed_post_operations', + COALESCE( + ( + SELECT jsonb_agg(operation_id) + FROM jsonb_object_keys( + COALESCE(d.config->'operation_policies', '{}'::jsonb) + ) operation_id + ), + '[]'::jsonb + ) + ) + """ + ) + op.alter_column("external_systems", "provider", server_default=None) + op.alter_column("external_systems", "config", server_default=None) + op.drop_column("external_systems", "last_error") + op.drop_column("external_systems", "verified_revision") + op.drop_column("external_systems", "credential_hint") + op.drop_table("external_system_grants") + op.drop_constraint( + "fk_external_system_definitions_owner_user", + "external_system_definitions", + type_="foreignkey", + ) + op.drop_index( + "uq_external_system_definition_user_name", + table_name="external_system_definitions", + ) + op.drop_index( + "uq_external_system_definition_platform_name", + table_name="external_system_definitions", + ) + op.create_unique_constraint( + "uq_external_system_definition_provider_name", + "external_system_definitions", + ["provider", "name"], + ) + op.drop_column("external_system_definitions", "egress_policy_id") + op.drop_column("external_system_definitions", "review_status") + op.drop_column("external_system_definitions", "trust_level") + op.drop_column("external_system_definitions", "visibility") + op.drop_column("external_system_definitions", "owner_user_id") + op.drop_column("external_system_definitions", "owner_type") + op.drop_column("external_system_definitions", "revision") diff --git a/tests/test_external_system_migration.py b/tests/test_external_system_migration.py new file mode 100644 index 0000000..0fa9ad4 --- /dev/null +++ b/tests/test_external_system_migration.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import importlib +import unittest +from unittest.mock import patch + +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import create_mock_engine +from sqlalchemy.dialects import postgresql + + +class ExternalSystemMigrationTests(unittest.TestCase): + def test_0027_upgrade_compiles_as_postgresql_ddl(self): + statements: list[str] = [] + + def capture(sql, *multiparams, **params): + statements.append(str(sql.compile(dialect=postgresql.dialect()))) + + engine = create_mock_engine("postgresql+psycopg://", capture) + connection = engine.connect() + operations = Operations(MigrationContext.configure(connection)) + migration = importlib.import_module( + "db.migrations.versions.20260807_1000_0027_external_system_governance" + ) + with patch.object(migration, "op", operations): + migration.upgrade() + + rendered = "\n".join(statements) + self.assertIn("external_system_grants", rendered) + self.assertIn("external_system_audits", rendered) + self.assertIn("operation_policies", rendered) + self.assertIn("operation_mode", rendered) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_external_systems.py b/tests/test_external_systems.py index feaa4e9..480b6d5 100644 --- a/tests/test_external_systems.py +++ b/tests/test_external_systems.py @@ -4,10 +4,13 @@ import json import os import sys import tempfile +import time import unittest import uuid +from concurrent.futures import ThreadPoolExecutor from copy import deepcopy from pathlib import Path +from threading import Event, Lock from types import SimpleNamespace from unittest.mock import patch @@ -25,29 +28,101 @@ class ExternalCredentialCryptoTests(unittest.TestCase): def test_roundtrip_uses_ciphertext(self): from core.external_systems.crypto import decrypt_secret, encrypt_secret - with patch.dict(os.environ, {"ZCBOT_CREDENTIAL_MASTER_KEY": "unit-test-key-at-least-32-characters"}, clear=False): + with patch.dict( + os.environ, + {"ZCBOT_CREDENTIAL_MASTER_KEY": "unit-test-key-at-least-32-characters"}, + clear=False, + ): stored = encrypt_secret("mes-password") - self.assertTrue(stored.startswith("v1:")) + self.assertTrue(stored.startswith("v2:primary:")) self.assertNotIn("mes-password", stored) self.assertEqual(decrypt_secret(stored), "mes-password") + def test_ciphertext_is_bound_to_context_and_supports_key_rotation(self): + from core.external_systems.crypto import decrypt_secret, encrypt_secret + + old_key = "old-unit-test-key-at-least-32-characters" + new_key = "new-unit-test-key-at-least-32-characters" + with patch.dict( + os.environ, + { + "ZCBOT_CREDENTIAL_MASTER_KEY": old_key, + "ZCBOT_CREDENTIAL_KEY_ID": "old", + }, + clear=False, + ): + stored = encrypt_secret("secret", aad="user:def:password") + with patch.dict( + os.environ, + { + "ZCBOT_CREDENTIAL_MASTER_KEY": new_key, + "ZCBOT_CREDENTIAL_KEY_ID": "new", + "ZCBOT_CREDENTIAL_PREVIOUS_KEYS": json.dumps({"old": old_key}), + }, + clear=False, + ): + self.assertEqual( + decrypt_secret(stored, aad="user:def:password"), + "secret", + ) + with self.assertRaisesRegex(RuntimeError, "绑定上下文"): + decrypt_secret(stored, aad="other:def:password") + def test_rejects_short_master_key(self): from core.external_systems.crypto import configured, encrypt_secret - with patch.dict(os.environ, {"ZCBOT_CREDENTIAL_MASTER_KEY": "too-short"}, clear=False): + with patch.dict( + os.environ, {"ZCBOT_CREDENTIAL_MASTER_KEY": "too-short"}, clear=False + ): self.assertFalse(configured()) with self.assertRaisesRegex(RuntimeError, "至少需要 32"): encrypt_secret("mes-password") -def _cfg(*, allowed=frozenset(), recommended=()): +class ExternalConnectionRevisionTests(unittest.TestCase): + def test_non_binding_definition_change_keeps_credentials_for_reverify(self): + from core.external_systems.service import _invalidate_connections_for_revision + + connection = SimpleNamespace( + credentials={"token": "ciphertext"}, + credential_hint="***", + status="active", + last_verified_at="old", + last_error=None, + ) + _invalidate_connections_for_revision( + [connection], credential_binding_changed=False + ) + self.assertEqual(connection.credentials, {"token": "ciphertext"}) + self.assertEqual(connection.status, "needs_reverify") + self.assertIsNone(connection.last_verified_at) + + def test_binding_definition_change_clears_credentials(self): + from core.external_systems.service import _invalidate_connections_for_revision + + connection = SimpleNamespace( + credentials={"token": "ciphertext"}, + credential_hint="ab***z", + status="active", + last_verified_at="old", + last_error=None, + ) + _invalidate_connections_for_revision( + [connection], credential_binding_changed=True + ) + self.assertEqual(connection.credentials, {}) + self.assertEqual(connection.credential_hint, "***") + self.assertEqual(connection.status, "needs_credentials") + + +def _cfg(*, allowed=frozenset(), recommended=(), operation_mode="query"): from core.external_systems.factory import FactoryMesConfig return FactoryMesConfig( base_url="https://factory.invalid", openapi_url="https://factory.invalid/swagger.json", login_path="/api/auth/token/", - allowed_post_operations=frozenset(allowed), + operation_policies={operation_id: "read" for operation_id in allowed}, timeout_seconds=5, max_result_bytes=65536, max_total_result_bytes=262144, @@ -55,6 +130,7 @@ def _cfg(*, allowed=frozenset(), recommended=()): verify_tls=True, query_guidance="先查数据集目录", recommended_operation_ids=tuple(recommended), + operation_mode=operation_mode, ) @@ -86,7 +162,12 @@ _SPEC = { "tags": ["quality"], "parameters": [ {"name": "batch", "in": "path", "required": True, "type": "string"}, - {"name": "page_size", "in": "query", "required": False, "type": "integer"}, + { + "name": "page_size", + "in": "query", + "required": False, + "type": "integer", + }, ], } }, @@ -116,6 +197,15 @@ class _Response: self.headers = {"content-type": "application/json"} self.text = json.dumps(payload, ensure_ascii=False) + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def iter_bytes(self): + yield self.text.encode("utf-8") + def json(self): return self._payload @@ -142,29 +232,76 @@ class _Http: self.calls.append((method, url, kwargs)) return _Response(payload={"count": 1, "results": [{"batch": "B/1"}]}) + def stream(self, method, url, **kwargs): + if method.upper() == "POST" and "/auth/" in url: + return self.post(url, **kwargs) + if method.upper() == "GET" and ("swagger" in url or "openapi" in url): + return self.get(url, **kwargs) + return self.request(method.upper(), url, **kwargs) + + +class ExternalRuntimeCacheTests(unittest.TestCase): + def test_lru_defers_client_close_until_active_lease_finishes(self): + from core.external_systems.runtime_cache import ExternalRuntimeCache + + class Client: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + cache = ExternalRuntimeCache(max_entries=2) + first = Client() + second = Client() + third = Client() + with cache.client("first", lambda: first): + with cache.client("second", lambda: second): + pass + with cache.client("third", lambda: third): + pass + self.assertFalse(first.closed) + self.assertTrue(first.closed) + cache.clear() + self.assertTrue(second.closed) + self.assertTrue(third.closed) + class FactoryOpenApiConnectorTests(unittest.TestCase): def setUp(self): - from core.external_systems import factory - factory._SPEC_CACHE.clear() + from core.external_systems.openapi import _SPEC_CACHE + + _SPEC_CACHE.clear() def test_admin_mapping_builds_bounded_runtime_config(self): from core.external_systems.factory import FactoryMesConfig - cfg = FactoryMesConfig.from_mapping({ - "base_url": "https://factory.invalid/", - "openapi_url": "https://factory.invalid/swagger.json", - "allowed_post_operations": "bi_dataset_exec, report_preview", - "timeout_seconds": 999, - "max_result_bytes": 1, - "verify_tls": True, - }) + cfg = FactoryMesConfig.from_mapping( + { + "base_url": "https://factory.invalid/", + "openapi_url": "https://factory.invalid/swagger.json", + "operation_policies": { + "bi_dataset_exec": "read", + "report_preview": "export", + }, + "timeout_seconds": 999, + "max_result_bytes": 1, + "verify_tls": True, + } + ) self.assertEqual(cfg.base_url, "https://factory.invalid") self.assertEqual(cfg.timeout_seconds, 60) self.assertEqual(cfg.max_result_bytes, 4096) self.assertEqual(cfg.max_total_result_bytes, 262144) self.assertEqual(cfg.max_page_size, 200) - self.assertEqual(cfg.allowed_post_operations, {"bi_dataset_exec", "report_preview"}) + self.assertEqual(cfg.operation_mode, "upstream_managed") + self.assertEqual( + cfg.operation_policies, + { + "bi_dataset_exec": "read", + "report_preview": "export", + }, + ) self.assertIn("dataset list", cfg.query_guidance) self.assertEqual( cfg.recommended_operation_ids, @@ -175,24 +312,46 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): from core.external_systems.factory import FactoryMesConfig, FactoryMesError with self.assertRaisesRegex(FactoryMesError, "不能内嵌凭据"): - FactoryMesConfig.from_mapping({ - "base_url": "https://user:secret@factory.invalid", - "openapi_url": "https://factory.invalid/swagger.json", - }) + FactoryMesConfig.from_mapping( + { + "base_url": "https://user:secret@factory.invalid", + "openapi_url": "https://factory.invalid/swagger.json", + } + ) + + def test_admin_mapping_requires_same_origin_openapi_document(self): + from core.external_systems.factory import FactoryMesConfig, FactoryMesError + + with self.assertRaisesRegex(FactoryMesError, "必须与 base_url 同源"): + FactoryMesConfig.from_mapping( + { + "base_url": "https://factory.invalid", + "openapi_url": "https://spec.attacker.invalid/swagger.json", + } + ) def test_generic_api_key_auth_uses_declared_header_without_login(self): - from core.external_systems.openapi import OpenApiClient, OpenApiConfig, _SPEC_CACHE + from core.external_systems.openapi import ( + _SPEC_CACHE, + OpenApiClient, + OpenApiConfig, + ) from core.external_systems.registry import credential_fields, merged_config _SPEC_CACHE.clear() - config = merged_config("generic_openapi", { - "base_url": "https://erp.invalid", - "openapi_url": "https://erp.invalid/openapi.json", - "auth_type": "api_key", - "auth_header_name": "X-ERP-Key", - "auth_header_template": "Key {token}", - }) - client = OpenApiClient({"api_key": "private-key"}, OpenApiConfig.from_mapping(config)) + config = merged_config( + "generic_openapi", + { + "base_url": "https://erp.invalid", + "openapi_url": "https://erp.invalid/openapi.json", + "auth_type": "api_key", + "auth_header_name": "X-ERP-Key", + "auth_header_template": "Key {token}", + }, + ) + client = OpenApiClient( + {"api_key": "private-key"}, OpenApiConfig.from_mapping(config) + ) http = _Http() with patch.object(client, "_client", return_value=http): result = client.test_connection() @@ -200,20 +359,33 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): self.assertFalse(any(call[0] == "POST" for call in http.calls)) get_call = next(call for call in http.calls if call[0] == "GET") self.assertEqual(get_call[2]["headers"], {"X-ERP-Key": "Key private-key"}) - self.assertEqual(credential_fields("generic_openapi", config)[0]["name"], "api_key") + self.assertEqual( + credential_fields("generic_openapi", config)[0]["name"], "api_key" + ) def test_openapi_spec_cache_is_isolated_by_connection_namespace(self): - from core.external_systems.openapi import OpenApiClient, OpenApiConfig, _SPEC_CACHE + from core.external_systems.openapi import ( + _SPEC_CACHE, + OpenApiClient, + OpenApiConfig, + ) from core.external_systems.registry import merged_config _SPEC_CACHE.clear() - config = OpenApiConfig.from_mapping(merged_config("generic_openapi", { - "base_url": "https://erp.invalid", - "openapi_url": "https://erp.invalid/openapi.json", - "auth_type": "bearer_token", - })) + config = OpenApiConfig.from_mapping( + merged_config( + "generic_openapi", + { + "base_url": "https://erp.invalid", + "openapi_url": "https://erp.invalid/openapi.json", + "auth_type": "bearer_token", + }, + ) + ) for namespace in ("definition:user-a", "definition:user-b"): - client = OpenApiClient({"token": namespace}, config, cache_namespace=namespace) + client = OpenApiClient( + {"token": namespace}, config, cache_namespace=namespace + ) http = _Http() with patch.object(client, "_client", return_value=http): client.test_connection() @@ -232,6 +404,174 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): self.assertNotIn("mes-password", rendered) self.assertNotIn("remote-jwt", rendered) + def test_runtime_reuses_http_auth_spec_and_compiled_catalog(self): + from core.external_systems.factory import FactoryMesClient + from core.external_systems.openapi import compile_operation_catalog + + http = _Http() + first = FactoryMesClient("mes-user", "mes-password", _cfg()) + second = FactoryMesClient("mes-user", "mes-password", _cfg()) + with ( + patch.object(first, "_client", return_value=http) as first_factory, + patch.object(second, "_client", return_value=_Http()) as second_factory, + patch( + "core.external_systems.openapi.compile_operation_catalog", + wraps=compile_operation_catalog, + ) as compile_catalog, + ): + first.search("成品检验") + second.search("成品检验") + first.call("qm_ftestwork_read", arguments={"batch": "B1"}) + second.call("qm_ftestwork_read", arguments={"batch": "B1"}) + + self.assertEqual(first_factory.call_count, 1) + self.assertEqual(second_factory.call_count, 0) + self.assertEqual(compile_catalog.call_count, 1) + self.assertEqual( + sum( + 1 + for method, url, _ in http.calls + if method == "POST" and "/auth/" in url + ), + 1, + ) + self.assertEqual( + sum( + 1 + for method, url, _ in http.calls + if method == "GET" and "swagger" in url + ), + 1, + ) + self.assertEqual( + sum( + 1 + for method, url, _ in http.calls + if method == "GET" and "/ftestwork/" in url + ), + 2, + ) + + def test_concurrent_identical_query_is_singleflight_only(self): + from core.external_systems.factory import FactoryMesClient + + class SlowHttp(_Http): + def __init__(self): + super().__init__() + self.query_started = Event() + self.release_query = Event() + self.query_count = 0 + self.query_lock = Lock() + + def request(self, method, url, **kwargs): + if method == "GET" and "/ftestwork/" in url: + with self.query_lock: + self.query_count += 1 + self.query_started.set() + self.release_query.wait(timeout=2) + return super().request(method, url, **kwargs) + + http = SlowHttp() + client = FactoryMesClient("mes-user", "mes-password", _cfg()) + with patch.object(client, "_client", return_value=http): + client.search("成品检验") # 预热认证、规格和 catalog，只测业务请求单飞。 + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit( + client.call, + "qm_ftestwork_read", + {"batch": "B1"}, + ) + self.assertTrue(http.query_started.wait(timeout=1)) + second = executor.submit( + client.call, + "qm_ftestwork_read", + {"batch": "B1"}, + ) + time.sleep(0.05) + http.release_query.set() + first_result = first.result(timeout=2) + second_result = second.result(timeout=2) + + self.assertEqual(http.query_count, 1) + self.assertEqual(first_result, second_result) + self.assertIsNot(first_result, second_result) + + def test_concurrent_cold_search_coalesces_login_and_spec_fetch(self): + from core.external_systems.factory import FactoryMesClient + + class SlowDiscoveryHttp(_Http): + def post(self, url, **kwargs): + response = super().post(url, **kwargs) + time.sleep(0.05) + return response + + def get(self, url, **kwargs): + response = super().get(url, **kwargs) + time.sleep(0.05) + return response + + http = SlowDiscoveryHttp() + first = FactoryMesClient("mes-user", "mes-password", _cfg()) + second = FactoryMesClient("mes-user", "mes-password", _cfg()) + with ( + patch.object(first, "_client", return_value=http), + patch.object(second, "_client", return_value=http), + ThreadPoolExecutor(max_workers=2) as executor, + ): + results = list( + executor.map(lambda client: client.search("成品检验"), (first, second)) + ) + + self.assertTrue(all(result for result in results)) + self.assertEqual( + sum( + 1 + for method, url, _ in http.calls + if method == "POST" and "/auth/" in url + ), + 1, + ) + self.assertEqual( + sum( + 1 + for method, url, _ in http.calls + if method == "GET" and "swagger" in url + ), + 1, + ) + + def test_cached_password_token_refreshes_once_after_401(self): + from core.external_systems.factory import FactoryMesClient + + class RefreshHttp(_Http): + def __init__(self): + super().__init__() + self.business_attempts = 0 + + def request(self, method, url, **kwargs): + if method == "GET" and "/ftestwork/" in url: + self.calls.append((method, url, kwargs)) + self.business_attempts += 1 + if self.business_attempts == 1: + return _Response(payload={"detail": "expired"}, status_code=401) + return super().request(method, url, **kwargs) + + http = RefreshHttp() + client = FactoryMesClient("mes-user", "mes-password", _cfg()) + with patch.object(client, "_client", return_value=http): + result = client.call("qm_ftestwork_read", arguments={"batch": "B1"}) + + self.assertEqual(result["status_code"], 200) + self.assertEqual(http.business_attempts, 2) + self.assertEqual( + sum( + 1 + for method, url, _ in http.calls + if method == "POST" and "/auth/" in url + ), + 2, + ) + def test_search_pins_callable_admin_recommendations_without_keyword_match(self): from core.external_systems.factory import FactoryMesClient @@ -316,6 +656,74 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): "string", ) + def test_catalog_resolves_referenced_header_parameter(self): + from core.external_systems.factory import FactoryMesClient + + spec = { + "openapi": "3.0.0", + "components": { + "parameters": { + "Trace": { + "name": "X-Trace-Id", + "in": "header", + "required": True, + "schema": {"type": "string"}, + } + } + }, + "paths": { + "/quality/": { + "get": { + "operationId": "quality_read", + "parameters": [{"$ref": "#/components/parameters/Trace"}], + } + } + }, + } + cfg = _cfg() + http = _Http() + client = FactoryMesClient("u", "p", cfg) + with ( + patch.object(client, "_client", return_value=http), + patch.object(client, "_fetch_spec", return_value=spec), + ): + client.call("quality_read", arguments={"X-Trace-Id": "trace-1"}) + request = next(call for call in http.calls if call[0] == "GET") + self.assertEqual(request[2]["headers"]["X-Trace-Id"], "trace-1") + + def test_swagger_array_query_uses_declared_collection_format(self): + from core.external_systems.factory import FactoryMesClient + + spec = { + "swagger": "2.0", + "paths": { + "/quality/": { + "get": { + "operationId": "quality_filter", + "parameters": [ + { + "name": "batches", + "in": "query", + "type": "array", + "items": {"type": "string"}, + "collectionFormat": "csv", + } + ], + } + } + }, + } + cfg = _cfg() + http = _Http() + client = FactoryMesClient("u", "p", cfg) + with ( + patch.object(client, "_client", return_value=http), + patch.object(client, "_fetch_spec", return_value=spec), + ): + client.call("quality_filter", arguments={"batches": ["B1", "B2"]}) + request = next(call for call in http.calls if call[0] == "GET") + self.assertEqual(request[2]["params"]["batches"], "B1,B2") + def test_get_call_resolves_encoded_path_and_query(self): from core.external_systems.factory import FactoryMesClient @@ -326,7 +734,9 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): "qm_ftestwork_read", arguments={"batch": "B/1", "page_size": 50}, ) - method, url, kwargs = [call for call in http.calls if call[0] == "GET" and "/api/" in call[1]][0] + method, url, kwargs = [ + call for call in http.calls if call[0] == "GET" and "/api/" in call[1] + ][0] self.assertEqual(method, "GET") self.assertIn("B%2F1", url) self.assertEqual(kwargs["params"], {"page_size": 50}) @@ -336,14 +746,22 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): from core.external_systems.factory import FactoryMesClient, FactoryMesError spec = deepcopy(_SPEC) - spec["paths"]["/api/qm/ftestwork/{batch}/"]["get"]["parameters"].extend([ - {"name": "page", "in": "query", "required": False, "type": "integer"}, - {"name": "pageoff", "in": "query", "required": False, "type": "boolean"}, - ]) + spec["paths"]["/api/qm/ftestwork/{batch}/"]["get"]["parameters"].extend( + [ + {"name": "page", "in": "query", "required": False, "type": "integer"}, + { + "name": "pageoff", + "in": "query", + "required": False, + "type": "boolean", + }, + ] + ) http = _Http() client = FactoryMesClient("u", "p", _cfg()) - with patch.object(client, "_client", return_value=http), patch.object( - client, "_fetch_spec", return_value=spec + with ( + patch.object(client, "_client", return_value=http), + patch.object(client, "_fetch_spec", return_value=spec), ): client.call( "qm_ftestwork_read", @@ -352,9 +770,11 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): request = next(call for call in http.calls if call[0] == "GET") self.assertEqual(request[2]["params"]["page_size"], 200) - with patch.object(client, "authenticate", return_value="jwt"), patch.object( - client, "_fetch_spec", return_value=spec - ), self.assertRaisesRegex(FactoryMesError, "不允许 page=0"): + with ( + patch.object(client, "authenticate", return_value="jwt"), + patch.object(client, "_fetch_spec", return_value=spec), + self.assertRaisesRegex(FactoryMesError, "不允许 page=0"), + ): client.call( "qm_ftestwork_read", arguments={"batch": "B1", "page": 0}, @@ -366,13 +786,13 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): spec = deepcopy(_SPEC) spec["basePath"] = "/api" spec["paths"] = { - path.removeprefix("/api"): value - for path, value in spec["paths"].items() + path.removeprefix("/api"): value for path, value in spec["paths"].items() } http = _Http() client = FactoryMesClient("u", "p", _cfg()) - with patch.object(client, "_client", return_value=http), patch.object( - client, "_fetch_spec", return_value=spec + with ( + patch.object(client, "_client", return_value=http), + patch.object(client, "_fetch_spec", return_value=spec), ): client.call("qm_ftestwork_read", arguments={"batch": "B1"}) request = next(call for call in http.calls if call[0] == "GET") @@ -394,8 +814,9 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): spec = {**deepcopy(_SPEC), "basePath": "/api"} http = _Http() client = FactoryMesClient("u", "p", _cfg()) - with patch.object(client, "_client", return_value=http), patch.object( - client, "_fetch_spec", return_value=spec + with ( + patch.object(client, "_client", return_value=http), + patch.object(client, "_fetch_spec", return_value=spec), ): client.call("qm_ftestwork_read", arguments={"batch": "B1"}) request = next(call for call in http.calls if call[0] == "GET") @@ -443,8 +864,9 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): from core.external_systems.factory import FactoryMesClient, FactoryMesError denied = FactoryMesClient("u", "p", _cfg()) - with patch.object(denied, "authenticate", return_value="jwt"), patch.object( - denied, "_fetch_spec", return_value=_SPEC + with ( + patch.object(denied, "authenticate", return_value="jwt"), + patch.object(denied, "_fetch_spec", return_value=_SPEC), ): with self.assertRaisesRegex(FactoryMesError, "只读调用范围"): denied.call("bi_dataset_exec", arguments={"code": "x", "payload": {}}) @@ -456,7 +878,9 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): "bi_dataset_exec", arguments={"code": "yield", "payload": {"query": {"month": "2026-08"}}}, ) - request = [call for call in http.calls if call[0] == "POST" and "/dataset/" in call[1]][0] + request = [ + call for call in http.calls if call[0] == "POST" and "/dataset/" in call[1] + ][0] self.assertEqual(request[2]["json"], {"query": {"month": "2026-08"}}) self.assertFalse(result["truncated"]) @@ -471,10 +895,85 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): arguments={"code": "quality"}, body={"query": {"batch": "B-1"}}, ) - request = [call for call in http.calls if call[0] == "POST" and "/dataset/" in call[1]][0] + request = [ + call for call in http.calls if call[0] == "POST" and "/dataset/" in call[1] + ][0] self.assertEqual(request[2]["json"], {"query": {"batch": "B-1"}}) self.assertFalse(result["truncated"]) + def test_upstream_managed_mode_allows_declared_write_method(self): + from core.external_systems.factory import FactoryMesClient + + spec = deepcopy(_SPEC) + spec["paths"]["/api/items/{item_id}/"] = { + "put": { + "operationId": "item_update", + "parameters": [ + { + "name": "item_id", + "in": "path", + "required": True, + "type": "string", + }, + { + "name": "payload", + "in": "body", + "required": True, + "schema": { + "type": "object", + "required": ["name"], + "properties": {"name": {"type": "string"}}, + }, + }, + ], + } + } + http = _Http() + client = FactoryMesClient("u", "p", _cfg(operation_mode="upstream_managed")) + with ( + patch.object(client, "_client", return_value=http), + patch.object(client, "authenticate", return_value={}), + patch.object(client, "_fetch_spec", return_value=spec), + ): + result = client.call( + "item_update", + arguments={"item_id": "A/B"}, + body={"name": "updated"}, + ) + request = next(call for call in http.calls if call[0] == "PUT") + self.assertIn("/api/items/A%2FB/", request[1]) + self.assertEqual(request[2]["json"], {"name": "updated"}) + self.assertEqual(result["status_code"], 200) + + def test_query_mode_still_rejects_declared_write_method(self): + from core.external_systems.factory import FactoryMesClient, FactoryMesError + + spec = { + "swagger": "2.0", + "paths": { + "/api/items/{item_id}/": { + "delete": { + "operationId": "item_delete", + "parameters": [ + { + "name": "item_id", + "in": "path", + "required": True, + "type": "string", + } + ], + } + } + }, + } + client = FactoryMesClient("u", "p", _cfg()) + with ( + patch.object(client, "authenticate", return_value={}), + patch.object(client, "_fetch_spec", return_value=spec), + ): + with self.assertRaisesRegex(FactoryMesError, "只读调用范围"): + client.call("item_delete", arguments={"item_id": "A-1"}) + def test_call_preserves_payload_larger_than_inline_limit(self): from core.external_systems.factory import FactoryMesClient @@ -483,9 +982,7 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): self.calls.append((method, url, kwargs)) return _Response(payload={"rows": "x" * 70000}) - client = FactoryMesClient( - "u", "p", _cfg(allowed={"bi_dataset_exec"}) - ) + client = FactoryMesClient("u", "p", _cfg(allowed={"bi_dataset_exec"})) with patch.object(client, "_client", return_value=LargeHttp()): result = client.call( "bi_dataset_exec", @@ -496,6 +993,25 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): self.assertGreater(result["response_bytes"], client.cfg.max_result_bytes) self.assertFalse(result["truncated"]) + def test_call_stops_stream_when_download_limit_is_exceeded(self): + from core.external_systems.factory import FactoryMesClient, FactoryMesError + from core.external_systems.results import MAX_STORED_RESULT_BYTES + + class OversizedResponse(_Response): + def iter_bytes(self): + yield b"x" * MAX_STORED_RESULT_BYTES + yield b"x" + + class OversizedHttp(_Http): + def request(self, method, url, **kwargs): + self.calls.append((method, url, kwargs)) + return OversizedResponse(payload={}) + + client = FactoryMesClient("u", "p", _cfg()) + with patch.object(client, "_client", return_value=OversizedHttp()): + with self.assertRaisesRegex(FactoryMesError, "安全下载上限"): + client.call("qm_ftestwork_read", arguments={"batch": "B1"}) + def test_call_surfaces_sanitized_upstream_error_detail(self): from core.external_systems.factory import FactoryMesClient, FactoryMesError @@ -516,7 +1032,7 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): client.call( "bi_dataset_exec", arguments={"code": "yield"}, - body={"wrong": "shape"}, + body={"query": {}}, ) message = str(raised.exception) self.assertIn("This field is required.", message) @@ -527,8 +1043,9 @@ class FactoryOpenApiConnectorTests(unittest.TestCase): from core.external_systems.factory import FactoryMesClient, FactoryMesError client = FactoryMesClient("u", "p", _cfg()) - with patch.object(client, "authenticate", return_value="jwt"), patch.object( - client, "_fetch_spec", return_value=_SPEC + with ( + patch.object(client, "authenticate", return_value="jwt"), + patch.object(client, "_fetch_spec", return_value=_SPEC), ): with self.assertRaisesRegex(FactoryMesError, "接口定义之外"): client.call( @@ -544,11 +1061,13 @@ class ExternalSystemToolSafetyTests(unittest.TestCase): uid = uuid.uuid4() with patch( "tools.external_systems.list_external_systems", - return_value=[{ - "external_system_id": str(uuid.uuid4()), - "status": "active", - "username_masked": "me***r", - }], + return_value=[ + { + "external_system_id": str(uuid.uuid4()), + "status": "active", + "username_masked": "me***r", + } + ], ) as listed: output = ExternalSystemListTool(uid).execute() listed.assert_called_once_with(uid) @@ -594,9 +1113,12 @@ class ExternalSystemToolSafetyTests(unittest.TestCase): "data": [{"id": index, "noise": "x" * 100} for index in range(20)], }, ) - with tempfile.TemporaryDirectory() as tmp, patch( - "tools.external_systems._row_and_client", - return_value=(SimpleNamespace(), client), + with ( + tempfile.TemporaryDirectory() as tmp, + patch( + "tools.external_systems._row_and_client", + return_value=(SimpleNamespace(), client), + ), ): call_tool = ExternalSystemCallTool( uid, @@ -624,13 +1146,15 @@ class ExternalSystemToolSafetyTests(unittest.TestCase): result_budget=budget, base_dir=Path(tmp), ) - page = json.loads(read_tool.execute( - spilled["result_ref"], - json_pointer="/data", - offset=5, - limit=2, - fields=["id"], - )) + page = json.loads( + read_tool.execute( + spilled["result_ref"], + json_pointer="/data", + offset=5, + limit=2, + fields=["id"], + ) + ) other_task = ExternalSystemResultReadTool( uid, task_id=uuid.uuid4(), @@ -654,7 +1178,9 @@ class ExternalSystemToolSafetyTests(unittest.TestCase): self.assertEqual(page["data"], [{"id": 5}, {"id": 6}]) self.assertTrue(page["has_more"]) self.assertIn("不存在或已过期", cross_task) - self.assertEqual(exported.artifacts[0].path, "data/external/detail_snapshot.json") + self.assertEqual( + exported.artifacts[0].path, "data/external/detail_snapshot.json" + ) self.assertEqual(export_payload["_zcbot"]["result_ref"], spilled["result_ref"]) self.assertEqual( export_payload["_zcbot"]["provenance"]["operation_id"], diff --git a/tests/test_web_routes_nodb.py b/tests/test_web_routes_nodb.py index a7ba0d8..16480cb 100644 --- a/tests/test_web_routes_nodb.py +++ b/tests/test_web_routes_nodb.py @@ -158,14 +158,20 @@ class ExternalSystemRoutesTests(unittest.TestCase): json={ "definition_id": str(definition_id), "name": "Factory MES", - "username": "mes-user", - "password": "secret", + "credentials": { + "username": "mes-user", + "password": "secret", + }, }, ) self.assertEqual(r.status_code, 201) self.assertEqual(r.json(), created) self.assertEqual(create.call_args.args[0], _UID) self.assertEqual(create.call_args.kwargs["definition_id"], definition_id) + self.assertEqual( + create.call_args.kwargs["credentials"], + {"username": "mes-user", "password": "secret"}, + ) def test_create_accepts_dynamic_credentials(self): created = {"external_system_id": str(uuid.uuid4()), "username_masked": "***"} diff --git a/tools/external_systems.py b/tools/external_systems.py index f91f641..950aa28 100644 --- a/tools/external_systems.py +++ b/tools/external_systems.py @@ -1,8 +1,10 @@ """Host-side 外部系统元工具；凭据只在 control plane 解密。""" + from __future__ import annotations import json import re +import time from datetime import datetime, timezone from pathlib import Path from uuid import UUID, uuid4 @@ -43,8 +45,8 @@ def _row_and_client(user_id: UUID, raw_system_id: str): class ExternalSystemListTool(Tool): name = "external_system_list" description = ( - "列出当前用户已连接且可供查询的外部系统。返回 system_id、管理员配置的查询规划提示" - "和推荐 operationId；查询外部系统前先调用并遵循对应提示。凭据永不返回。" + "列出当前用户已连接的外部系统。返回 system_id、执行模式、管理员配置的查询规划提示" + "和推荐 operationId；使用外部系统前先调用并遵循对应提示。凭据永不返回。" ) parameters = {"type": "object", "properties": {}} @@ -53,7 +55,9 @@ class ExternalSystemListTool(Tool): self.user_id = user_id def execute(self, **kwargs) -> str: - systems = [x for x in list_external_systems(self.user_id) if x["status"] == "active"] + systems = [ + x for x in list_external_systems(self.user_id) if x["status"] == "active" + ] return _json({"systems": systems}) @@ -67,7 +71,10 @@ class ExternalSystemSearchTool(Tool): parameters = { "type": "object", "properties": { - "system_id": {"type": "string", "description": "external_system_list 返回的 UUID"}, + "system_id": { + "type": "string", + "description": "external_system_list 返回的 UUID", + }, "query": {"type": "string", "description": "业务对象、字段或动作关键词"}, "limit": {"type": "integer", "minimum": 1, "maximum": 30, "default": 12}, }, @@ -82,12 +89,16 @@ class ExternalSystemSearchTool(Tool): try: _, client = _row_and_client(self.user_id, system_id) results = client.search(query, limit=limit) - return _json({ - "query_guidance": client.cfg.query_guidance, - "recommended_operation_ids": list(client.cfg.recommended_operation_ids), - "results": results, - "count": len(results), - }) + return _json( + { + "query_guidance": client.cfg.query_guidance, + "recommended_operation_ids": list( + client.cfg.recommended_operation_ids + ), + "results": results, + "count": len(results), + } + ) except (ExternalSystemError, FactoryMesError) as exc: print(f"[WARN] external system search failed: {type(exc).__name__}") return f"[Error] {exc}" @@ -96,14 +107,18 @@ class ExternalSystemSearchTool(Tool): class ExternalSystemCallTool(Tool): name = "external_system_call" description = ( - "调用已连接外部系统的受控只读 OpenAPI operation。必须使用 search 返回的 operation_id；" - "不接受 URL。GET/HEAD 默认可用，POST 仅限管理员声明的只读 operation。" + "调用已连接外部系统中 search 返回的 OpenAPI operation，不接受 URL。" + "query 模式仅开放 GET/HEAD 和管理员声明的只读 POST；upstream_managed 模式" + "开放可信规格中的全部方法并由上游按当前用户凭据鉴权，非查询操作仅在用户明确要求时调用。" "大响应会完整保存并返回 result_ref，使用 external_system_result_read 分段读取。" ) parameters = { "type": "object", "properties": { - "system_id": {"type": "string", "description": "external_system_list 返回的 UUID"}, + "system_id": { + "type": "string", + "description": "external_system_list 返回的 UUID", + }, "operation_id": {"type": "string"}, "arguments": { "type": "object", @@ -113,9 +128,9 @@ class ExternalSystemCallTool(Tool): "body": { "type": "object", "description": ( - "仅对管理员放行的只读 POST 操作提供原始 JSON 请求体；" + "为规格声明了 JSON 请求体的操作提供原始 body；" "严格遵循 search 返回的 body.schema，不要按 Swagger body 参数名再包一层" - ) + ), }, }, "required": ["system_id", "operation_id"], @@ -127,15 +142,58 @@ class ExternalSystemCallTool(Tool): *, task_id: UUID | str = "default", result_budget: dict[str, int] | None = None, + audit_recorder=None, **kwargs, ): super().__init__(**kwargs) self.user_id = user_id + self.task_id = str(task_id) + self._audit_recorder = audit_recorder self._result_bytes = result_budget if result_budget is not None else {} self._result_store = ExternalResultStore( self.user_root or self.base_dir, str(task_id) ) + def _audit( + self, + *, + row=None, + system_id: str, + operation_id: str, + outcome: str, + started: float, + status_code: int | None = None, + response_bytes: int | None = None, + error_type: str | None = None, + ) -> None: + if self._audit_recorder is None: + return + try: + task_id = UUID(self.task_id) + except (TypeError, ValueError): + task_id = None + try: + external_system_id = UUID(str(system_id)) + except (TypeError, ValueError): + external_system_id = None + try: + self._audit_recorder( + user_id=self.user_id, + task_id=task_id, + external_system_id=external_system_id, + definition_id=getattr(row, "definition_id", None), + definition_revision=getattr(row, "verified_revision", 0), + event="call", + operation_id=operation_id, + outcome=outcome, + status_code=status_code, + duration_ms=round((time.perf_counter() - started) * 1000), + response_bytes=response_bytes, + detail={"error_type": error_type} if error_type else {}, + ) + except Exception as exc: + print(f"[WARN] external system audit failed: {type(exc).__name__}") + def _bounded_output( self, system_id: str, @@ -192,8 +250,10 @@ class ExternalSystemCallTool(Tool): body=None, **kwargs, ) -> str: + started = time.perf_counter() + row = None try: - _, client = _row_and_client(self.user_id, system_id) + row, client = _row_and_client(self.user_id, system_id) used = self._result_bytes.get(system_id, 0) if used >= client.cfg.max_total_result_bytes: return ( @@ -201,7 +261,7 @@ class ExternalSystemCallTool(Tool): "请在下一轮继续查询，或读取之前返回的 result_ref。" ) result = client.call(operation_id, arguments=arguments, body=body) - return self._bounded_output( + output = self._bounded_output( system_id, result, per_result_limit=client.cfg.max_result_bytes, @@ -213,7 +273,25 @@ class ExternalSystemCallTool(Tool): "queried_at": datetime.now(timezone.utc).isoformat(), }, ) + self._audit( + row=row, + system_id=system_id, + operation_id=operation_id, + outcome="ok", + started=started, + status_code=result.get("status_code"), + response_bytes=result.get("response_bytes"), + ) + return output except (ExternalSystemError, FactoryMesError, ExternalResultError) as exc: + self._audit( + row=row, + system_id=system_id, + operation_id=operation_id, + outcome="error", + started=started, + error_type=type(exc).__name__, + ) print(f"[WARN] external system call failed: {type(exc).__name__}") return f"[Error] {exc}" @@ -287,21 +365,25 @@ class ExternalSystemResultReadTool(Tool): inline_limit = min(client.cfg.max_result_bytes, remaining) if len(output.encode("utf-8")) > inline_limit: preview, reads = build_result_preview(response) - output = _json({ - "result_ref": result_ref, - "json_pointer": json_pointer, - "inline_complete": False, - "preview": preview, - "available_reads": reads, - "hint": "减小 limit、指定更深的 json_pointer 或使用 fields 投影", - }) + output = _json( + { + "result_ref": result_ref, + "json_pointer": json_pointer, + "inline_complete": False, + "preview": preview, + "available_reads": reads, + "hint": "减小 limit、指定更深的 json_pointer 或使用 fields 投影", + } + ) if len(output.encode("utf-8")) > inline_limit: - output = _json({ - "result_ref": result_ref, - "json_pointer": json_pointer, - "inline_complete": False, - "hint": "当前分段仍过大；请减小 limit、指定更深的 json_pointer 或使用 fields", - }) + output = _json( + { + "result_ref": result_ref, + "json_pointer": json_pointer, + "inline_complete": False, + "hint": "当前分段仍过大；请减小 limit、指定更深的 json_pointer 或使用 fields", + } + ) if len(output.encode("utf-8")) > remaining: self._result_bytes[system_id] = client.cfg.max_total_result_bytes return "[Error] 本轮外部系统内联返回量已达上限，请在下一轮继续读取。" @@ -368,10 +450,7 @@ class ExternalSystemResultExportTool(Tool): filename = filename.strip() if not filename.lower().endswith(".json"): filename += ".json" - if ( - filename in {".", ".."} - or not re.fullmatch(r"[\w.-]+", filename) - ): + if filename in {".", ".."} or not re.fullmatch(r"[\w.-]+", filename): raise ExternalResultError("filename 包含非法路径字符") target = self._working_dir / "data" / "external" / filename if target.exists(): @@ -403,7 +482,9 @@ class ExternalSystemResultExportTool(Tool): working_dir=self.base_dir, user_root=self._user_root, ) - content = f"saved: {rel}\n完整外部系统结果已持久导出；该文件不受缓存 TTL 影响。" + content = ( + f"saved: {rel}\n完整外部系统结果已持久导出；该文件不受缓存 TTL 影响。" + ) return ToolExecutionResult(content, artifacts=(ArtifactRef(path=rel),)) except ( ExternalSystemError, diff --git a/web/admin.py b/web/admin.py index f579625..8a6744c 100644 --- a/web/admin.py +++ b/web/admin.py @@ -194,7 +194,8 @@ class ExternalSystemDefinitionRequest(BaseModel): token_field: str = "access" auth_header_name: str = "Authorization" auth_header_template: str = "Bearer {token}" - allowed_post_operations: list[str] = Field(default_factory=list) + operation_mode: str | None = None + operation_policies: dict[str, str] = Field(default_factory=dict) timeout_seconds: float = 15 max_result_bytes: int = 65536 max_total_result_bytes: int = 262144 @@ -205,12 +206,12 @@ class ExternalSystemDefinitionRequest(BaseModel): default_factory=lambda: ["bi_dataset_list", "bi_dataset_exec"] ) enabled: bool = True - access_mode: str = "selected" + visibility: str = "selected" selected_user_ids: list[UUID] = Field(default_factory=list) def _external_definition_config(body: ExternalSystemDefinitionRequest) -> dict[str, Any]: - return { + config = { "base_url": body.base_url, "openapi_url": body.openapi_url, "login_path": body.login_path, @@ -220,7 +221,7 @@ def _external_definition_config(body: ExternalSystemDefinitionRequest) -> dict[s "token_field": body.token_field, "auth_header_name": body.auth_header_name, "auth_header_template": body.auth_header_template, - "allowed_post_operations": body.allowed_post_operations, + "operation_policies": body.operation_policies, "timeout_seconds": body.timeout_seconds, "max_result_bytes": body.max_result_bytes, "max_total_result_bytes": body.max_total_result_bytes, @@ -229,6 +230,9 @@ def _external_definition_config(body: ExternalSystemDefinitionRequest) -> dict[s "query_guidance": body.query_guidance, "recommended_operation_ids": body.recommended_operation_ids, } + if body.operation_mode is not None: + config["operation_mode"] = body.operation_mode + return config def register_admin_routes(app: FastAPI, require_admin) -> None: @@ -288,7 +292,7 @@ def register_admin_routes(app: FastAPI, require_admin) -> None: name=body.name, config=_external_definition_config(body), enabled=body.enabled, - access_mode=body.access_mode, + visibility=body.visibility, selected_user_ids=body.selected_user_ids, ) except ExternalSystemError as exc: @@ -310,7 +314,7 @@ def register_admin_routes(app: FastAPI, require_admin) -> None: name=body.name, config=_external_definition_config(body), enabled=body.enabled, - access_mode=body.access_mode, + visibility=body.visibility, selected_user_ids=body.selected_user_ids, ) except ExternalSystemError as exc: diff --git a/web/routers/external_systems.py b/web/routers/external_systems.py index 7532244..a22810d 100644 --- a/web/routers/external_systems.py +++ b/web/routers/external_systems.py @@ -54,9 +54,7 @@ def register_external_system_routes(app, *, require_user) -> None: user_id, definition_id=body.definition_id, name=body.name, - credentials=body.credentials or None, - username=body.username, - password=body.password, + credentials=body.credentials, ) except ExternalSystemError as exc: raise _bad_request(exc) @@ -71,9 +69,7 @@ def register_external_system_routes(app, *, require_user) -> None: return update_external_system_credentials( user_id, _uuid(system_id), - credentials=body.credentials or None, - username=body.username, - password=body.password, + credentials=body.credentials, ) except ExternalSystemError as exc: raise _bad_request(exc) diff --git a/web/schemas.py b/web/schemas.py index eaec142..61da74b 100644 --- a/web/schemas.py +++ b/web/schemas.py @@ -103,11 +103,7 @@ class ExternalSystemCreateRequest(BaseModel): definition_id: UUID name: str = "" credentials: dict[str, str] = Field(default_factory=dict) - username: str = "" # deprecated: 兼容旧版 Factory 客户端 - password: str = "" # deprecated: 兼容旧版 Factory 客户端 class ExternalSystemCredentialsRequest(BaseModel): credentials: dict[str, str] = Field(default_factory=dict) - username: str = "" # deprecated - password: str = "" # deprecated diff --git a/web/static/js/admin.js b/web/static/js/admin.js index 9e31d88..2c8404c 100644 --- a/web/static/js/admin.js +++ b/web/static/js/admin.js @@ -168,9 +168,9 @@ function renderExternalDefinitions() { const cfg = r.config || {}; return `` + `${escapeHtml(r.name)} ${escapeHtml(r.provider_title || r.provider)}${r.enabled ? "" : ' 停用'}` - + ` ${r.access_mode === "all" ? "全部用户" : `指定 ${((r.selected_user_ids || []).length)} 人`}` + + ` ${r.visibility === "organization" ? "全部用户" : `指定 ${((r.selected_user_ids || []).length)} 人`}` + `${escapeHtml(r.host || cfg.base_url || "—")}` - + `${(cfg.allowed_post_operations || []).length}` + + `${cfg.operation_mode === "upstream_managed" ? "上游托管" : `查询（${Object.keys(cfg.operation_policies || {}).length} 个 POST）`}` + ` ` + ``; }).join("") || `尚未配置外部系统`; @@ -179,6 +179,7 @@ function renderExternalDefinitions() { + `
` + `` + `` + + `` + `` + `` + `` @@ -188,7 +189,7 @@ function renderExternalDefinitions() { + `` + `` + `` - + `` + + `` + `` + `` + `` + `` - + `` + + `` + `` + `
` + `
` - + `
` + + `
系统主机只读 POST操作
` + `${rows}
系统主机执行模式操作
`; $("ext-admin-form").onsubmit = saveExternalDefinition; @@ -212,11 +213,13 @@ function renderExternalDefinitions() { $("exa-guidance-edit").onclick = editExternalGuidance; $("exa-provider").onchange = applyExternalProviderDefaults; $("exa-auth").onchange = applyExternalAuthDefaults; + $("exa-operation-mode").onchange = updateExternalOperationMode; $("exa-access").onchange = () => { $("exa-users-wrap").hidden = $("exa-access").value !== "selected"; }; $("exa-cancel").onclick = () => { externalEditingId = ""; renderExternalDefinitions(); }; updateExternalAuthForm(); + updateExternalOperationMode(); $("s-external").onclick = (e) => { const tr = e.target.closest("tr[data-definition-id]"); if (!tr) return; @@ -240,13 +243,24 @@ function updateExternalAuthForm() { function applyExternalProviderDefaults() { const factory = $("exa-provider").value === "factory_mes"; $("exa-auth").value = "password_jwt"; + $("exa-operation-mode").value = factory ? "upstream_managed" : "query"; + $("exa-operations").value = factory ? "bi_dataset_exec" : ""; $("exa-recommended").value = factory ? "bi_dataset_list, bi_dataset_exec" : ""; $("exa-guidance").value = factory ? DEFAULT_EXTERNAL_QUERY_GUIDANCE : ""; $("exa-name").placeholder = factory ? "Factory MES" : "ERP / LIMS / 其他系统"; applyExternalAuthDefaults(); + updateExternalOperationMode(); updateExternalGuidanceSummary(); } +function updateExternalOperationMode() { + const managed = $("exa-operation-mode").value === "upstream_managed"; + $("exa-operations-wrap").hidden = managed; + $("exa-operation-mode-hint").textContent = managed + ? "规格中声明的 POST/PUT/PATCH/DELETE 等操作均可被调用，Factory 使用当前用户凭据做最终鉴权。" + : "GET/HEAD 默认开放；只有这里列出的只读 POST 可以调用。"; +} + function applyExternalAuthDefaults() { const auth = $("exa-auth").value; $("exa-auth-header").value = auth === "api_key" ? "X-API-Key" : "Authorization"; @@ -292,7 +306,12 @@ function fillExternalDefinition(row) { $("exa-auth-header").value = cfg.auth_header_name || "Authorization"; $("exa-auth-template").value = cfg.auth_header_template || "Bearer {token}"; updateExternalAuthForm(); - $("exa-post").value = (cfg.allowed_post_operations || []).join(", "); + $("exa-operation-mode").value = cfg.operation_mode + || (row.provider === "factory_mes" ? "upstream_managed" : "query"); + updateExternalOperationMode(); + $("exa-operations").value = Object.entries(cfg.operation_policies || {}) + .filter(([, policy]) => policy === "read" || policy === "export") + .map(([operationId]) => operationId).join(", "); $("exa-recommended").value = ( cfg.recommended_operation_ids || ["bi_dataset_list", "bi_dataset_exec"] ).join(", "); @@ -300,7 +319,7 @@ function fillExternalDefinition(row) { updateExternalGuidanceSummary(); $("exa-tls").checked = cfg.verify_tls !== false; $("exa-enabled").checked = row.enabled !== false; - $("exa-access").value = row.access_mode || "selected"; + $("exa-access").value = row.visibility || "selected"; const selected = new Set(row.selected_user_ids || []); Array.from($("exa-users").options).forEach(o => { o.selected = selected.has(o.value); }); $("exa-users-wrap").hidden = $("exa-access").value !== "selected"; @@ -322,12 +341,16 @@ async function saveExternalDefinition(e) { token_field: $("exa-token-field").value.trim() || "access", auth_header_name: $("exa-auth-header").value.trim() || "Authorization", auth_header_template: $("exa-auth-template").value || "Bearer {token}", - allowed_post_operations: $("exa-post").value.split(",").map(x => x.trim()).filter(Boolean), + operation_mode: $("exa-operation-mode").value, + operation_policies: Object.fromEntries( + $("exa-operations").value.split(",").map(x => x.trim()).filter(Boolean) + .map(operationId => [operationId, "read"]), + ), recommended_operation_ids: $("exa-recommended").value.split(",").map(x => x.trim()).filter(Boolean), query_guidance: $("exa-guidance").value.trim(), verify_tls: $("exa-tls").checked, enabled: $("exa-enabled").checked, - access_mode: $("exa-access").value, + visibility: $("exa-access").value, selected_user_ids: Array.from($("exa-users").selectedOptions).map(o => o.value), }; const current = externalDefinitions.find(x => x.definition_id === externalEditingId); diff --git a/web/static/js/external_systems.js b/web/static/js/external_systems.js index 40b3db2..6cdff27 100644 --- a/web/static/js/external_systems.js +++ b/web/static/js/external_systems.js @@ -48,7 +48,11 @@ async function openExternalSystemsModal() { function cardHtml(item) { const good = item.status === "active"; - const badge = good ? "已连接" : (item.status === "disabled" ? "系统已停用" : "需更新凭据"); + const badge = good + ? "已连接" + : (item.status === "disabled" + ? "系统已停用" + : (item.status === "needs_reverify" ? "需重新验证" : "需更新凭据")); const checked = item.last_verified_at ? fmtTime(item.last_verified_at) : "尚未验证"; return `
${escapeHtml(item.name)}${badge}