"""外部系统目录、用户可见授权和密文连接的持久化服务层。""" from __future__ import annotations from collections.abc import Sequence from datetime import datetime, timezone from typing import Any, Optional from urllib.parse import urlparse from uuid import UUID from sqlalchemy import delete, exists, or_, select from sqlalchemy.exc import IntegrityError from core.storage import session_scope from core.storage.models import ( ExternalSystem, ExternalSystemAudit, ExternalSystemDefinition, ExternalSystemGrant, User, ) from .crypto import configured as crypto_configured from .crypto import decrypt_secret, encrypt_secret, mask_username from .openapi import OpenApiClient, OpenApiConfig, OpenApiError from .registry import credential_fields, get_provider, merged_config, provider_specs class ExternalSystemError(RuntimeError): pass _REVERIFY_KEYS = frozenset( { "base_url", "openapi_url", "login_path", "auth_type", "username_field", "password_field", "token_field", "auth_header_name", "auth_header_template", "verify_tls", } ) def _runtime_config(provider: str, data: dict[str, Any]) -> OpenApiConfig: try: return OpenApiConfig.from_mapping(merged_config(provider, data)) except (OpenApiError, TypeError, ValueError) as exc: raise ExternalSystemError(str(exc)) from exc def _normalized_config(provider: str, data: dict[str, Any]) -> dict[str, Any]: cfg = _runtime_config(provider, data) return { "base_url": cfg.base_url, "openapi_url": cfg.openapi_url, "login_path": cfg.login_path, "operation_mode": cfg.operation_mode, "operation_policies": dict(sorted(cfg.operation_policies.items())), "timeout_seconds": cfg.timeout_seconds, "max_result_bytes": cfg.max_result_bytes, "max_total_result_bytes": cfg.max_total_result_bytes, "max_page_size": cfg.max_page_size, "verify_tls": cfg.verify_tls, "query_guidance": cfg.query_guidance, "recommended_operation_ids": list(cfg.recommended_operation_ids), "auth_type": cfg.auth_type, **cfg.auth_config, } def _classify_definition_config_change( provider: str, old_data: dict[str, Any], new_data: dict[str, Any], ) -> tuple[dict[str, Any], dict[str, Any], frozenset[str], str]: """按运行语义比较配置,避免旧 JSON 缺省字段被误判成认证变化。""" old_config = _normalized_config(provider, old_data) new_config = _normalized_config(provider, new_data) changed = frozenset( key for key in old_config.keys() | new_config.keys() if old_config.get(key) != new_config.get(key) ) if not changed: impact = "none" elif changed & _REVERIFY_KEYS: impact = "reverify" else: impact = "runtime" return old_config, new_config, changed, impact def _definition_view( row: ExternalSystemDefinition, *, include_config: bool ) -> dict[str, Any]: config = row.config or {} result = { "definition_id": str(row.definition_id), "provider": row.provider, "provider_title": get_provider(row.provider).title, "connector": get_provider(row.provider).connector, "name": row.name, "enabled": row.enabled, "revision": row.revision, "owner_type": row.owner_type, "owner_user_id": str(row.owner_user_id) if row.owner_user_id else None, "visibility": row.visibility, "trust_level": row.trust_level, "review_status": row.review_status, "egress_policy_id": row.egress_policy_id, "host": urlparse(str(config.get("base_url") or "")).hostname or "", "created_at": row.created_at.isoformat() if row.created_at else None, "updated_at": row.updated_at.isoformat() if row.updated_at else None, "credential_fields": credential_fields(row.provider, config), } if include_config: result["config"] = config return result def _validate_visibility(visibility: str) -> str: mode = (visibility or "selected").strip().lower() if mode not in {"organization", "selected", "private"}: raise ExternalSystemError("visibility 必须是 organization、selected 或 private") return mode def _mark_connections_for_reverify(connections: Sequence[ExternalSystem]) -> None: """保留密文凭据,但在用户主动测试前阻止 agent 使用变更后的目标。""" for connection in connections: connection.last_verified_at = None connection.last_error = "系统定义已更新,请重新验证连接" connection.status = ( "needs_reverify" if connection.credentials else "needs_credentials" ) def _advance_active_connections( connections: Sequence[ExternalSystem], *, revision: int ) -> None: """仅运行策略变化不影响连接有效性,active 连接原子跟随新 revision。""" for connection in connections: if connection.status != "active": continue if connection.credentials: connection.verified_revision = revision else: connection.status = "needs_credentials" connection.last_verified_at = None connection.last_error = "连接凭据缺失,请重新填写凭据" def _selected_user_ids(s: Any, definition_id: UUID) -> list[str]: return [ str(uid) for uid in s.execute( select(ExternalSystemGrant.user_id) .where(ExternalSystemGrant.definition_id == definition_id) .order_by(ExternalSystemGrant.user_id) ) .scalars() .all() ] def _sync_selected_users( s: Any, definition: ExternalSystemDefinition, selected_user_ids: list[UUID], *, granted_by: UUID | None, ) -> None: wanted = set(selected_user_ids) if wanted: existing_users = set( s.execute(select(User.user_id).where(User.user_id.in_(wanted))) .scalars() .all() ) missing = wanted - existing_users if missing: raise ExternalSystemError( "包含不存在的用户: " + ", ".join(sorted(map(str, missing))) ) current_rows = ( s.execute( select(ExternalSystemGrant).where( ExternalSystemGrant.definition_id == definition.definition_id ) ) .scalars() .all() ) current = {row.user_id: row for row in current_rows} for uid, row in current.items(): if uid not in wanted: s.delete(row) connection = s.execute( select(ExternalSystem).where( ExternalSystem.definition_id == definition.definition_id, ExternalSystem.user_id == uid, ) ).scalar_one_or_none() if connection is not None: s.delete(connection) # 撤权同时删除该用户的密文凭据 connections = ( s.execute( select(ExternalSystem).where( ExternalSystem.definition_id == definition.definition_id ) ) .scalars() .all() ) for connection in connections: if connection.user_id not in wanted: s.delete(connection) for uid in wanted - set(current): s.add( ExternalSystemGrant( user_id=uid, definition_id=definition.definition_id, granted_by=granted_by, ) ) def provider_catalog(user_id: UUID) -> list[dict[str, Any]]: try: with session_scope() as s: rows = ( s.execute( select(ExternalSystemDefinition) .where( ExternalSystemDefinition.enabled.is_(True), or_( ExternalSystemDefinition.visibility == "organization", ExternalSystemDefinition.owner_user_id == user_id, exists( select(ExternalSystemGrant.user_id).where( ExternalSystemGrant.definition_id == ExternalSystemDefinition.definition_id, ExternalSystemGrant.user_id == user_id, ) ), ), ExternalSystemDefinition.review_status == "active", ) .order_by(ExternalSystemDefinition.name) ) .scalars() .all() ) definitions_by_provider: dict[str, list[dict[str, Any]]] = {} for row in rows: definitions_by_provider.setdefault(row.provider, []).append( _definition_view(row, include_config=False) ) except Exception: definitions_by_provider = {} key_ok = crypto_configured() return [ { "provider": spec.provider, "title": spec.title, "connector": spec.connector, "default_auth_type": spec.default_auth_type, "allowed_auth_types": list(spec.allowed_auth_types), "configured": bool(definitions_by_provider.get(spec.provider) and key_ok), "reason": "" if key_ok else "ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符", "definitions": definitions_by_provider.get(spec.provider, []), } for spec in provider_specs() ] def list_external_system_definitions() -> list[dict[str, Any]]: with session_scope() as s: rows = ( s.execute( select(ExternalSystemDefinition).order_by(ExternalSystemDefinition.name) ) .scalars() .all() ) results = [] for row in rows: item = _definition_view(row, include_config=True) item["selected_user_ids"] = _selected_user_ids(s, row.definition_id) results.append(item) return results def create_external_system_definition( admin_user_id: UUID, *, provider: str, name: str, config: dict[str, Any], enabled: bool = True, visibility: str = "selected", selected_user_ids: Optional[list[UUID]] = None, ) -> dict[str, Any]: provider = (provider or "").strip() name = (name or "").strip() try: get_provider(provider) except ValueError as exc: raise ExternalSystemError(str(exc)) from exc if not name or len(name) > 80: raise ExternalSystemError("系统名称不能为空且不能超过 80 字符") row = ExternalSystemDefinition( provider=provider, name=name, config=_normalized_config(provider, config), enabled=bool(enabled), visibility=_validate_visibility(visibility), owner_type="platform", trust_level="managed", review_status="active", created_by=admin_user_id, ) try: with session_scope() as s: s.add(row) s.flush() if row.visibility == "selected": _sync_selected_users( s, row, selected_user_ids or [], granted_by=admin_user_id ) s.flush() result = _definition_view(row, include_config=True) result["selected_user_ids"] = _selected_user_ids(s, row.definition_id) return result except IntegrityError as exc: raise ExternalSystemError("同名外部系统定义已存在") from exc def update_external_system_definition( definition_id: UUID, *, name: str, config: dict[str, Any], enabled: bool, visibility: str, selected_user_ids: Optional[list[UUID]] = None, ) -> dict[str, Any]: name = (name or "").strip() if not name or len(name) > 80: raise ExternalSystemError("系统名称不能为空且不能超过 80 字符") try: with session_scope() as s: row = s.execute( select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id ) ).scalar_one_or_none() if row is None: raise ExternalSystemError("external system definition not found") _, new_config, _, impact = ( _classify_definition_config_change( row.provider, row.config or {}, config, ) ) config_changed = impact != "none" connections = ( s.execute( select(ExternalSystem).where( ExternalSystem.definition_id == definition_id ) ) .scalars() .all() if config_changed else [] ) row.name = name row.config = new_config row.enabled = bool(enabled) row.visibility = _validate_visibility(visibility) if config_changed: row.revision += 1 if impact == "runtime": _advance_active_connections(connections, revision=row.revision) else: _mark_connections_for_reverify(connections) if row.visibility == "selected": _sync_selected_users( s, row, selected_user_ids or [], granted_by=row.created_by ) else: grants = ( s.execute( select(ExternalSystemGrant).where( ExternalSystemGrant.definition_id == definition_id ) ) .scalars() .all() ) for grant in grants: s.delete(grant) s.flush() result = _definition_view(row, include_config=True) result["selected_user_ids"] = _selected_user_ids(s, row.definition_id) return result except IntegrityError as exc: raise ExternalSystemError("同名外部系统定义已存在") from exc def delete_external_system_definition(definition_id: UUID) -> bool: try: with session_scope() as s: result = s.execute( delete(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id ) ) return bool(getattr(result, "rowcount", 0)) except IntegrityError as exc: raise ExternalSystemError("该系统已有用户连接,请先停用而不是删除") from exc def get_definition( definition_id: UUID, *, enabled_only: bool = False ) -> ExternalSystemDefinition: with session_scope() as s: stmt = select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id ) if enabled_only: stmt = stmt.where(ExternalSystemDefinition.enabled.is_(True)) row = s.execute(stmt).scalar_one_or_none() if row is None: raise ExternalSystemError("external system definition not found") s.expunge(row) return row def get_definition_for_user( user_id: UUID, definition_id: UUID ) -> ExternalSystemDefinition: with session_scope() as s: row = s.execute( select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id, ExternalSystemDefinition.enabled.is_(True), ExternalSystemDefinition.review_status == "active", or_( ExternalSystemDefinition.visibility == "organization", ExternalSystemDefinition.owner_user_id == user_id, exists( select(ExternalSystemGrant.user_id).where( ExternalSystemGrant.definition_id == definition_id, ExternalSystemGrant.user_id == user_id, ) ), ), ) ).scalar_one_or_none() if row is None: raise ExternalSystemError("external system definition not found") s.expunge(row) return row def _client( provider: str, credentials: dict[str, str], config: dict[str, Any], *, cache_namespace: str = "", ) -> OpenApiClient: spec = get_provider(provider) if spec.connector != "openapi": raise ExternalSystemError( f"unsupported external system connector: {spec.connector}" ) return OpenApiClient( credentials, _runtime_config(provider, config), cache_namespace=cache_namespace, ) def _credential_values( provider: str, config: dict[str, Any], credentials: dict[str, str] ) -> dict[str, str]: fields = credential_fields(provider, config) normalized = { field["name"]: str(credentials.get(field["name"]) or "").strip() for field in fields } missing = [field["label"] for field in fields if not normalized[field["name"]]] if missing: raise ExternalSystemError("请填写" + "、".join(missing)) return normalized def _credentials( provider: str, config: dict[str, Any], credentials: dict[str, str], *, user_id: UUID, definition_id: UUID, ) -> dict[str, str]: normalized = _credential_values(provider, config, credentials) try: return { name: encrypt_secret( value, aad=f"{user_id}:{definition_id}:{name}", ) for name, value in normalized.items() } except (RuntimeError, ValueError) as exc: raise ExternalSystemError(str(exc)) from exc def credentials_for(row: ExternalSystem) -> dict[str, str]: try: return { name: decrypt_secret( value, aad=f"{row.user_id}:{row.definition_id}:{name}", ) for name, value in row.credentials.items() } except (AttributeError, RuntimeError) as exc: raise ExternalSystemError(str(exc)) from exc def client_for_external_system(row: ExternalSystem) -> OpenApiClient: definition = get_definition_for_user(row.user_id, row.definition_id) if row.status != "active" or row.verified_revision != definition.revision: raise ExternalSystemError("外部系统连接需要重新验证") return _client( definition.provider, credentials_for(row), definition.config or {}, cache_namespace=( f"connection:{row.external_system_id}:revision:{row.verified_revision}" ), ) def _view(row: ExternalSystem, definition: ExternalSystemDefinition) -> dict[str, Any]: try: credential_ok = bool(row.credentials) except (AttributeError, TypeError): credential_ok = False runtime_config = _runtime_config(definition.provider, definition.config or {}) return { "external_system_id": str(row.external_system_id), "definition_id": str(row.definition_id), "system_name": definition.name, "provider": definition.provider, "connector": get_provider(definition.provider).connector, "name": row.name, "status": row.status if definition.enabled else "disabled", "username_masked": row.credential_hint or "***", "credential_configured": credential_ok, "credential_fields": credential_fields( definition.provider, definition.config or {} ), "query_guidance": runtime_config.query_guidance, "recommended_operation_ids": list(runtime_config.recommended_operation_ids), "operation_mode": runtime_config.operation_mode, "last_verified_at": row.last_verified_at.isoformat() if row.last_verified_at else None, "definition_revision": definition.revision, "verified_revision": row.verified_revision, "last_error": row.last_error or "", "created_at": row.created_at.isoformat() if row.created_at else None, "updated_at": row.updated_at.isoformat() if row.updated_at else None, } def list_external_systems(user_id: UUID) -> list[dict[str, Any]]: with session_scope() as s: rows = s.execute( select(ExternalSystem, ExternalSystemDefinition) .join( ExternalSystemDefinition, ExternalSystemDefinition.definition_id == ExternalSystem.definition_id, ) .where(ExternalSystem.user_id == user_id) .order_by(ExternalSystem.created_at) ).all() return [_view(row, definition) for row, definition in rows] def get_external_system( user_id: UUID, system_id: UUID, *, active_only: bool = False ) -> ExternalSystem: with session_scope() as s: stmt = select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, ExternalSystem.user_id == user_id, ) if active_only: stmt = stmt.where(ExternalSystem.status == "active") row = s.execute(stmt).scalar_one_or_none() if row is None: raise ExternalSystemError("external system not found") s.expunge(row) return row def create_external_system( user_id: UUID, *, definition_id: UUID, name: str, credentials: dict[str, str], ) -> dict[str, Any]: if not crypto_configured(): raise ExternalSystemError("ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符") definition = get_definition_for_user(user_id, definition_id) name = (name or definition.name).strip() if not name or len(name) > 80: raise ExternalSystemError("连接名称不能为空且不能超过 80 字符") plain = _credential_values( definition.provider, definition.config or {}, credentials, ) try: _client( definition.provider, plain, definition.config, cache_namespace=( f"probe:{definition.definition_id}:revision:{definition.revision}:" f"user:{user_id}" ), ).test_connection() except OpenApiError as exc: raise ExternalSystemError(str(exc)) from exc try: with session_scope() as s: row = s.execute( select(ExternalSystem).where( ExternalSystem.user_id == user_id, ExternalSystem.definition_id == definition.definition_id, ) ).scalar_one_or_none() if row is not None: raise ExternalSystemError("该外部系统已连接,请使用更新凭据") row = ExternalSystem( user_id=user_id, definition_id=definition.definition_id, ) s.add(row) row.name = name row.credentials = _credentials( definition.provider, definition.config or {}, plain, user_id=user_id, definition_id=definition.definition_id, ) row.credential_hint = ( mask_username(plain["username"]) if plain.get("username") else "***" ) row.status = "active" row.verified_revision = definition.revision row.last_error = None row.last_verified_at = datetime.now(timezone.utc) s.flush() return _view(row, definition) except IntegrityError as exc: raise ExternalSystemError("同名外部系统连接已存在") from exc def update_external_system_credentials( user_id: UUID, system_id: UUID, *, credentials: dict[str, str], ) -> dict[str, Any]: row = get_external_system(user_id, system_id) definition = get_definition(row.definition_id, enabled_only=True) plain = _credential_values( definition.provider, definition.config or {}, credentials, ) try: _client( definition.provider, plain, definition.config, cache_namespace=( f"probe:{system_id}:revision:{definition.revision}:user:{user_id}" ), ).test_connection() except OpenApiError as exc: raise ExternalSystemError(str(exc)) from exc with session_scope() as s: current = s.execute( select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, ExternalSystem.user_id == user_id, ) ).scalar_one() current.credentials = _credentials( definition.provider, definition.config or {}, plain, user_id=user_id, definition_id=definition.definition_id, ) current.credential_hint = ( mask_username(plain["username"]) if plain.get("username") else "***" ) current.status = "active" current.verified_revision = definition.revision current.last_error = None current.last_verified_at = datetime.now(timezone.utc) s.flush() return _view(current, definition) def test_external_system(user_id: UUID, system_id: UUID) -> dict[str, Any]: row = get_external_system(user_id, system_id) ok, error, probe = False, "", {} try: definition = get_definition_for_user(user_id, row.definition_id) probe = _client( definition.provider, credentials_for(row), definition.config or {}, cache_namespace=( f"connection:{row.external_system_id}:revision:{definition.revision}" ), ).test_connection() ok = True except (ExternalSystemError, OpenApiError) as exc: error = str(exc) with session_scope() as s: current = s.execute( select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, ExternalSystem.user_id == user_id, ) ).scalar_one() current.status = "active" if ok else "invalid" current.last_error = None if ok else error[:2000] if ok: current.last_verified_at = datetime.now(timezone.utc) current_definition = s.execute( select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == current.definition_id ) ).scalar_one() current.verified_revision = current_definition.revision return {"ok": ok, "error": error if not ok else "", **probe} def delete_external_system(user_id: UUID, system_id: UUID) -> bool: with session_scope() as s: row = s.execute( select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, ExternalSystem.user_id == user_id, ) ).scalar_one_or_none() if row is None: return False s.delete(row) # grants 独立存在,断开连接不撤销可见授权。 return True def external_system_tools_available(user_id: UUID) -> bool: if not crypto_configured(): return False try: with session_scope() as s: return ( s.execute( select(ExternalSystem.external_system_id) .join( ExternalSystemDefinition, ExternalSystemDefinition.definition_id == ExternalSystem.definition_id, ) .where( ExternalSystem.user_id == user_id, ExternalSystem.status == "active", ExternalSystemDefinition.enabled.is_(True), ExternalSystemDefinition.review_status == "active", ExternalSystem.verified_revision == ExternalSystemDefinition.revision, ) .limit(1) ).scalar_one_or_none() is not None ) except Exception: return False def external_system_status_available(user_id: UUID) -> bool: """只读状态工具不需要解密凭据;有连接即向 agent 暴露状态。""" try: with session_scope() as s: return ( s.execute( select(ExternalSystem.external_system_id) .where(ExternalSystem.user_id == user_id) .limit(1) ).scalar_one_or_none() is not None ) except Exception: return False def record_external_system_audit( *, user_id: UUID, task_id: UUID | None, external_system_id: UUID | None, definition_id: UUID | None, definition_revision: int, event: str, operation_id: str | None, outcome: str, duration_ms: int, status_code: int | None = None, response_bytes: int | None = None, detail: dict[str, Any] | None = None, ) -> None: """写入无敏感载荷的调用审计;调用方决定失败时是否降级。""" with session_scope() as s: s.add( ExternalSystemAudit( user_id=user_id, task_id=task_id, external_system_id=external_system_id, definition_id=definition_id, definition_revision=max(0, int(definition_revision)), event=event, operation_id=operation_id, outcome=outcome, status_code=status_code, duration_ms=max(0, int(duration_ms)), response_bytes=response_bytes, detail=detail or {}, ) )