"""外部系统目录、用户可见授权和密文连接的持久化服务层。""" from __future__ import annotations from datetime import datetime, timezone from typing import Any, Optional from urllib.parse import urlparse from uuid import UUID from sqlalchemy import delete, exists, or_, select from sqlalchemy.exc import IntegrityError from core.storage import session_scope from core.storage.models import ExternalSystem, ExternalSystemDefinition, User from .crypto import configured as crypto_configured from .crypto import decrypt_secret, encrypt_secret, mask_username from .openapi import OpenApiClient, OpenApiConfig, OpenApiError from .registry import credential_fields, get_provider, merged_config, provider_specs class ExternalSystemError(RuntimeError): pass def _runtime_config(provider: str, data: dict[str, Any]) -> OpenApiConfig: try: return OpenApiConfig.from_mapping(merged_config(provider, data)) except (OpenApiError, TypeError, ValueError) as exc: raise ExternalSystemError(str(exc)) from exc def _normalized_config(provider: str, data: dict[str, Any]) -> dict[str, Any]: cfg = _runtime_config(provider, data) return { "base_url": cfg.base_url, "openapi_url": cfg.openapi_url, "login_path": cfg.login_path, "allowed_post_operations": sorted(cfg.allowed_post_operations), "timeout_seconds": cfg.timeout_seconds, "max_result_bytes": cfg.max_result_bytes, "max_total_result_bytes": cfg.max_total_result_bytes, "max_page_size": cfg.max_page_size, "verify_tls": cfg.verify_tls, "query_guidance": cfg.query_guidance, "recommended_operation_ids": list(cfg.recommended_operation_ids), "auth_type": cfg.auth_type, **cfg.auth_config, } def _definition_view(row: ExternalSystemDefinition, *, include_config: bool) -> dict[str, Any]: config = row.config or {} result = { "definition_id": str(row.definition_id), "provider": row.provider, "provider_title": get_provider(row.provider).title, "connector": get_provider(row.provider).connector, "name": row.name, "enabled": row.enabled, "access_mode": row.access_mode, "host": urlparse(str(config.get("base_url") or "")).hostname or "", "created_at": row.created_at.isoformat() if row.created_at else None, "updated_at": row.updated_at.isoformat() if row.updated_at else None, "credential_fields": credential_fields(row.provider, config), } if include_config: result["config"] = config return result def _validate_access_mode(access_mode: str) -> str: mode = (access_mode or "selected").strip().lower() if mode not in {"all", "selected"}: raise ExternalSystemError("access_mode 必须是 all 或 selected") return mode def _selected_user_ids(s: Any, definition_id: UUID) -> list[str]: return [ str(uid) for uid in s.execute( select(ExternalSystem.user_id) .where(ExternalSystem.definition_id == definition_id) .order_by(ExternalSystem.user_id) ).scalars().all() ] def _sync_selected_users( s: Any, definition: ExternalSystemDefinition, selected_user_ids: list[UUID], ) -> None: wanted = set(selected_user_ids) if wanted: existing_users = set(s.execute( select(User.user_id).where(User.user_id.in_(wanted)) ).scalars().all()) missing = wanted - existing_users if missing: raise ExternalSystemError("包含不存在的用户: " + ", ".join(sorted(map(str, missing)))) current_rows = s.execute( select(ExternalSystem).where( ExternalSystem.definition_id == definition.definition_id ) ).scalars().all() current = {row.user_id: row for row in current_rows} for uid, row in current.items(): if uid not in wanted: s.delete(row) # 撤权同时删除该用户的密文凭据 for uid in wanted - set(current): s.add(ExternalSystem( user_id=uid, definition_id=definition.definition_id, provider=definition.provider, connector=get_provider(definition.provider).connector, name=definition.name, credentials={}, config={}, status="pending", )) def provider_catalog(user_id: UUID) -> list[dict[str, Any]]: try: with session_scope() as s: rows = s.execute( select(ExternalSystemDefinition) .where( ExternalSystemDefinition.enabled.is_(True), or_( ExternalSystemDefinition.access_mode == "all", exists( select(ExternalSystem.external_system_id).where( ExternalSystem.definition_id == ExternalSystemDefinition.definition_id, ExternalSystem.user_id == user_id, ) ), ), ) .order_by(ExternalSystemDefinition.name) ).scalars().all() definitions_by_provider: dict[str, list[dict[str, Any]]] = {} for row in rows: definitions_by_provider.setdefault(row.provider, []).append( _definition_view(row, include_config=False) ) except Exception: definitions_by_provider = {} key_ok = crypto_configured() return [ { "provider": spec.provider, "title": spec.title, "connector": spec.connector, "default_auth_type": spec.default_auth_type, "allowed_auth_types": list(spec.allowed_auth_types), "configured": bool(definitions_by_provider.get(spec.provider) and key_ok), "reason": "" if key_ok else "ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符", "definitions": definitions_by_provider.get(spec.provider, []), } for spec in provider_specs() ] def list_external_system_definitions() -> list[dict[str, Any]]: with session_scope() as s: rows = s.execute( select(ExternalSystemDefinition).order_by(ExternalSystemDefinition.name) ).scalars().all() results = [] for row in rows: item = _definition_view(row, include_config=True) item["selected_user_ids"] = _selected_user_ids(s, row.definition_id) results.append(item) return results def create_external_system_definition( admin_user_id: UUID, *, provider: str, name: str, config: dict[str, Any], enabled: bool = True, access_mode: str = "selected", selected_user_ids: Optional[list[UUID]] = None, ) -> dict[str, Any]: provider = (provider or "").strip() name = (name or "").strip() try: get_provider(provider) except ValueError as exc: raise ExternalSystemError(str(exc)) from exc if not name or len(name) > 80: raise ExternalSystemError("系统名称不能为空且不能超过 80 字符") row = ExternalSystemDefinition( provider=provider, name=name, config=_normalized_config(provider, config), enabled=bool(enabled), access_mode=_validate_access_mode(access_mode), created_by=admin_user_id, ) try: with session_scope() as s: s.add(row) s.flush() if row.access_mode == "selected": _sync_selected_users(s, row, selected_user_ids or []) s.flush() result = _definition_view(row, include_config=True) result["selected_user_ids"] = _selected_user_ids(s, row.definition_id) return result except IntegrityError as exc: raise ExternalSystemError("同名外部系统定义已存在") from exc def update_external_system_definition( definition_id: UUID, *, name: str, config: dict[str, Any], enabled: bool, access_mode: str, selected_user_ids: Optional[list[UUID]] = None, ) -> dict[str, Any]: name = (name or "").strip() if not name or len(name) > 80: raise ExternalSystemError("系统名称不能为空且不能超过 80 字符") try: with session_scope() as s: row = s.execute( select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id ) ).scalar_one_or_none() if row is None: raise ExternalSystemError("external system definition not found") row.name = name row.config = _normalized_config(row.provider, config) row.enabled = bool(enabled) row.access_mode = _validate_access_mode(access_mode) if row.access_mode == "selected": _sync_selected_users(s, row, selected_user_ids or []) s.flush() result = _definition_view(row, include_config=True) result["selected_user_ids"] = _selected_user_ids(s, row.definition_id) return result except IntegrityError as exc: raise ExternalSystemError("同名外部系统定义已存在") from exc def delete_external_system_definition(definition_id: UUID) -> bool: try: with session_scope() as s: result = s.execute( delete(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id ) ) return bool(result.rowcount) except IntegrityError as exc: raise ExternalSystemError("该系统已有用户连接,请先停用而不是删除") from exc def get_definition(definition_id: UUID, *, enabled_only: bool = False) -> ExternalSystemDefinition: with session_scope() as s: stmt = select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id ) if enabled_only: stmt = stmt.where(ExternalSystemDefinition.enabled.is_(True)) row = s.execute(stmt).scalar_one_or_none() if row is None: raise ExternalSystemError("external system definition not found") s.expunge(row) return row def get_definition_for_user(user_id: UUID, definition_id: UUID) -> ExternalSystemDefinition: with session_scope() as s: row = s.execute( select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == definition_id, ExternalSystemDefinition.enabled.is_(True), or_( ExternalSystemDefinition.access_mode == "all", exists( select(ExternalSystem.external_system_id).where( ExternalSystem.definition_id == definition_id, ExternalSystem.user_id == user_id, ) ), ), ) ).scalar_one_or_none() if row is None: raise ExternalSystemError("external system definition not found") s.expunge(row) return row def _client( provider: str, credentials: dict[str, str], config: dict[str, Any], *, cache_namespace: str = "", ) -> OpenApiClient: spec = get_provider(provider) if spec.connector != "openapi": raise ExternalSystemError(f"unsupported external system connector: {spec.connector}") return OpenApiClient( credentials, _runtime_config(provider, config), cache_namespace=cache_namespace, ) def _credential_values( provider: str, config: dict[str, Any], credentials: dict[str, str] ) -> dict[str, str]: fields = credential_fields(provider, config) normalized = { field["name"]: str(credentials.get(field["name"]) or "").strip() for field in fields } missing = [field["label"] for field in fields if not normalized[field["name"]]] if missing: raise ExternalSystemError("请填写" + "、".join(missing)) return normalized def _credentials( provider: str, config: dict[str, Any], credentials: dict[str, str] ) -> dict[str, str]: normalized = _credential_values(provider, config, credentials) try: return {name: encrypt_secret(value) for name, value in normalized.items()} except (RuntimeError, ValueError) as exc: raise ExternalSystemError(str(exc)) from exc def credentials_for(row: ExternalSystem) -> dict[str, str]: try: return {name: decrypt_secret(value) for name, value in row.credentials.items()} except (AttributeError, RuntimeError) as exc: raise ExternalSystemError(str(exc)) from exc def client_for_external_system(row: ExternalSystem) -> OpenApiClient: definition = get_definition_for_user(row.user_id, row.definition_id) return _client( definition.provider, credentials_for(row), definition.config or {}, cache_namespace=f"{definition.definition_id}:{row.user_id}", ) def _view(row: ExternalSystem, definition: ExternalSystemDefinition) -> dict[str, Any]: try: credentials = credentials_for(row) identity = credentials.get("username") or next(iter(credentials.values())) masked = mask_username(identity) if credentials.get("username") else "***" credential_ok = True except (ExternalSystemError, StopIteration): masked = "***" credential_ok = False runtime_config = _runtime_config(definition.provider, definition.config or {}) return { "external_system_id": str(row.external_system_id), "definition_id": str(row.definition_id), "system_name": definition.name, "provider": definition.provider, "connector": row.connector, "name": row.name, "status": row.status if definition.enabled else "disabled", "username_masked": masked, "credential_configured": credential_ok, "credential_fields": credential_fields(definition.provider, definition.config or {}), "query_guidance": runtime_config.query_guidance, "recommended_operation_ids": list(runtime_config.recommended_operation_ids), "last_verified_at": row.last_verified_at.isoformat() if row.last_verified_at else None, "created_at": row.created_at.isoformat() if row.created_at else None, "updated_at": row.updated_at.isoformat() if row.updated_at else None, } def list_external_systems(user_id: UUID) -> list[dict[str, Any]]: with session_scope() as s: rows = s.execute( select(ExternalSystem, ExternalSystemDefinition) .join( ExternalSystemDefinition, ExternalSystemDefinition.definition_id == ExternalSystem.definition_id, ) .where(ExternalSystem.user_id == user_id) .where(ExternalSystem.status != "pending") .order_by(ExternalSystem.created_at) ).all() return [_view(row, definition) for row, definition in rows] def get_external_system(user_id: UUID, system_id: UUID, *, active_only: bool = False) -> ExternalSystem: with session_scope() as s: stmt = select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, ExternalSystem.user_id == user_id, ) if active_only: stmt = stmt.where(ExternalSystem.status == "active") row = s.execute(stmt).scalar_one_or_none() if row is None: raise ExternalSystemError("external system not found") s.expunge(row) return row def create_external_system( user_id: UUID, *, definition_id: UUID, name: str, credentials: Optional[dict[str, str]] = None, username: str = "", password: str = "", ) -> dict[str, Any]: if not crypto_configured(): raise ExternalSystemError("ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符") definition = get_definition_for_user(user_id, definition_id) name = (name or definition.name).strip() if not name or len(name) > 80: raise ExternalSystemError("连接名称不能为空且不能超过 80 字符") plain = _credential_values( definition.provider, definition.config or {}, credentials or {"username": username, "password": password}, ) try: probe = _client( definition.provider, plain, definition.config, cache_namespace=f"{definition.definition_id}:{user_id}", ).test_connection() except OpenApiError as exc: raise ExternalSystemError(str(exc)) from exc try: with session_scope() as s: row = s.execute( select(ExternalSystem).where( ExternalSystem.user_id == user_id, ExternalSystem.definition_id == definition.definition_id, ) ).scalar_one_or_none() if row is not None and row.status != "pending": raise ExternalSystemError("该外部系统已连接,请使用更新凭据") if row is None: row = ExternalSystem( user_id=user_id, definition_id=definition.definition_id, provider=definition.provider, connector=get_provider(definition.provider).connector, ) s.add(row) row.name = name row.credentials = _credentials(definition.provider, definition.config or {}, plain) row.config = {"operation_count": probe.get("operation_count", 0)} row.status = "active" row.last_verified_at = datetime.now(timezone.utc) s.flush() return _view(row, definition) except IntegrityError as exc: raise ExternalSystemError("同名外部系统连接已存在") from exc def update_external_system_credentials( user_id: UUID, system_id: UUID, *, credentials: Optional[dict[str, str]] = None, username: str = "", password: str = "", ) -> dict[str, Any]: row = get_external_system(user_id, system_id) definition = get_definition(row.definition_id, enabled_only=True) plain = _credential_values( definition.provider, definition.config or {}, credentials or {"username": username, "password": password}, ) try: probe = _client( definition.provider, plain, definition.config, cache_namespace=f"{definition.definition_id}:{user_id}", ).test_connection() except OpenApiError as exc: raise ExternalSystemError(str(exc)) from exc with session_scope() as s: current = s.execute( select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, ExternalSystem.user_id == user_id, ) ).scalar_one() current.credentials = _credentials(definition.provider, definition.config or {}, plain) current.config = {**(current.config or {}), "operation_count": probe.get("operation_count", 0)} current.status = "active" current.last_verified_at = datetime.now(timezone.utc) s.flush() return _view(current, definition) def test_external_system(user_id: UUID, system_id: UUID) -> dict[str, Any]: row = get_external_system(user_id, system_id) ok, error, probe = False, "", {} try: probe = client_for_external_system(row).test_connection() ok = True except (ExternalSystemError, OpenApiError) as exc: error = str(exc) with session_scope() as s: current = s.execute( select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, ExternalSystem.user_id == user_id, ) ).scalar_one() current.status = "active" if ok else "invalid" if ok: current.last_verified_at = datetime.now(timezone.utc) current.config = {**(current.config or {}), **probe} return {"ok": ok, "error": error if not ok else "", **probe} def delete_external_system(user_id: UUID, system_id: UUID) -> bool: with session_scope() as s: row = s.execute( select(ExternalSystem).where( ExternalSystem.external_system_id == system_id, ExternalSystem.user_id == user_id, ) ).scalar_one_or_none() if row is None: return False definition = s.execute( select(ExternalSystemDefinition).where( ExternalSystemDefinition.definition_id == row.definition_id ) ).scalar_one() if definition.access_mode == "selected": # 用户断开只清凭据,保留管理员授予的可见权。 row.credentials = {} row.config = {} row.status = "pending" row.last_verified_at = None else: s.delete(row) return True def external_system_tools_available(user_id: UUID) -> bool: if not crypto_configured(): return False try: with session_scope() as s: return s.execute( select(ExternalSystem.external_system_id) .join( ExternalSystemDefinition, ExternalSystemDefinition.definition_id == ExternalSystem.definition_id, ) .where( ExternalSystem.user_id == user_id, ExternalSystem.status == "active", ExternalSystemDefinition.enabled.is_(True), ) .limit(1) ).scalar_one_or_none() is not None except Exception: return False