586 lines
22 KiB
Python
586 lines
22 KiB
Python
"""外部系统目录、用户可见授权和密文连接的持久化服务层。"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Optional
|
|
from urllib.parse import urlparse
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy import delete, exists, or_, select
|
|
from sqlalchemy.exc import IntegrityError
|
|
|
|
from core.storage import session_scope
|
|
from core.storage.models import ExternalSystem, ExternalSystemDefinition, User
|
|
|
|
from .crypto import configured as crypto_configured
|
|
from .crypto import decrypt_secret, encrypt_secret, mask_username
|
|
from .openapi import OpenApiClient, OpenApiConfig, OpenApiError
|
|
from .registry import credential_fields, get_provider, merged_config, provider_specs
|
|
|
|
|
|
class ExternalSystemError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _runtime_config(provider: str, data: dict[str, Any]) -> OpenApiConfig:
|
|
try:
|
|
return OpenApiConfig.from_mapping(merged_config(provider, data))
|
|
except (OpenApiError, TypeError, ValueError) as exc:
|
|
raise ExternalSystemError(str(exc)) from exc
|
|
|
|
|
|
def _normalized_config(provider: str, data: dict[str, Any]) -> dict[str, Any]:
|
|
cfg = _runtime_config(provider, data)
|
|
return {
|
|
"base_url": cfg.base_url,
|
|
"openapi_url": cfg.openapi_url,
|
|
"login_path": cfg.login_path,
|
|
"allowed_post_operations": sorted(cfg.allowed_post_operations),
|
|
"timeout_seconds": cfg.timeout_seconds,
|
|
"max_result_bytes": cfg.max_result_bytes,
|
|
"max_total_result_bytes": cfg.max_total_result_bytes,
|
|
"max_page_size": cfg.max_page_size,
|
|
"verify_tls": cfg.verify_tls,
|
|
"query_guidance": cfg.query_guidance,
|
|
"recommended_operation_ids": list(cfg.recommended_operation_ids),
|
|
"auth_type": cfg.auth_type,
|
|
**cfg.auth_config,
|
|
}
|
|
|
|
|
|
def _definition_view(row: ExternalSystemDefinition, *, include_config: bool) -> dict[str, Any]:
|
|
config = row.config or {}
|
|
result = {
|
|
"definition_id": str(row.definition_id),
|
|
"provider": row.provider,
|
|
"provider_title": get_provider(row.provider).title,
|
|
"connector": get_provider(row.provider).connector,
|
|
"name": row.name,
|
|
"enabled": row.enabled,
|
|
"access_mode": row.access_mode,
|
|
"host": urlparse(str(config.get("base_url") or "")).hostname or "",
|
|
"created_at": row.created_at.isoformat() if row.created_at else None,
|
|
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
|
"credential_fields": credential_fields(row.provider, config),
|
|
}
|
|
if include_config:
|
|
result["config"] = config
|
|
return result
|
|
|
|
|
|
def _validate_access_mode(access_mode: str) -> str:
|
|
mode = (access_mode or "selected").strip().lower()
|
|
if mode not in {"all", "selected"}:
|
|
raise ExternalSystemError("access_mode 必须是 all 或 selected")
|
|
return mode
|
|
|
|
|
|
def _selected_user_ids(s: Any, definition_id: UUID) -> list[str]:
|
|
return [
|
|
str(uid) for uid in s.execute(
|
|
select(ExternalSystem.user_id)
|
|
.where(ExternalSystem.definition_id == definition_id)
|
|
.order_by(ExternalSystem.user_id)
|
|
).scalars().all()
|
|
]
|
|
|
|
|
|
def _sync_selected_users(
|
|
s: Any,
|
|
definition: ExternalSystemDefinition,
|
|
selected_user_ids: list[UUID],
|
|
) -> None:
|
|
wanted = set(selected_user_ids)
|
|
if wanted:
|
|
existing_users = set(s.execute(
|
|
select(User.user_id).where(User.user_id.in_(wanted))
|
|
).scalars().all())
|
|
missing = wanted - existing_users
|
|
if missing:
|
|
raise ExternalSystemError("包含不存在的用户: " + ", ".join(sorted(map(str, missing))))
|
|
current_rows = s.execute(
|
|
select(ExternalSystem).where(
|
|
ExternalSystem.definition_id == definition.definition_id
|
|
)
|
|
).scalars().all()
|
|
current = {row.user_id: row for row in current_rows}
|
|
for uid, row in current.items():
|
|
if uid not in wanted:
|
|
s.delete(row) # 撤权同时删除该用户的密文凭据
|
|
for uid in wanted - set(current):
|
|
s.add(ExternalSystem(
|
|
user_id=uid,
|
|
definition_id=definition.definition_id,
|
|
provider=definition.provider,
|
|
connector=get_provider(definition.provider).connector,
|
|
name=definition.name,
|
|
credentials={},
|
|
config={},
|
|
status="pending",
|
|
))
|
|
|
|
|
|
def provider_catalog(user_id: UUID) -> list[dict[str, Any]]:
|
|
try:
|
|
with session_scope() as s:
|
|
rows = s.execute(
|
|
select(ExternalSystemDefinition)
|
|
.where(
|
|
ExternalSystemDefinition.enabled.is_(True),
|
|
or_(
|
|
ExternalSystemDefinition.access_mode == "all",
|
|
exists(
|
|
select(ExternalSystem.external_system_id).where(
|
|
ExternalSystem.definition_id
|
|
== ExternalSystemDefinition.definition_id,
|
|
ExternalSystem.user_id == user_id,
|
|
)
|
|
),
|
|
),
|
|
)
|
|
.order_by(ExternalSystemDefinition.name)
|
|
).scalars().all()
|
|
definitions_by_provider: dict[str, list[dict[str, Any]]] = {}
|
|
for row in rows:
|
|
definitions_by_provider.setdefault(row.provider, []).append(
|
|
_definition_view(row, include_config=False)
|
|
)
|
|
except Exception:
|
|
definitions_by_provider = {}
|
|
key_ok = crypto_configured()
|
|
return [
|
|
{
|
|
"provider": spec.provider,
|
|
"title": spec.title,
|
|
"connector": spec.connector,
|
|
"default_auth_type": spec.default_auth_type,
|
|
"allowed_auth_types": list(spec.allowed_auth_types),
|
|
"configured": bool(definitions_by_provider.get(spec.provider) and key_ok),
|
|
"reason": "" if key_ok else "ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符",
|
|
"definitions": definitions_by_provider.get(spec.provider, []),
|
|
}
|
|
for spec in provider_specs()
|
|
]
|
|
|
|
|
|
def list_external_system_definitions() -> list[dict[str, Any]]:
|
|
with session_scope() as s:
|
|
rows = s.execute(
|
|
select(ExternalSystemDefinition).order_by(ExternalSystemDefinition.name)
|
|
).scalars().all()
|
|
results = []
|
|
for row in rows:
|
|
item = _definition_view(row, include_config=True)
|
|
item["selected_user_ids"] = _selected_user_ids(s, row.definition_id)
|
|
results.append(item)
|
|
return results
|
|
|
|
|
|
def create_external_system_definition(
|
|
admin_user_id: UUID,
|
|
*,
|
|
provider: str,
|
|
name: str,
|
|
config: dict[str, Any],
|
|
enabled: bool = True,
|
|
access_mode: str = "selected",
|
|
selected_user_ids: Optional[list[UUID]] = None,
|
|
) -> dict[str, Any]:
|
|
provider = (provider or "").strip()
|
|
name = (name or "").strip()
|
|
try:
|
|
get_provider(provider)
|
|
except ValueError as exc:
|
|
raise ExternalSystemError(str(exc)) from exc
|
|
if not name or len(name) > 80:
|
|
raise ExternalSystemError("系统名称不能为空且不能超过 80 字符")
|
|
row = ExternalSystemDefinition(
|
|
provider=provider,
|
|
name=name,
|
|
config=_normalized_config(provider, config),
|
|
enabled=bool(enabled),
|
|
access_mode=_validate_access_mode(access_mode),
|
|
created_by=admin_user_id,
|
|
)
|
|
try:
|
|
with session_scope() as s:
|
|
s.add(row)
|
|
s.flush()
|
|
if row.access_mode == "selected":
|
|
_sync_selected_users(s, row, selected_user_ids or [])
|
|
s.flush()
|
|
result = _definition_view(row, include_config=True)
|
|
result["selected_user_ids"] = _selected_user_ids(s, row.definition_id)
|
|
return result
|
|
except IntegrityError as exc:
|
|
raise ExternalSystemError("同名外部系统定义已存在") from exc
|
|
|
|
|
|
def update_external_system_definition(
|
|
definition_id: UUID,
|
|
*,
|
|
name: str,
|
|
config: dict[str, Any],
|
|
enabled: bool,
|
|
access_mode: str,
|
|
selected_user_ids: Optional[list[UUID]] = None,
|
|
) -> dict[str, Any]:
|
|
name = (name or "").strip()
|
|
if not name or len(name) > 80:
|
|
raise ExternalSystemError("系统名称不能为空且不能超过 80 字符")
|
|
try:
|
|
with session_scope() as s:
|
|
row = s.execute(
|
|
select(ExternalSystemDefinition).where(
|
|
ExternalSystemDefinition.definition_id == definition_id
|
|
)
|
|
).scalar_one_or_none()
|
|
if row is None:
|
|
raise ExternalSystemError("external system definition not found")
|
|
row.name = name
|
|
row.config = _normalized_config(row.provider, config)
|
|
row.enabled = bool(enabled)
|
|
row.access_mode = _validate_access_mode(access_mode)
|
|
if row.access_mode == "selected":
|
|
_sync_selected_users(s, row, selected_user_ids or [])
|
|
s.flush()
|
|
result = _definition_view(row, include_config=True)
|
|
result["selected_user_ids"] = _selected_user_ids(s, row.definition_id)
|
|
return result
|
|
except IntegrityError as exc:
|
|
raise ExternalSystemError("同名外部系统定义已存在") from exc
|
|
|
|
|
|
def delete_external_system_definition(definition_id: UUID) -> bool:
|
|
try:
|
|
with session_scope() as s:
|
|
result = s.execute(
|
|
delete(ExternalSystemDefinition).where(
|
|
ExternalSystemDefinition.definition_id == definition_id
|
|
)
|
|
)
|
|
return bool(result.rowcount)
|
|
except IntegrityError as exc:
|
|
raise ExternalSystemError("该系统已有用户连接，请先停用而不是删除") from exc
|
|
|
|
|
|
def get_definition(definition_id: UUID, *, enabled_only: bool = False) -> ExternalSystemDefinition:
|
|
with session_scope() as s:
|
|
stmt = select(ExternalSystemDefinition).where(
|
|
ExternalSystemDefinition.definition_id == definition_id
|
|
)
|
|
if enabled_only:
|
|
stmt = stmt.where(ExternalSystemDefinition.enabled.is_(True))
|
|
row = s.execute(stmt).scalar_one_or_none()
|
|
if row is None:
|
|
raise ExternalSystemError("external system definition not found")
|
|
s.expunge(row)
|
|
return row
|
|
|
|
|
|
def get_definition_for_user(user_id: UUID, definition_id: UUID) -> ExternalSystemDefinition:
|
|
with session_scope() as s:
|
|
row = s.execute(
|
|
select(ExternalSystemDefinition).where(
|
|
ExternalSystemDefinition.definition_id == definition_id,
|
|
ExternalSystemDefinition.enabled.is_(True),
|
|
or_(
|
|
ExternalSystemDefinition.access_mode == "all",
|
|
exists(
|
|
select(ExternalSystem.external_system_id).where(
|
|
ExternalSystem.definition_id == definition_id,
|
|
ExternalSystem.user_id == user_id,
|
|
)
|
|
),
|
|
),
|
|
)
|
|
).scalar_one_or_none()
|
|
if row is None:
|
|
raise ExternalSystemError("external system definition not found")
|
|
s.expunge(row)
|
|
return row
|
|
|
|
|
|
def _client(
|
|
provider: str,
|
|
credentials: dict[str, str],
|
|
config: dict[str, Any],
|
|
*,
|
|
cache_namespace: str = "",
|
|
) -> OpenApiClient:
|
|
spec = get_provider(provider)
|
|
if spec.connector != "openapi":
|
|
raise ExternalSystemError(f"unsupported external system connector: {spec.connector}")
|
|
return OpenApiClient(
|
|
credentials,
|
|
_runtime_config(provider, config),
|
|
cache_namespace=cache_namespace,
|
|
)
|
|
|
|
|
|
def _credential_values(
|
|
provider: str, config: dict[str, Any], credentials: dict[str, str]
|
|
) -> dict[str, str]:
|
|
fields = credential_fields(provider, config)
|
|
normalized = {
|
|
field["name"]: str(credentials.get(field["name"]) or "").strip()
|
|
for field in fields
|
|
}
|
|
missing = [field["label"] for field in fields if not normalized[field["name"]]]
|
|
if missing:
|
|
raise ExternalSystemError("请填写" + "、".join(missing))
|
|
return normalized
|
|
|
|
|
|
def _credentials(
|
|
provider: str, config: dict[str, Any], credentials: dict[str, str]
|
|
) -> dict[str, str]:
|
|
normalized = _credential_values(provider, config, credentials)
|
|
try:
|
|
return {name: encrypt_secret(value) for name, value in normalized.items()}
|
|
except (RuntimeError, ValueError) as exc:
|
|
raise ExternalSystemError(str(exc)) from exc
|
|
|
|
|
|
def credentials_for(row: ExternalSystem) -> dict[str, str]:
|
|
try:
|
|
return {name: decrypt_secret(value) for name, value in row.credentials.items()}
|
|
except (AttributeError, RuntimeError) as exc:
|
|
raise ExternalSystemError(str(exc)) from exc
|
|
|
|
|
|
def client_for_external_system(row: ExternalSystem) -> OpenApiClient:
|
|
definition = get_definition_for_user(row.user_id, row.definition_id)
|
|
return _client(
|
|
definition.provider,
|
|
credentials_for(row),
|
|
definition.config or {},
|
|
cache_namespace=f"{definition.definition_id}:{row.user_id}",
|
|
)
|
|
|
|
|
|
def _view(row: ExternalSystem, definition: ExternalSystemDefinition) -> dict[str, Any]:
|
|
try:
|
|
credentials = credentials_for(row)
|
|
identity = credentials.get("username") or next(iter(credentials.values()))
|
|
masked = mask_username(identity) if credentials.get("username") else "***"
|
|
credential_ok = True
|
|
except (ExternalSystemError, StopIteration):
|
|
masked = "***"
|
|
credential_ok = False
|
|
runtime_config = _runtime_config(definition.provider, definition.config or {})
|
|
return {
|
|
"external_system_id": str(row.external_system_id),
|
|
"definition_id": str(row.definition_id),
|
|
"system_name": definition.name,
|
|
"provider": definition.provider,
|
|
"connector": row.connector,
|
|
"name": row.name,
|
|
"status": row.status if definition.enabled else "disabled",
|
|
"username_masked": masked,
|
|
"credential_configured": credential_ok,
|
|
"credential_fields": credential_fields(definition.provider, definition.config or {}),
|
|
"query_guidance": runtime_config.query_guidance,
|
|
"recommended_operation_ids": list(runtime_config.recommended_operation_ids),
|
|
"last_verified_at": row.last_verified_at.isoformat() if row.last_verified_at else None,
|
|
"created_at": row.created_at.isoformat() if row.created_at else None,
|
|
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
|
}
|
|
|
|
|
|
def list_external_systems(user_id: UUID) -> list[dict[str, Any]]:
|
|
with session_scope() as s:
|
|
rows = s.execute(
|
|
select(ExternalSystem, ExternalSystemDefinition)
|
|
.join(
|
|
ExternalSystemDefinition,
|
|
ExternalSystemDefinition.definition_id == ExternalSystem.definition_id,
|
|
)
|
|
.where(ExternalSystem.user_id == user_id)
|
|
.where(ExternalSystem.status != "pending")
|
|
.order_by(ExternalSystem.created_at)
|
|
).all()
|
|
return [_view(row, definition) for row, definition in rows]
|
|
|
|
|
|
def get_external_system(user_id: UUID, system_id: UUID, *, active_only: bool = False) -> ExternalSystem:
|
|
with session_scope() as s:
|
|
stmt = select(ExternalSystem).where(
|
|
ExternalSystem.external_system_id == system_id,
|
|
ExternalSystem.user_id == user_id,
|
|
)
|
|
if active_only:
|
|
stmt = stmt.where(ExternalSystem.status == "active")
|
|
row = s.execute(stmt).scalar_one_or_none()
|
|
if row is None:
|
|
raise ExternalSystemError("external system not found")
|
|
s.expunge(row)
|
|
return row
|
|
|
|
|
|
def create_external_system(
|
|
user_id: UUID,
|
|
*,
|
|
definition_id: UUID,
|
|
name: str,
|
|
credentials: Optional[dict[str, str]] = None,
|
|
username: str = "",
|
|
password: str = "",
|
|
) -> dict[str, Any]:
|
|
if not crypto_configured():
|
|
raise ExternalSystemError("ZCBOT_CREDENTIAL_MASTER_KEY 未配置或少于 32 字符")
|
|
definition = get_definition_for_user(user_id, definition_id)
|
|
name = (name or definition.name).strip()
|
|
if not name or len(name) > 80:
|
|
raise ExternalSystemError("连接名称不能为空且不能超过 80 字符")
|
|
plain = _credential_values(
|
|
definition.provider,
|
|
definition.config or {},
|
|
credentials or {"username": username, "password": password},
|
|
)
|
|
try:
|
|
probe = _client(
|
|
definition.provider,
|
|
plain,
|
|
definition.config,
|
|
cache_namespace=f"{definition.definition_id}:{user_id}",
|
|
).test_connection()
|
|
except OpenApiError as exc:
|
|
raise ExternalSystemError(str(exc)) from exc
|
|
try:
|
|
with session_scope() as s:
|
|
row = s.execute(
|
|
select(ExternalSystem).where(
|
|
ExternalSystem.user_id == user_id,
|
|
ExternalSystem.definition_id == definition.definition_id,
|
|
)
|
|
).scalar_one_or_none()
|
|
if row is not None and row.status != "pending":
|
|
raise ExternalSystemError("该外部系统已连接，请使用更新凭据")
|
|
if row is None:
|
|
row = ExternalSystem(
|
|
user_id=user_id,
|
|
definition_id=definition.definition_id,
|
|
provider=definition.provider,
|
|
connector=get_provider(definition.provider).connector,
|
|
)
|
|
s.add(row)
|
|
row.name = name
|
|
row.credentials = _credentials(definition.provider, definition.config or {}, plain)
|
|
row.config = {"operation_count": probe.get("operation_count", 0)}
|
|
row.status = "active"
|
|
row.last_verified_at = datetime.now(timezone.utc)
|
|
s.flush()
|
|
return _view(row, definition)
|
|
except IntegrityError as exc:
|
|
raise ExternalSystemError("同名外部系统连接已存在") from exc
|
|
|
|
|
|
def update_external_system_credentials(
|
|
user_id: UUID,
|
|
system_id: UUID,
|
|
*,
|
|
credentials: Optional[dict[str, str]] = None,
|
|
username: str = "",
|
|
password: str = "",
|
|
) -> dict[str, Any]:
|
|
row = get_external_system(user_id, system_id)
|
|
definition = get_definition(row.definition_id, enabled_only=True)
|
|
plain = _credential_values(
|
|
definition.provider,
|
|
definition.config or {},
|
|
credentials or {"username": username, "password": password},
|
|
)
|
|
try:
|
|
probe = _client(
|
|
definition.provider,
|
|
plain,
|
|
definition.config,
|
|
cache_namespace=f"{definition.definition_id}:{user_id}",
|
|
).test_connection()
|
|
except OpenApiError as exc:
|
|
raise ExternalSystemError(str(exc)) from exc
|
|
with session_scope() as s:
|
|
current = s.execute(
|
|
select(ExternalSystem).where(
|
|
ExternalSystem.external_system_id == system_id,
|
|
ExternalSystem.user_id == user_id,
|
|
)
|
|
).scalar_one()
|
|
current.credentials = _credentials(definition.provider, definition.config or {}, plain)
|
|
current.config = {**(current.config or {}), "operation_count": probe.get("operation_count", 0)}
|
|
current.status = "active"
|
|
current.last_verified_at = datetime.now(timezone.utc)
|
|
s.flush()
|
|
return _view(current, definition)
|
|
|
|
|
|
def test_external_system(user_id: UUID, system_id: UUID) -> dict[str, Any]:
|
|
row = get_external_system(user_id, system_id)
|
|
ok, error, probe = False, "", {}
|
|
try:
|
|
probe = client_for_external_system(row).test_connection()
|
|
ok = True
|
|
except (ExternalSystemError, OpenApiError) as exc:
|
|
error = str(exc)
|
|
with session_scope() as s:
|
|
current = s.execute(
|
|
select(ExternalSystem).where(
|
|
ExternalSystem.external_system_id == system_id,
|
|
ExternalSystem.user_id == user_id,
|
|
)
|
|
).scalar_one()
|
|
current.status = "active" if ok else "invalid"
|
|
if ok:
|
|
current.last_verified_at = datetime.now(timezone.utc)
|
|
current.config = {**(current.config or {}), **probe}
|
|
return {"ok": ok, "error": error if not ok else "", **probe}
|
|
|
|
|
|
def delete_external_system(user_id: UUID, system_id: UUID) -> bool:
|
|
with session_scope() as s:
|
|
row = s.execute(
|
|
select(ExternalSystem).where(
|
|
ExternalSystem.external_system_id == system_id,
|
|
ExternalSystem.user_id == user_id,
|
|
)
|
|
).scalar_one_or_none()
|
|
if row is None:
|
|
return False
|
|
definition = s.execute(
|
|
select(ExternalSystemDefinition).where(
|
|
ExternalSystemDefinition.definition_id == row.definition_id
|
|
)
|
|
).scalar_one()
|
|
if definition.access_mode == "selected":
|
|
# 用户断开只清凭据，保留管理员授予的可见权。
|
|
row.credentials = {}
|
|
row.config = {}
|
|
row.status = "pending"
|
|
row.last_verified_at = None
|
|
else:
|
|
s.delete(row)
|
|
return True
|
|
|
|
|
|
def external_system_tools_available(user_id: UUID) -> bool:
|
|
if not crypto_configured():
|
|
return False
|
|
try:
|
|
with session_scope() as s:
|
|
return s.execute(
|
|
select(ExternalSystem.external_system_id)
|
|
.join(
|
|
ExternalSystemDefinition,
|
|
ExternalSystemDefinition.definition_id == ExternalSystem.definition_id,
|
|
)
|
|
.where(
|
|
ExternalSystem.user_id == user_id,
|
|
ExternalSystem.status == "active",
|
|
ExternalSystemDefinition.enabled.is_(True),
|
|
)
|
|
.limit(1)
|
|
).scalar_one_or_none() is not None
|
|
except Exception:
|
|
return False
|