986 lines
38 KiB
Python
986 lines
38 KiB
Python
"""通用 OpenAPI 外部系统连接器。
|
||
|
||
目标地址全部来自管理员维护的可信系统目录；模型和普通用户只能传 operation_id
|
||
与结构化参数，不能传 URL。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import binascii
|
||
import copy
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import time
|
||
from contextlib import contextmanager
|
||
from dataclasses import dataclass, field
|
||
from typing import Any, Iterator, Optional
|
||
from urllib.parse import quote, urljoin, urlparse
|
||
|
||
import httpx
|
||
|
||
from .auth import ExternalAuthError, get_auth_strategy
|
||
from .catalog import (
|
||
compile_operation_catalog,
|
||
operation_id,
|
||
resolve_local_object,
|
||
validate_json_value,
|
||
)
|
||
from .results import MAX_STORED_RESULT_BYTES
|
||
from .runtime_cache import RUNTIME_CACHE
|
||
|
||
|
||
class OpenApiError(RuntimeError):
|
||
pass
|
||
|
||
|
||
_AUTH_CACHE_TTL_SECONDS = 300
|
||
_SPEC_CACHE_TTL_SECONDS = 300
|
||
_SCHEMA_MAX_DEPTH = 5
|
||
_SCHEMA_MAX_PROPERTIES = 50
|
||
_SCHEMA_MAX_ENUM_ITEMS = 30
|
||
_SCHEMA_MAX_NODES = 100
|
||
_SCHEMA_TEXT_MAX_CHARS = 1000
|
||
_ERROR_DETAIL_MAX_CHARS = 2000
|
||
_MAX_SPEC_BYTES = 5 * 1024 * 1024
|
||
_SENSITIVE_KEY_RE = re.compile(
|
||
r"(?:password|passwd|secret|token|api[_-]?key|authorization|cookie|credential)",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
class _SpecCacheView:
|
||
"""保留测试和诊断入口；实际数据由统一运行态缓存持有。"""
|
||
|
||
@staticmethod
|
||
def clear() -> None:
|
||
RUNTIME_CACHE.clear()
|
||
|
||
@staticmethod
|
||
def __len__() -> int:
|
||
return RUNTIME_CACHE.spec_count()
|
||
|
||
|
||
_SPEC_CACHE = _SpecCacheView()
|
||
|
||
|
||
def _bool_value(value: Any, default: bool) -> bool:
|
||
raw = str(value if value is not None else "").strip().lower()
|
||
if not raw:
|
||
return default
|
||
return raw in {"1", "true", "yes", "on"}
|
||
|
||
|
||
def _validated_http_url(raw: str, label: str) -> str:
|
||
value = (raw or "").strip().rstrip("/")
|
||
parsed = urlparse(value)
|
||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||
raise OpenApiError(f"{label} 必须是有效的 http(s) URL")
|
||
if parsed.username or parsed.password:
|
||
raise OpenApiError(f"{label} 不能内嵌凭据")
|
||
return value
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class OpenApiConfig:
|
||
base_url: str
|
||
openapi_url: str
|
||
login_path: str
|
||
operation_policies: dict[str, str]
|
||
timeout_seconds: float
|
||
max_result_bytes: int
|
||
max_total_result_bytes: int
|
||
max_page_size: int
|
||
verify_tls: bool
|
||
query_guidance: str
|
||
recommended_operation_ids: tuple[str, ...]
|
||
operation_mode: str = "query"
|
||
auth_type: str = "password_jwt"
|
||
auth_config: dict[str, Any] = field(
|
||
default_factory=lambda: {
|
||
"login_path": "/api/auth/token/",
|
||
"username_field": "username",
|
||
"password_field": "password",
|
||
"token_field": "access",
|
||
"auth_header_name": "Authorization",
|
||
"auth_header_template": "Bearer {token}",
|
||
}
|
||
)
|
||
|
||
@classmethod
|
||
def from_mapping(cls, data: dict[str, Any]) -> "OpenApiConfig":
|
||
"""从管理员保存的可信目录配置构建运行态配置。"""
|
||
base = _validated_http_url(str(data.get("base_url") or ""), "base_url")
|
||
spec = _validated_http_url(str(data.get("openapi_url") or ""), "openapi_url")
|
||
base_origin = urlparse(base)
|
||
spec_origin = urlparse(spec)
|
||
if (base_origin.scheme, base_origin.netloc) != (
|
||
spec_origin.scheme,
|
||
spec_origin.netloc,
|
||
):
|
||
raise OpenApiError("openapi_url 必须与 base_url 同源")
|
||
login_path = str(data.get("login_path") or "/api/auth/token/").strip()
|
||
if not login_path.startswith("/") or "://" in login_path:
|
||
raise OpenApiError("login_path 必须是站内绝对路径")
|
||
raw_policies = data.get("operation_policies") or {}
|
||
if not isinstance(raw_policies, dict):
|
||
raise OpenApiError("operation_policies 必须是 operationId 到策略的对象")
|
||
policies = {
|
||
str(operation_id).strip(): str(policy).strip().lower()
|
||
for operation_id, policy in raw_policies.items()
|
||
if str(operation_id).strip()
|
||
}
|
||
if len(policies) > 500 or any(len(key) > 200 for key in policies):
|
||
raise OpenApiError(
|
||
"operation_policies 最多 500 项且 operationId 不超过 200 字符"
|
||
)
|
||
invalid_policies = sorted(set(policies.values()) - {"read", "export"})
|
||
if invalid_policies:
|
||
raise OpenApiError(
|
||
"不支持的 operation policy: " + ", ".join(invalid_policies)
|
||
)
|
||
operation_mode = str(data.get("operation_mode") or "query").strip().lower()
|
||
if operation_mode not in {"query", "upstream_managed"}:
|
||
raise OpenApiError("operation_mode 必须是 query 或 upstream_managed")
|
||
guidance = str(data.get("query_guidance") or "").strip()
|
||
if len(guidance) > 4000:
|
||
raise OpenApiError("query_guidance 不能超过 4000 字符")
|
||
raw_recommended = data.get("recommended_operation_ids", [])
|
||
if isinstance(raw_recommended, str):
|
||
raw_recommended = raw_recommended.split(",")
|
||
if not isinstance(raw_recommended, (list, tuple, set)):
|
||
raise OpenApiError("recommended_operation_ids 必须是字符串数组")
|
||
recommended = tuple(
|
||
dict.fromkeys(
|
||
str(item).strip() for item in raw_recommended if str(item).strip()
|
||
)
|
||
)
|
||
if len(recommended) > 30 or any(len(item) > 200 for item in recommended):
|
||
raise OpenApiError(
|
||
"recommended_operation_ids 最多 30 项且每项不超过 200 字符"
|
||
)
|
||
max_result = max(4096, min(int(data.get("max_result_bytes", 65536)), 1048576))
|
||
return cls(
|
||
base_url=base,
|
||
openapi_url=spec,
|
||
login_path=login_path,
|
||
operation_policies=policies,
|
||
timeout_seconds=max(1.0, min(float(data.get("timeout_seconds", 15)), 60.0)),
|
||
max_result_bytes=max_result,
|
||
max_total_result_bytes=max(
|
||
max_result,
|
||
min(int(data.get("max_total_result_bytes", 262144)), 4194304),
|
||
),
|
||
max_page_size=max(1, min(int(data.get("max_page_size", 200)), 1000)),
|
||
verify_tls=_bool_value(data.get("verify_tls"), True),
|
||
query_guidance=guidance,
|
||
recommended_operation_ids=recommended,
|
||
operation_mode=operation_mode,
|
||
auth_type=str(data.get("auth_type") or "password_jwt").strip(),
|
||
auth_config={
|
||
key: data[key]
|
||
for key in (
|
||
"login_path",
|
||
"username_field",
|
||
"password_field",
|
||
"token_field",
|
||
"auth_header_name",
|
||
"auth_header_template",
|
||
)
|
||
if key in data
|
||
},
|
||
)
|
||
|
||
|
||
class OpenApiClient:
|
||
def __init__(
|
||
self,
|
||
credentials: dict[str, str],
|
||
cfg: OpenApiConfig,
|
||
*,
|
||
cache_namespace: str = "",
|
||
):
|
||
self.credentials = credentials
|
||
self.cfg = cfg
|
||
identity = json.dumps(
|
||
{
|
||
"namespace": cache_namespace,
|
||
"credentials": credentials,
|
||
"config": cfg.__dict__,
|
||
},
|
||
sort_keys=True,
|
||
ensure_ascii=False,
|
||
default=str,
|
||
)
|
||
digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||
self._runtime_identity = digest
|
||
|
||
def _client(self) -> httpx.Client:
|
||
return httpx.Client(
|
||
timeout=self.cfg.timeout_seconds,
|
||
verify=self.cfg.verify_tls,
|
||
follow_redirects=False,
|
||
)
|
||
|
||
@contextmanager
|
||
def _runtime_client(self) -> Iterator[httpx.Client]:
|
||
with RUNTIME_CACHE.client(self._runtime_identity, self._client) as client:
|
||
yield client
|
||
|
||
@staticmethod
|
||
def _auth_cache_ttl(headers: dict[str, str]) -> float:
|
||
authorization = next(
|
||
(
|
||
value
|
||
for name, value in headers.items()
|
||
if name.lower() == "authorization"
|
||
),
|
||
"",
|
||
)
|
||
token = authorization.split(" ", 1)[-1].strip()
|
||
parts = token.split(".")
|
||
if len(parts) == 3:
|
||
try:
|
||
padding = "=" * (-len(parts[1]) % 4)
|
||
payload = json.loads(
|
||
base64.urlsafe_b64decode(parts[1] + padding).decode("utf-8")
|
||
)
|
||
expires_at = float(payload.get("exp"))
|
||
return max(
|
||
0.0,
|
||
min(_AUTH_CACHE_TTL_SECONDS, expires_at - time.time() - 30),
|
||
)
|
||
except (binascii.Error, TypeError, ValueError, UnicodeDecodeError):
|
||
pass
|
||
return _AUTH_CACHE_TTL_SECONDS
|
||
|
||
def authenticate(
|
||
self, *, client: httpx.Client | None = None, force: bool = False
|
||
) -> dict[str, str]:
|
||
if client is None:
|
||
with self._runtime_client() as runtime_client:
|
||
return self.authenticate(client=runtime_client, force=force)
|
||
if force:
|
||
RUNTIME_CACHE.invalidate_auth(self._runtime_identity)
|
||
else:
|
||
cached = RUNTIME_CACHE.get_auth(self._runtime_identity)
|
||
if cached is not None:
|
||
return cached
|
||
|
||
def load() -> dict[str, str]:
|
||
if not force:
|
||
cached = RUNTIME_CACHE.get_auth(self._runtime_identity)
|
||
if cached is not None:
|
||
return cached
|
||
try:
|
||
headers = get_auth_strategy(self.cfg.auth_type).headers(
|
||
client=client,
|
||
base_url=self.cfg.base_url,
|
||
credentials=self.credentials,
|
||
config=self.cfg.auth_config,
|
||
)
|
||
except ExternalAuthError as exc:
|
||
raise OpenApiError(str(exc)) from exc
|
||
RUNTIME_CACHE.set_auth(
|
||
self._runtime_identity,
|
||
headers,
|
||
ttl_seconds=self._auth_cache_ttl(headers),
|
||
)
|
||
return dict(headers)
|
||
|
||
return RUNTIME_CACHE.singleflight("auth", self._runtime_identity, load)
|
||
|
||
def _refresh_auth(
|
||
self, client: httpx.Client, *, failed_generation: int
|
||
) -> dict[str, str]:
|
||
def refresh() -> dict[str, str]:
|
||
cached, current_generation = RUNTIME_CACHE.auth_state(
|
||
self._runtime_identity
|
||
)
|
||
if cached is not None and current_generation != failed_generation:
|
||
return cached
|
||
return self.authenticate(client=client, force=True)
|
||
|
||
return RUNTIME_CACHE.singleflight(
|
||
"auth-refresh", self._runtime_identity, refresh
|
||
)
|
||
|
||
@staticmethod
|
||
def _limited_request(
|
||
client: httpx.Client,
|
||
method: str,
|
||
url: str,
|
||
*,
|
||
limit: int,
|
||
label: str,
|
||
**kwargs: Any,
|
||
) -> tuple[int, dict[str, str], bytes]:
|
||
"""流式读取远端响应，在解析 JSON 前执行硬字节上限。"""
|
||
with client.stream(method, url, **kwargs) as response:
|
||
raw_length = response.headers.get("content-length")
|
||
if raw_length:
|
||
try:
|
||
if int(raw_length) > limit:
|
||
raise OpenApiError(f"{label}超过安全下载上限({limit} bytes)")
|
||
except ValueError:
|
||
pass
|
||
chunks: list[bytes] = []
|
||
total = 0
|
||
for chunk in response.iter_bytes():
|
||
total += len(chunk)
|
||
if total > limit:
|
||
raise OpenApiError(f"{label}超过安全下载上限({limit} bytes)")
|
||
chunks.append(chunk)
|
||
return response.status_code, dict(response.headers), b"".join(chunks)
|
||
|
||
def _fetch_spec(
|
||
self,
|
||
headers: dict[str, str],
|
||
*,
|
||
client: httpx.Client | None = None,
|
||
) -> dict[str, Any]:
|
||
if client is None:
|
||
with self._runtime_client() as runtime_client:
|
||
return self._fetch_spec(headers, client=runtime_client)
|
||
cached = RUNTIME_CACHE.get_spec(self._runtime_identity)
|
||
if cached is not None:
|
||
return cached
|
||
|
||
def load() -> dict[str, Any]:
|
||
cached = RUNTIME_CACHE.get_spec(self._runtime_identity)
|
||
if cached is not None:
|
||
return cached
|
||
request_client = client
|
||
try:
|
||
_, auth_generation = RUNTIME_CACHE.auth_state(self._runtime_identity)
|
||
status_code, _, content = self._limited_request(
|
||
request_client,
|
||
"GET",
|
||
self.cfg.openapi_url,
|
||
headers=headers,
|
||
limit=_MAX_SPEC_BYTES,
|
||
label="OpenAPI 文档",
|
||
)
|
||
if status_code == 401:
|
||
refreshed_headers = self._refresh_auth(
|
||
request_client, failed_generation=auth_generation
|
||
)
|
||
status_code, _, content = self._limited_request(
|
||
request_client,
|
||
"GET",
|
||
self.cfg.openapi_url,
|
||
headers=refreshed_headers,
|
||
limit=_MAX_SPEC_BYTES,
|
||
label="OpenAPI 文档",
|
||
)
|
||
except httpx.HTTPError as exc:
|
||
raise OpenApiError(f"OpenAPI 获取失败: {type(exc).__name__}") from exc
|
||
if status_code >= 400:
|
||
raise OpenApiError(f"OpenAPI 获取失败(HTTP {status_code})")
|
||
try:
|
||
spec = json.loads(content.decode("utf-8-sig"))
|
||
except (UnicodeDecodeError, ValueError) as exc:
|
||
raise OpenApiError("OpenAPI 文档不是有效 JSON") from exc
|
||
if not isinstance(spec, dict) or not isinstance(spec.get("paths"), dict):
|
||
raise OpenApiError("OpenAPI 文档缺少 paths")
|
||
RUNTIME_CACHE.set_spec(
|
||
self._runtime_identity, spec, ttl_seconds=_SPEC_CACHE_TTL_SECONDS
|
||
)
|
||
return spec
|
||
|
||
return RUNTIME_CACHE.singleflight("spec", self._runtime_identity, load)
|
||
|
||
@staticmethod
|
||
def _operation_id(method: str, path: str, operation: dict[str, Any]) -> str:
|
||
return operation_id(method, path, operation)
|
||
|
||
@staticmethod
|
||
def _resolve_object(spec: dict[str, Any], value: Any) -> Any:
|
||
return resolve_local_object(spec, value)
|
||
|
||
def _operations(self, spec: dict[str, Any]) -> list[dict[str, Any]]:
|
||
cached = RUNTIME_CACHE.get_catalog(self._runtime_identity, spec)
|
||
if cached is None:
|
||
|
||
def compile_catalog():
|
||
current = RUNTIME_CACHE.get_catalog(self._runtime_identity, spec)
|
||
if current is not None:
|
||
return current
|
||
catalog = compile_operation_catalog(spec)
|
||
RUNTIME_CACHE.set_catalog(self._runtime_identity, spec, catalog)
|
||
return catalog
|
||
|
||
cached = RUNTIME_CACHE.singleflight(
|
||
"catalog", self._runtime_identity, compile_catalog
|
||
)
|
||
return list(cached.operations)
|
||
|
||
@classmethod
|
||
def _compact_schema(
|
||
cls,
|
||
spec: dict[str, Any],
|
||
schema: Any,
|
||
*,
|
||
depth: int = 0,
|
||
seen_refs: frozenset[str] = frozenset(),
|
||
budget: Optional[list[int]] = None,
|
||
) -> dict[str, Any]:
|
||
"""展开本地 schema 引用并限制体量，供模型构造请求而非做完整规范校验。"""
|
||
if budget is None:
|
||
budget = [_SCHEMA_MAX_NODES]
|
||
if budget[0] <= 0:
|
||
return {"truncated": True}
|
||
budget[0] -= 1
|
||
if not isinstance(schema, dict) or depth > _SCHEMA_MAX_DEPTH:
|
||
return {}
|
||
ref = schema.get("$ref")
|
||
if isinstance(ref, str):
|
||
if ref in seen_refs:
|
||
return {"$ref": ref, "recursive": True}
|
||
prefixes = (
|
||
("#/definitions/", spec.get("definitions")),
|
||
(
|
||
"#/components/schemas/",
|
||
(spec.get("components") or {}).get("schemas"),
|
||
),
|
||
)
|
||
for prefix, definitions in prefixes:
|
||
if ref.startswith(prefix) and isinstance(definitions, dict):
|
||
target = definitions.get(ref[len(prefix) :])
|
||
if isinstance(target, dict):
|
||
return cls._compact_schema(
|
||
spec,
|
||
target,
|
||
depth=depth + 1,
|
||
seen_refs=seen_refs | {ref},
|
||
budget=budget,
|
||
)
|
||
return {"$ref": ref}
|
||
|
||
compact: dict[str, Any] = {}
|
||
for key in (
|
||
"type",
|
||
"title",
|
||
"description",
|
||
"format",
|
||
"default",
|
||
"nullable",
|
||
"x-nullable",
|
||
"minimum",
|
||
"maximum",
|
||
"minLength",
|
||
"maxLength",
|
||
"pattern",
|
||
):
|
||
value = schema.get(key)
|
||
if isinstance(value, str):
|
||
compact[key] = value[:_SCHEMA_TEXT_MAX_CHARS]
|
||
elif (
|
||
isinstance(value, (int, float, bool)) or value is None and key in schema
|
||
):
|
||
compact[key] = value
|
||
required = schema.get("required")
|
||
if isinstance(required, list):
|
||
compact["required"] = [
|
||
str(item) for item in required[:_SCHEMA_MAX_PROPERTIES]
|
||
]
|
||
enum = schema.get("enum")
|
||
if isinstance(enum, list):
|
||
compact["enum"] = [
|
||
item[:_SCHEMA_TEXT_MAX_CHARS] if isinstance(item, str) else item
|
||
for item in enum[:_SCHEMA_MAX_ENUM_ITEMS]
|
||
]
|
||
properties = schema.get("properties")
|
||
if isinstance(properties, dict) and depth < _SCHEMA_MAX_DEPTH:
|
||
compact["properties"] = {
|
||
str(name): cls._compact_schema(
|
||
spec,
|
||
value,
|
||
depth=depth + 1,
|
||
seen_refs=seen_refs,
|
||
budget=budget,
|
||
)
|
||
for name, value in list(properties.items())[:_SCHEMA_MAX_PROPERTIES]
|
||
if isinstance(value, dict)
|
||
}
|
||
items = schema.get("items")
|
||
if isinstance(items, dict) and depth < _SCHEMA_MAX_DEPTH:
|
||
compact["items"] = cls._compact_schema(
|
||
spec,
|
||
items,
|
||
depth=depth + 1,
|
||
seen_refs=seen_refs,
|
||
budget=budget,
|
||
)
|
||
additional = schema.get("additionalProperties")
|
||
if isinstance(additional, bool):
|
||
compact["additionalProperties"] = additional
|
||
elif isinstance(additional, dict) and depth < _SCHEMA_MAX_DEPTH:
|
||
compact["additionalProperties"] = cls._compact_schema(
|
||
spec,
|
||
additional,
|
||
depth=depth + 1,
|
||
seen_refs=seen_refs,
|
||
budget=budget,
|
||
)
|
||
for key in ("allOf", "anyOf", "oneOf"):
|
||
variants = schema.get(key)
|
||
if isinstance(variants, list) and depth < _SCHEMA_MAX_DEPTH:
|
||
compact[key] = [
|
||
cls._compact_schema(
|
||
spec,
|
||
item,
|
||
depth=depth + 1,
|
||
seen_refs=seen_refs,
|
||
budget=budget,
|
||
)
|
||
for item in variants[:10]
|
||
if isinstance(item, dict)
|
||
]
|
||
return compact
|
||
|
||
@classmethod
|
||
def _body_contract(
|
||
cls, spec: dict[str, Any], operation: dict[str, Any]
|
||
) -> Optional[dict[str, Any]]:
|
||
"""统一 Swagger 2 body parameter 与 OpenAPI 3 requestBody。"""
|
||
for param in operation.get("parameters") or []:
|
||
if isinstance(param, dict) and param.get("in") == "body":
|
||
return {
|
||
"parameter_name": param.get("name") or "body",
|
||
"required": bool(param.get("required")),
|
||
"content_type": "application/json",
|
||
"schema": cls._compact_schema(spec, param.get("schema")),
|
||
}
|
||
request_body = operation.get("request_body") or operation.get("requestBody")
|
||
if not isinstance(request_body, dict):
|
||
return None
|
||
content = request_body.get("content")
|
||
if not isinstance(content, dict) or not content:
|
||
return None
|
||
content_type = (
|
||
"application/json" if "application/json" in content else next(iter(content))
|
||
)
|
||
media = content.get(content_type)
|
||
if not isinstance(media, dict):
|
||
return None
|
||
return {
|
||
"required": bool(request_body.get("required")),
|
||
"content_type": content_type,
|
||
"schema": cls._compact_schema(spec, media.get("schema")),
|
||
}
|
||
|
||
@staticmethod
|
||
def _error_detail(response: httpx.Response) -> str:
|
||
"""返回限长、递归脱敏的上游错误详情，帮助模型停止盲猜参数。"""
|
||
|
||
def redact(value: Any) -> Any:
|
||
if isinstance(value, dict):
|
||
return {
|
||
str(key): "[REDACTED]"
|
||
if _SENSITIVE_KEY_RE.search(str(key))
|
||
else redact(item)
|
||
for key, item in value.items()
|
||
}
|
||
if isinstance(value, list):
|
||
return [redact(item) for item in value]
|
||
return value
|
||
|
||
try:
|
||
detail = json.dumps(
|
||
redact(response.json()), ensure_ascii=False, default=str
|
||
)
|
||
except ValueError:
|
||
detail = response.text.strip()
|
||
if not detail:
|
||
return ""
|
||
if len(detail) > _ERROR_DETAIL_MAX_CHARS:
|
||
detail = detail[:_ERROR_DETAIL_MAX_CHARS] + "...[truncated]"
|
||
return detail
|
||
|
||
def _spec_base_path(self, spec: dict[str, Any]) -> str:
|
||
"""Return the API path prefix declared by Swagger 2 / OpenAPI 3.
|
||
|
||
The remote specification may describe a different host, but an external
|
||
system definition is the only authority allowed to choose the target
|
||
origin. OpenAPI ``servers`` therefore contributes only a same-origin
|
||
path prefix.
|
||
"""
|
||
raw_base_path = spec.get("basePath")
|
||
if raw_base_path is not None:
|
||
if not isinstance(raw_base_path, str) or not raw_base_path.startswith("/"):
|
||
raise OpenApiError("Swagger basePath 必须是站内绝对路径")
|
||
parsed = urlparse(raw_base_path)
|
||
if (
|
||
parsed.netloc
|
||
or parsed.query
|
||
or parsed.fragment
|
||
or "://" in raw_base_path
|
||
):
|
||
raise OpenApiError("Swagger basePath 非法")
|
||
return parsed.path.rstrip("/")
|
||
|
||
servers = spec.get("servers")
|
||
if not isinstance(servers, list) or not servers:
|
||
return ""
|
||
server = servers[0]
|
||
raw_url = server.get("url") if isinstance(server, dict) else None
|
||
if not isinstance(raw_url, str) or not raw_url.strip():
|
||
raise OpenApiError("OpenAPI server URL 无效")
|
||
raw_url = raw_url.strip()
|
||
if "{" in raw_url or "}" in raw_url:
|
||
raise OpenApiError("OpenAPI server URL 包含未解析变量")
|
||
declared = urlparse(raw_url)
|
||
base = urlparse(self.cfg.base_url)
|
||
if declared.netloc and (declared.scheme, declared.netloc) != (
|
||
base.scheme,
|
||
base.netloc,
|
||
):
|
||
raise OpenApiError("OpenAPI server 越出 Factory MES 主机")
|
||
if declared.query or declared.fragment:
|
||
raise OpenApiError("OpenAPI server URL 不能包含查询或片段")
|
||
return ("/" + declared.path.lstrip("/")).rstrip("/")
|
||
|
||
def _operation_url(self, spec: dict[str, Any], operation_path: str) -> str:
|
||
prefix = self._spec_base_path(spec)
|
||
base = urlparse(self.cfg.base_url)
|
||
configured_path = base.path.rstrip("/")
|
||
path = operation_path
|
||
configured_has_prefix = bool(prefix) and (
|
||
configured_path == prefix or configured_path.endswith(prefix)
|
||
)
|
||
operation_has_prefix = bool(prefix) and (
|
||
path == prefix or path.startswith(prefix + "/")
|
||
)
|
||
if configured_has_prefix and operation_has_prefix:
|
||
path = path[len(prefix) :] or "/"
|
||
elif prefix and not configured_has_prefix and not operation_has_prefix:
|
||
path = prefix + "/" + path.lstrip("/")
|
||
combined_path = "/".join(
|
||
part.strip("/") for part in (configured_path, path) if part.strip("/")
|
||
)
|
||
if operation_path.endswith("/") and combined_path:
|
||
combined_path += "/"
|
||
url = urljoin(f"{base.scheme}://{base.netloc}/", combined_path)
|
||
call_origin = urlparse(url)
|
||
if (call_origin.scheme, call_origin.netloc) != (base.scheme, base.netloc):
|
||
raise OpenApiError("接口目标越出管理员配置的主机")
|
||
return url
|
||
|
||
def test_connection(self) -> dict[str, Any]:
|
||
RUNTIME_CACHE.invalidate_auth(self._runtime_identity)
|
||
RUNTIME_CACHE.invalidate_spec(self._runtime_identity)
|
||
with self._runtime_client() as client:
|
||
headers = self.authenticate(client=client)
|
||
spec = self._fetch_spec(headers, client=client)
|
||
return {"operation_count": len(self._operations(spec))}
|
||
|
||
def _operation_allowed(self, operation: dict[str, Any]) -> bool:
|
||
if self.cfg.operation_mode == "upstream_managed":
|
||
return True
|
||
method = operation["method"].lower()
|
||
return method in {"get", "head"} or (
|
||
method == "post"
|
||
and self.cfg.operation_policies.get(operation["operation_id"])
|
||
in {"read", "export"}
|
||
)
|
||
|
||
def search(self, query: str, limit: int = 12) -> list[dict[str, Any]]:
|
||
query = (query or "").strip().lower()
|
||
if not query:
|
||
raise OpenApiError("query 不能为空")
|
||
with self._runtime_client() as client:
|
||
headers = self.authenticate(client=client)
|
||
spec = self._fetch_spec(headers, client=client)
|
||
terms = list(
|
||
dict.fromkeys(
|
||
[query] + [x for x in re.split(r"[\s,，。/]+", query) if len(x) >= 2]
|
||
)
|
||
)
|
||
recommended_order = {
|
||
operation_id: index
|
||
for index, operation_id in enumerate(self.cfg.recommended_operation_ids)
|
||
}
|
||
scored: list[tuple[int, int, dict[str, Any]]] = []
|
||
for op in self._operations(spec):
|
||
if not self._operation_allowed(op):
|
||
continue
|
||
hay = " ".join(
|
||
[
|
||
op["operation_id"],
|
||
op["path"],
|
||
op["summary"],
|
||
op["description"],
|
||
" ".join(str(x) for x in op["tags"]),
|
||
]
|
||
).lower()
|
||
score = sum(
|
||
5 if term == query and term in hay else 1
|
||
for term in terms
|
||
if term in hay
|
||
)
|
||
recommended = op["operation_id"] in recommended_order
|
||
if score or recommended:
|
||
compact = dict(op)
|
||
compact["parameters"] = [
|
||
{
|
||
"name": p.get("name"),
|
||
"in": p.get("in"),
|
||
"required": bool(p.get("required")),
|
||
"type": p.get("type") or (p.get("schema") or {}).get("type"),
|
||
"description": p.get("description") or "",
|
||
}
|
||
for p in op["parameters"]
|
||
if isinstance(p, dict) and "$ref" not in p
|
||
]
|
||
body_contract = self._body_contract(spec, op)
|
||
if body_contract is not None:
|
||
compact["body"] = body_contract
|
||
compact.pop("request_body", None)
|
||
compact["recommended"] = recommended
|
||
scored.append(
|
||
(
|
||
0 if recommended else 1,
|
||
recommended_order.get(op["operation_id"], -score),
|
||
compact,
|
||
)
|
||
)
|
||
scored.sort(key=lambda item: (item[0], item[1], item[2]["operation_id"]))
|
||
return [item[2] for item in scored[: max(1, min(int(limit), 30))]]
|
||
|
||
def call(
|
||
self,
|
||
operation_id: str,
|
||
arguments: Optional[dict[str, Any]] = None,
|
||
body: Any = None,
|
||
) -> dict[str, Any]:
|
||
with self._runtime_client() as client:
|
||
return self._call_with_client(client, operation_id, arguments, body)
|
||
|
||
def _call_with_client(
|
||
self,
|
||
client: httpx.Client,
|
||
operation_id: str,
|
||
arguments: Optional[dict[str, Any]],
|
||
body: Any,
|
||
) -> dict[str, Any]:
|
||
headers = self.authenticate(client=client)
|
||
spec = self._fetch_spec(headers, client=client)
|
||
headers = self.authenticate(client=client)
|
||
matches = [
|
||
op for op in self._operations(spec) if op["operation_id"] == operation_id
|
||
]
|
||
if len(matches) != 1:
|
||
raise OpenApiError("operation_id 不存在或不唯一，请先搜索接口")
|
||
op = matches[0]
|
||
if not op["path"].startswith("/") or "://" in op["path"]:
|
||
raise OpenApiError("OpenAPI operation path 非法")
|
||
method = op["method"].lower()
|
||
if not self._operation_allowed(op):
|
||
raise OpenApiError(f"operation {operation_id} 未列入只读调用范围")
|
||
|
||
supplied = dict(arguments or {})
|
||
path = op["path"]
|
||
query: dict[str, Any] = {}
|
||
parameter_headers: dict[str, str] = {}
|
||
cookies: dict[str, str] = {}
|
||
request_body = body
|
||
for param in op["parameters"]:
|
||
if not isinstance(param, dict) or "$ref" in param:
|
||
continue
|
||
name = param.get("name")
|
||
location = param.get("in")
|
||
if not isinstance(name, str):
|
||
continue
|
||
# Swagger 2 的 body 参数既可按搜索结果中的参数名放在 arguments，
|
||
# 也可使用元工具独立的 body 字段；两者只取一个。
|
||
present = name in supplied or (
|
||
location == "body" and request_body is not None
|
||
)
|
||
if param.get("required") and not present:
|
||
raise OpenApiError(f"缺少必填参数: {name}")
|
||
if name not in supplied:
|
||
continue
|
||
value = supplied.pop(name)
|
||
parameter_schema = param.get("schema") or {
|
||
key: param[key] for key in ("type", "enum", "items") if key in param
|
||
}
|
||
try:
|
||
validate_json_value(
|
||
spec,
|
||
value,
|
||
parameter_schema,
|
||
path=f"arguments.{name}",
|
||
)
|
||
except ValueError as exc:
|
||
raise OpenApiError(str(exc)) from exc
|
||
if location == "path":
|
||
path = path.replace("{" + name + "}", quote(str(value), safe=""))
|
||
elif location == "query":
|
||
if name == "page_size":
|
||
try:
|
||
value = max(1, min(int(value), self.cfg.max_page_size))
|
||
except (TypeError, ValueError) as exc:
|
||
raise OpenApiError("page_size 必须是整数") from exc
|
||
elif name == "page" and str(value).strip() == "0":
|
||
raise OpenApiError(
|
||
"外部系统查询不允许 page=0 关闭分页，请使用 dataset 或分页查看明细"
|
||
)
|
||
elif name == "pageoff" and _bool_value(value, False):
|
||
raise OpenApiError(
|
||
"外部系统查询不允许关闭分页，请使用 dataset 或分页查看明细"
|
||
)
|
||
if isinstance(value, list):
|
||
raw_collection_format = param.get("collectionFormat")
|
||
collection_format = (
|
||
raw_collection_format
|
||
if isinstance(raw_collection_format, str)
|
||
else ""
|
||
)
|
||
raw_style = param.get("style")
|
||
style = raw_style if isinstance(raw_style, str) else ""
|
||
explode = param.get("explode", True)
|
||
separators = {
|
||
"csv": ",",
|
||
"ssv": " ",
|
||
"tsv": "\t",
|
||
"pipes": "|",
|
||
"spaceDelimited": " ",
|
||
"pipeDelimited": "|",
|
||
}
|
||
separator = separators.get(collection_format) or separators.get(
|
||
style
|
||
)
|
||
if separator:
|
||
query[name] = separator.join(str(item) for item in value)
|
||
elif style == "form" and explode is False:
|
||
query[name] = ",".join(str(item) for item in value)
|
||
else:
|
||
query[name] = value
|
||
else:
|
||
query[name] = value
|
||
elif location == "header":
|
||
if _SENSITIVE_KEY_RE.search(name):
|
||
raise OpenApiError(f"接口参数不允许覆盖敏感 Header: {name}")
|
||
parameter_headers[name] = str(value)
|
||
elif location == "cookie":
|
||
if _SENSITIVE_KEY_RE.search(name):
|
||
raise OpenApiError(f"接口参数不允许覆盖敏感 Cookie: {name}")
|
||
cookies[name] = str(value)
|
||
elif location == "body" and request_body is None:
|
||
request_body = value
|
||
elif location not in {"body"}:
|
||
raise OpenApiError(f"暂不支持参数位置: {location}")
|
||
if supplied:
|
||
raise OpenApiError("存在接口定义之外的参数: " + ", ".join(sorted(supplied)))
|
||
if "{" in path or "}" in path:
|
||
raise OpenApiError("路径参数未完整提供")
|
||
body_contract = self._body_contract(spec, op)
|
||
if request_body is not None and body_contract is not None:
|
||
if "json" not in str(body_contract.get("content_type") or "").lower():
|
||
raise OpenApiError("当前连接器仅支持 JSON 请求体")
|
||
try:
|
||
validate_json_value(spec, request_body, body_contract.get("schema"))
|
||
except ValueError as exc:
|
||
raise OpenApiError(str(exc)) from exc
|
||
elif body_contract and body_contract.get("required"):
|
||
raise OpenApiError("缺少必填请求体")
|
||
|
||
url = self._operation_url(spec, path)
|
||
|
||
def execute_request() -> dict[str, Any]:
|
||
request_headers = {**headers, **parameter_headers}
|
||
try:
|
||
_, auth_generation = RUNTIME_CACHE.auth_state(self._runtime_identity)
|
||
status_code, response_headers, content = self._limited_request(
|
||
client,
|
||
method.upper(),
|
||
url,
|
||
params=query,
|
||
json=request_body if request_body is not None else None,
|
||
headers=request_headers,
|
||
cookies=cookies,
|
||
limit=MAX_STORED_RESULT_BYTES,
|
||
label="外部系统响应",
|
||
)
|
||
if status_code == 401 and self.cfg.auth_type == "password_jwt":
|
||
refreshed = self._refresh_auth(
|
||
client, failed_generation=auth_generation
|
||
)
|
||
status_code, response_headers, content = self._limited_request(
|
||
client,
|
||
method.upper(),
|
||
url,
|
||
params=query,
|
||
json=request_body if request_body is not None else None,
|
||
headers={**refreshed, **parameter_headers},
|
||
cookies=cookies,
|
||
limit=MAX_STORED_RESULT_BYTES,
|
||
label="外部系统响应",
|
||
)
|
||
except httpx.HTTPError as exc:
|
||
raise OpenApiError(
|
||
f"外部系统接口调用失败: {type(exc).__name__}"
|
||
) from exc
|
||
response = httpx.Response(
|
||
status_code,
|
||
headers=response_headers,
|
||
content=content,
|
||
)
|
||
if status_code >= 400:
|
||
detail = self._error_detail(response)
|
||
suffix = f": {detail}" if detail else ""
|
||
raise OpenApiError(f"外部系统接口返回 HTTP {status_code}{suffix}")
|
||
content_type = response_headers.get("content-type", "")
|
||
try:
|
||
payload: Any = (
|
||
json.loads(content.decode("utf-8-sig"))
|
||
if "json" in content_type
|
||
else content.decode("utf-8", errors="replace")
|
||
)
|
||
except ValueError:
|
||
payload = content.decode("utf-8", errors="replace")
|
||
encoded = json.dumps(payload, ensure_ascii=False, default=str)
|
||
response_bytes = len(encoded.encode("utf-8"))
|
||
if response_bytes > MAX_STORED_RESULT_BYTES:
|
||
raise OpenApiError(
|
||
f"外部系统响应超过安全下载上限({MAX_STORED_RESULT_BYTES} bytes)，"
|
||
"请缩小查询范围或使用远端分页/聚合接口"
|
||
)
|
||
return {
|
||
"operation_id": operation_id,
|
||
"status_code": status_code,
|
||
"truncated": False,
|
||
"response_bytes": response_bytes,
|
||
"data": payload,
|
||
}
|
||
|
||
query_like = method in {"get", "head"} or (
|
||
method == "post"
|
||
and self.cfg.operation_policies.get(operation_id) in {"read", "export"}
|
||
)
|
||
if not query_like:
|
||
return execute_request()
|
||
request_fingerprint = hashlib.sha256(
|
||
json.dumps(
|
||
{
|
||
"method": method,
|
||
"url": url,
|
||
"query": query,
|
||
"headers": parameter_headers,
|
||
"cookies": cookies,
|
||
"body": request_body,
|
||
},
|
||
sort_keys=True,
|
||
ensure_ascii=False,
|
||
default=str,
|
||
).encode("utf-8")
|
||
).hexdigest()
|
||
result = RUNTIME_CACHE.singleflight(
|
||
"query",
|
||
f"{self._runtime_identity}:{request_fingerprint}",
|
||
execute_request,
|
||
)
|
||
return copy.deepcopy(result)
|