diff --git a/config/media/unifyllm.yaml b/config/media/unifyllm.yaml index 79675c4..72360ef 100644 --- a/config/media/unifyllm.yaml +++ b/config/media/unifyllm.yaml @@ -9,9 +9,8 @@ # HTTP 400 "images endpoint requires an image model"。 # - gpt-image-2 支持 size / quality:尺寸可用 auto 或满足约束的 WIDTHxHEIGHT, # quality 可用 auto / low / medium / high。输出固定 PNG,透明背景当前模型不支持。 -# - 2026-07-31 实测网关 /images/edits 暂不可用:官方 JSON 返回 -# convert_request_failed,官方 multipart(image[] / image)均返回 NextPart: EOF; -# Responses API 也静默丢弃强制 image_generation tool。网关修复前不暴露改图参数。 +# - 2026-08-03 实测 gpt-image-2 已可通过 /images/edits multipart 单图改图, +# 响应同时含 b64_json 与 data URL;当前账号未开放 gpt-image-2-ad。 # - 复杂图片可能需 ~2min(慢于 seedream) # - 价格:网关未公布价目,price 暂 0(usage tokens 记进 units,拿到价目后回填对账) # - 服务器需代理出口(直连 unifyllm.ai TLS 失败),同文本模型 @@ -24,6 +23,7 @@ image: model_id: gpt-image-2 display_name: GPT 生图 endpoint: /images/generations + edit_endpoint: /images/edits default_size: auto default_quality: auto price_cny_per_image: 0 # 网关价目未知,成本先记 0;拿到价目改这里 + 重启 diff --git a/core/agent_builder.py b/core/agent_builder.py index 044a058..cdeb63c 100644 --- a/core/agent_builder.py +++ b/core/agent_builder.py @@ -74,7 +74,8 @@ _MEDIA_SEEDREAM_SEG = """\ - 兜底硬约束(即使没 load skill 也守):用户没主动要图就别装饰性生成;同一目的不满意**不要连发**,先口头校准 prompt 再调。用户消息里出现 `[用户上传的参考图] <路径>` = 用户贴了图,要看图 / 改图时用那个路径。""" _MEDIA_GPT_IMAGE_SEG = """\ - `gpt_image` —— GPT 图像生成(本 run 用户在顶栏选了「GPT 生图」,seedream 不可用;其他地方提到 seedream 的指引按 gpt_image 理解)。产物自动落 `/figures/`,复杂图可能需 **~2min**(慢,调用前告知用户稍等)。 - - **仅文生图**:支持 `size`(`auto` 或合法 `WIDTHxHEIGHT`)与 `quality`(`auto/low/medium/high`)。当前网关改图端点不可用;用户要修改已有图片时明确说明,并建议在顶栏切回「豆包 Seedream」。 + - **文生图**(不传 `reference_images`):从零按 prompt 画。**改图 i2i**(传 `reference_images=["图片路径"]`):基于单张已有图片修改,原图保留、结果另存。用户要修改刚生成/上传的图时必须走改图,不要重新文生图。 + - **改图调用前必须明确提示用户**:将基于「参考图文件名」编辑,会消耗 1 次图片额度,预计 1–2 分钟,原图保留且结果另存;提示后再调工具。支持 `size` 与 `quality`(`auto/low/medium/high`),当前仅支持单张参考图。 - **调用前必须先 `load_skill('imagegen')`** —— 其中「何时该用 / mermaid 反向选型 / 模糊度诊断 / prompt 装配 / 先给用户过目再调」的流程完全适用;参数以本工具 schema 为准。 - 兜底硬约束(即使没 load skill 也守):用户没主动要图就别装饰性生成;同一目的不满意**不要连发**,先口头校准 prompt 再调。""" _MEDIA_DIAGRAM_FORK_SEG = """\ diff --git a/core/ark_client.py b/core/ark_client.py index d84d9fc..0e02346 100644 --- a/core/ark_client.py +++ b/core/ark_client.py @@ -15,7 +15,6 @@ import yaml from core.paths import ROOT - _DOUBAO_YAML = ROOT / "config" / "media" / "doubao.yaml" @@ -75,7 +74,6 @@ class ArkClient: base_url=cfg.base_url, headers={ "Authorization": f"Bearer {cfg.api_key}", - "Content-Type": "application/json", }, timeout=timeout_s, ) @@ -89,6 +87,28 @@ class ArkClient: raise ArkTimeoutError(f"network error calling POST {path}: {e}") from e return self._parse(resp, f"POST {path}") + def post_multipart( + self, + path: str, + data: dict[str, str], + files: dict[str, tuple[str, bytes, str]], + *, + timeout_s: float | None = None, + ) -> dict: + """POST multipart/form-data；边界与 Content-Type 交给 httpx 生成。""" + try: + resp = self._client.post( + path, + data=data, + files=files, + timeout=timeout_s or self.timeout_s, + ) + except httpx.TimeoutException as e: + raise ArkTimeoutError(f"timeout calling POST {path}: {e}") from e + except httpx.HTTPError as e: + raise ArkTimeoutError(f"network error calling POST {path}: {e}") from e + return self._parse(resp, f"POST {path}") + def get_json(self, path: str, *, timeout_s: Optional[float] = None) -> dict: try: resp = self._client.get(path, timeout=timeout_s or self.timeout_s) diff --git a/scripts/test_gpt_image_manual.py b/scripts/test_gpt_image_manual.py index c1a881c..bfbf452 100644 --- a/scripts/test_gpt_image_manual.py +++ b/scripts/test_gpt_image_manual.py @@ -28,8 +28,8 @@ def main() -> int: dry = "--dry" in sys.argv fails = 0 - from core.ark_client import ArkConfig from core.agent_builder import _choose_image_variant, _media_tools_block + from core.ark_client import ArkConfig gw = ArkConfig.load(ROOT / "config" / "media" / "unifyllm.yaml") ark = ArkConfig.load() @@ -44,6 +44,12 @@ def main() -> int: f"{image_cfg.get('model_id') or 'MISSING'}" ) fails += 0 if model_ok else 1 + edit_endpoint_ok = image_cfg.get("edit_endpoint") == "/images/edits" + print( + f"[{'OK' if edit_endpoint_ok else 'FAIL'}] gpt_image edit_endpoint=" + f"{image_cfg.get('edit_endpoint') or 'MISSING'}" + ) + fails += 0 if edit_endpoint_ok else 1 # variant 选择:显式 gpt_image / 显式 seedream_5 / 空 fallback cases = [ @@ -66,8 +72,13 @@ def main() -> int: and "`seedream`" not in blk.split("\n")[0] and "size" in gpt_seg and "quality" in gpt_seg + and "reference_images" in gpt_seg + and "1 次图片额度" in gpt_seg + ) + print( + f"[{'OK' if ok else 'FAIL'}] media block(gpt_image) includes " + "size/quality/edit guidance" ) - print(f"[{'OK' if ok else 'FAIL'}] media block(gpt_image) includes size/quality guidance") fails += 0 if ok else 1 blk2 = _media_tools_block(ark is not None, "seedream") ok = ("- `seedream`" in blk2) and ("- `gpt_image`" not in blk2) diff --git a/tests/test_gpt_image.py b/tests/test_gpt_image.py index a4858bc..03e9c6c 100644 --- a/tests/test_gpt_image.py +++ b/tests/test_gpt_image.py @@ -1,4 +1,5 @@ import base64 +import json import struct import tempfile import unittest @@ -6,7 +7,9 @@ import uuid from pathlib import Path from unittest.mock import patch -from core.ark_client import ArkConfig +import httpx + +from core.ark_client import ArkClient, ArkConfig from tools.gpt_image import GptImageTool @@ -16,6 +19,7 @@ def _png_stub(width: int = 1536, height: int = 864) -> bytes: class _FakeArkClient: json_call = None + multipart_call = None def __init__(self, *_args, **_kwargs): pass @@ -33,10 +37,51 @@ class _FakeArkClient: "usage": {"output_tokens": 123}, } + def post_multipart(self, endpoint, data, files, *, timeout_s=None): + type(self).multipart_call = (endpoint, data, files, timeout_s) + return { + "data": [{"b64_json": base64.b64encode(_png_stub()).decode("ascii")}], + } + + +class ArkClientMultipartTests(unittest.TestCase): + def test_json_and_multipart_set_their_own_content_types(self): + seen = [] + + def handler(request): + seen.append(request) + return httpx.Response(200, json={"ok": True}) + + client = ArkClient( + ArkConfig(api_key="test", base_url="https://example.test/v1", raw={}) + ) + client._client.close() + client._client = httpx.Client( + base_url="https://example.test/v1", + headers={"Authorization": "Bearer test"}, + transport=httpx.MockTransport(handler), + ) + try: + client.post_json("/images/generations", {"prompt": "draw"}) + client.post_multipart( + "/images/edits", + {"prompt": "edit"}, + {"image": ("reference.png", b"png-bytes", "image/png")}, + ) + finally: + client.close() + + self.assertEqual(seen[0].headers["content-type"], "application/json") + self.assertTrue(seen[1].headers["content-type"].startswith("multipart/form-data;")) + self.assertIn(b'name="prompt"', seen[1].content) + self.assertIn(b'filename="reference.png"', seen[1].content) + self.assertIn(b"png-bytes", seen[1].content) + class GptImageToolTests(unittest.TestCase): def setUp(self): _FakeArkClient.json_call = None + _FakeArkClient.multipart_call = None self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) self.working_dir = self.root / "task" @@ -44,6 +89,7 @@ class GptImageToolTests(unittest.TestCase): self.cfg = { "model_id": "gpt-image-2", "endpoint": "/images/generations", + "edit_endpoint": "/images/edits", "default_size": "auto", "default_quality": "auto", "request_timeout_s": 300, @@ -86,6 +132,69 @@ class GptImageToolTests(unittest.TestCase): self.assertEqual(body["quality"], "high") self.assertIn("size=1536x864", result) self.assertIn("quality=high", result) + self.assertIsNone(_FakeArkClient.multipart_call) + + def test_image_edit_uses_multipart_and_records_derivation(self): + reference = self.working_dir / "reference.png" + reference.write_bytes(_png_stub(180, 252)) + + result = self._execute( + prompt="add a blue border", + reference_images=["reference.png"], + size="1024x1024", + quality="low", + ) + + self.assertIsNone(_FakeArkClient.json_call) + endpoint, form, files, timeout = _FakeArkClient.multipart_call + self.assertEqual(endpoint, "/images/edits") + self.assertEqual(timeout, 300) + self.assertEqual(form["model"], "gpt-image-2") + self.assertEqual(form["prompt"], "add a blue border") + self.assertEqual(form["n"], "1") + self.assertEqual(form["size"], "1024x1024") + self.assertEqual(form["quality"], "low") + self.assertEqual(form["response_format"], "b64_json") + filename, raw, mime = files["image"] + self.assertEqual(filename, "reference.png") + self.assertEqual(raw, reference.read_bytes()) + self.assertEqual(mime, "image/png") + self.assertIn("mode=i2i", result) + self.assertIn("reference=", result) + + meta_path = next((self.working_dir / "figures").glob("*.meta.json")) + meta = json.loads(meta_path.read_text(encoding="utf-8")) + self.assertEqual(meta["mode"], "i2i") + self.assertEqual(meta["reference_images"], ["task/reference.png"]) + + def test_image_edit_rejects_multiple_or_missing_references(self): + result = self._execute( + prompt="edit", + reference_images=["one.png", "two.png"], + ) + self.assertIn("仅支持单张", result) + self.assertIsNone(_FakeArkClient.multipart_call) + + result = self._execute(prompt="edit", reference_images=["missing.png"]) + self.assertIn("图片找不到或越界", result) + self.assertIsNone(_FakeArkClient.multipart_call) + + def test_image_edit_accepts_data_url_response_fallback(self): + reference = self.working_dir / "reference.png" + reference.write_bytes(_png_stub(180, 252)) + encoded = base64.b64encode(_png_stub()).decode("ascii") + + with patch.object( + _FakeArkClient, + "post_multipart", + return_value={"data": [{"url": f"data:image/png;base64,{encoded}"}]}, + ): + result = self._execute( + prompt="edit", + reference_images=["reference.png"], + ) + + self.assertTrue(result.startswith("[gpt_image]")) def test_size_validation(self): self.assertEqual(self.tool._normalize_size("auto"), ("auto", "")) diff --git a/tools/gpt_image.py b/tools/gpt_image.py index 04bfc62..088cede 100644 --- a/tools/gpt_image.py +++ b/tools/gpt_image.py @@ -1,9 +1,9 @@ -"""gpt_image: 调 unifyllm 网关的 OpenAI Images API 生图,产物落 working_dir/figures/。 +"""gpt_image: 调 unifyllm 网关的 OpenAI Images API 生图 / 改图。 第二图像后端(第一个是豆包 seedream):模型 ID + 单价全在 `config/media/unifyllm.yaml`, 本 tool 只装配: - 文生图走 /images/generations JSON,gpt-image-2 支持自定义尺寸和质量档; -- 当前 unifyllm 网关的 /images/edits 不可用,暂不暴露改图参数; +- 单图改图走 /images/edits multipart/form-data; - 响应直接返 b64_json,无需二次下载; - 复杂图片可能需 ~2min。 完成后: @@ -23,11 +23,12 @@ from pathlib import Path from typing import Optional from uuid import UUID -from core.artifacts import ArtifactRef, ToolExecutionResult, resolve_artifact_path from core.ark_client import ArkClient, ArkConfig, ArkError +from core.artifacts import ArtifactRef, ToolExecutionResult, resolve_artifact_path from core.storage.usage import record_image_usage from .base import Tool +from .image_ref import load_image_as_data_url from .media_common import quota_gate, record_usage_safe, stamped_path, write_meta @@ -35,10 +36,10 @@ class GptImageTool(Tool): name = "gpt_image" description = ( "Generate an image via GPT Image, saved to working_dir/figures/. Supports custom size " - "and quality. Complex images may take up to ~2 minutes. Image-to-image editing is not " - "available through the current gateway; ask the user to switch to 豆包 Seedream for " - "editing an existing image. Don't generate decoratively — only when the user actually " - "wants an image. Returns the saved relative path." + "and quality, plus single-reference image editing. For editing, pass reference_images " + "with the existing image path; the original is preserved and the result is saved as a " + "new image. Complex images may take up to ~2 minutes. Don't generate decoratively — " + "only when the user actually wants an image. Returns the saved relative path." ) parameters = { "type": "object", @@ -47,6 +48,14 @@ class GptImageTool(Tool): "type": "string", "description": "中文或英文都行,详尽描述画面(主体/风格/光线/构图)。", }, + "reference_images": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "改图(image-to-image):传 1 张 task_dir 内已存在图片的相对路径。" + "不传 = 从零文生图;当前仅支持单张参考图。" + ), + }, "size": { "type": "string", "description": ( @@ -89,12 +98,36 @@ class GptImageTool(Tool): def execute( self, prompt: str, + reference_images: list | None = None, size: Optional[str] = None, quality: Optional[str] = None, ) -> str: if not (prompt or "").strip(): return "[Error] prompt 不能为空" + refs = [str(r).strip() for r in (reference_images or []) if str(r).strip()] + if len(refs) > 1: + return ( + "[Error] reference_images 当前仅支持单张参考图(传了 " + f"{len(refs)} 张)。请只传 1 张。" + ) + ref_bytes = b"" + ref_mime = "" + ref_disp = "" + if refs: + data_url, ref_disp, ref_err = load_image_as_data_url( + refs[0], + working_dir=self.working_dir, + user_root=self.user_root, + display_fn=self._display, + ) + if ref_err: + return ref_err + header, encoded = data_url.split(",", 1) + ref_mime = header.removeprefix("data:").removesuffix(";base64") + ref_bytes = base64.b64decode(encoded) + is_i2i = bool(ref_bytes) + cfg = self.cfg chosen_size, size_err = self._normalize_size( size if size is not None else cfg.get("default_size", "auto") @@ -115,7 +148,11 @@ class GptImageTool(Tool): return quota_err model_id = cfg["model_id"] - endpoint = cfg.get("endpoint", "/images/generations") + endpoint = ( + cfg.get("edit_endpoint", "/images/edits") + if is_i2i + else cfg.get("endpoint", "/images/generations") + ) timeout_s = float(cfg.get("request_timeout_s", 300)) price = float(cfg.get("price_cny_per_image", 0)) @@ -130,7 +167,20 @@ class GptImageTool(Tool): t0 = time.monotonic() try: with ArkClient(self.gw_cfg, timeout_s=timeout_s) as client: - resp = client.post_json(endpoint, body, timeout_s=timeout_s) + if is_i2i: + form = { + key: str(value).lower() if isinstance(value, bool) else str(value) + for key, value in body.items() + } + form["response_format"] = "b64_json" + resp = client.post_multipart( + endpoint, + form, + {"image": (Path(refs[0]).name or "reference.png", ref_bytes, ref_mime)}, + timeout_s=timeout_s, + ) + else: + resp = client.post_json(endpoint, body, timeout_s=timeout_s) except ArkError as e: return f"[Error] gpt_image API: {e}" @@ -138,6 +188,10 @@ class GptImageTool(Tool): b64 = "" if isinstance(data, list) and data and isinstance(data[0], dict): b64 = data[0].get("b64_json") or "" + if not b64: + image_url = str(data[0].get("url") or "") + if image_url.startswith("data:") and ";base64," in image_url: + b64 = image_url.split(",", 1)[1] if not b64: return f"[Error] gpt_image response 缺 b64_json: {json.dumps(resp, ensure_ascii=False)[:300]}" try: @@ -162,7 +216,8 @@ class GptImageTool(Tool): "requested_size": chosen_size, "quality": actual_quality, "requested_quality": chosen_quality, - "mode": "t2i", + "mode": "i2i" if is_i2i else "t2i", + "reference_images": [ref_disp] if is_i2i else [], "cost_cny": price, "output_tokens": output_tokens, "elapsed_s": round(elapsed, 2), @@ -185,10 +240,12 @@ class GptImageTool(Tool): # 首行 banner 协议同 seedream(`key=value · ` 分隔,前端 extractMediaBanner 解析); # 价格未知(price=0)时不放 cost 段,避免"¥0.00 = 免费"的误导。 cost_seg = f" · cost=¥{price:.2f}" if price > 0 else "" + mode_seg = " · mode=i2i" if is_i2i else "" + ref_line = f"\nreference={ref_disp}" if is_i2i else "" result = ( f"[gpt_image] model={model_id} · size={actual_size} · quality={actual_quality}" - f"{cost_seg} · elapsed={elapsed:.1f}s\n" - f"saved: {disp}\n" + f"{cost_seg} · elapsed={elapsed:.1f}s{mode_seg}\n" + f"saved: {disp}{ref_line}\n" f"prompt={prompt!r}" ) if self.user_root is None: