From c4418711abdb5be50160faa58842978b9cbab95e Mon Sep 17 00:00:00 2001 From: caoqianming Date: Thu, 13 Aug 2026 09:59:49 +0800 Subject: [PATCH] =?UTF-8?q?perf(vision):=20=E6=94=B6=E6=95=9B=E7=9C=8B?= =?UTF-8?q?=E5=9B=BE=E8=AF=B7=E6=B1=82=E8=8C=83=E5=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/agent_builder.py | 3 +- tests/test_look_at_image.py | 61 +++++++++++++++++++++++++++++++++++++ tools/look_at_image.py | 22 ++++++------- 3 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 tests/test_look_at_image.py diff --git a/core/agent_builder.py b/core/agent_builder.py index c4dda02..b390446 100644 --- a/core/agent_builder.py +++ b/core/agent_builder.py @@ -53,7 +53,8 @@ from tools.office_to_pdf import soffice_available # 二选一(seedream=豆包 / gpt_image=unifyllm 网关),同一 run 只挂一个图像工具。 _MEDIA_LOOK_SEG = """\ - `look_at_image` —— 看图 / 读图(豆包 Seed 2.0 Lite 视觉)。**你(主模型)是纯文本看不见图,要"看"图就调它**:OCR 文字、描述画面、读图表/表格/示意图、识别物体。**很慢(一次要几十秒),谨慎调用** —— 只在确实需要图的实际内容才调。 - - **何时调**:用户消息里出现 `[用户上传的参考图] <路径>` 且需要据图内容回答(问"这图里写了啥 / 是什么 / 表格数据多少");或要基于 task 内某张图(`figures/xxx.png`)的**实际内容**做事(不是改图,改图走 seedream)。传 `image=<路径>` + 可选 `question`,**把想知道的一次在 `question` 里问全**,别分多次看。 + - **何时调**:用户消息里出现 `[用户上传的参考图] <路径>` 且需要据图内容回答(问"这图里写了啥 / 是什么 / 表格数据多少");或要基于 task 内某张图(`figures/xxx.png`)的**实际内容**做事(不是改图,改图走 seedream)。传 `image=<路径>`，并且**必须在 `question` 里一次写清完成当前任务所需的具体信息**，别分多次看。 + - **问题范围**:只问完成用户任务所需内容。例如问仪表读数就只读数，问某列数据就只提取该列；仅当用户明确要求"全文 OCR / 转文字 / 完整识别整张图"时，才要求逐字识别全部文字。不要无条件同时要求画面描述、全文 OCR 和图表解析。 - **何时不调**:用户只是要改图(走 seedream i2i)/ 只要文件名不关心内容 / 图是你自己刚生成的且 prompt 已知(无需再读)/ 图的内容对当前任务可有可无。**绝不对同一张图反复看**(每看一次都是几十秒等待)。""" _MEDIA_READDOC_SEG = """\ - `read_document` —— 读 PDF(豆包 Seed 2.0 Lite 文档理解),**专治扫描件**:markitdown 对某 PDF 转出**空 / 近空**(纯图页无文本层)→ 用它逐页 OCR 成 markdown。每页约 1-2 厘钱,单次上限 100 页(更长先拆分卷)。 diff --git a/tests/test_look_at_image.py b/tests/test_look_at_image.py new file mode 100644 index 0000000..e7b5bb0 --- /dev/null +++ b/tests/test_look_at_image.py @@ -0,0 +1,61 @@ +"""look_at_image 的问题收敛与兼容兜底测试（不碰网络和数据库）。""" +from __future__ import annotations + +import unittest +import uuid +from pathlib import Path +from unittest import mock + +from core.ark_client import ArkConfig +from tools.look_at_image import LookAtImageTool, _DEFAULT_QUESTION + + +class LookAtImageQuestionTests(unittest.TestCase): + def _tool(self) -> LookAtImageTool: + return LookAtImageTool( + ark_cfg=ArkConfig(api_key="test", base_url="https://example.invalid", raw={}), + vision_variant_cfg={ + "model_id": "vision-test", + "request_timeout_s": 1, + "timeout_retries": 0, + }, + variant_key="test", + working_dir=Path("."), + task_id=uuid.uuid4(), + user_id=uuid.uuid4(), + base_dir=Path("."), + user_root=Path("."), + ) + + def _execute_and_question(self, question=None) -> str: + captured = {} + + def fake_chat(_cfg, _endpoint, body, **_kwargs): + captured["question"] = body["messages"][0]["content"][0]["text"] + return { + "choices": [{"finish_reason": "stop", "message": {"content": "ok"}}], + "usage": {}, + }, "" + + with mock.patch( + "tools.look_at_image.load_image_as_data_url", + return_value=("data:image/png;base64,AA==", "image.png", ""), + ), mock.patch("tools.look_at_image.ark_chat_with_retry", side_effect=fake_chat), \ + mock.patch("tools.look_at_image.record_usage_safe", return_value=0): + self._tool().execute("image.png", question=question) + return captured["question"] + + def test_specific_question_is_forwarded_unchanged(self): + question = "只读出仪表盘当前数值，不要描述其他内容。" + self.assertEqual(self._execute_and_question(question), question) + + def test_missing_question_uses_concise_compatibility_fallback(self): + self.assertEqual(self._execute_and_question(), _DEFAULT_QUESTION) + self.assertIn("简洁", _DEFAULT_QUESTION) + self.assertIn("不要主动全文 OCR", _DEFAULT_QUESTION) + self.assertNotIn("完整描述画面内容", _DEFAULT_QUESTION) + self.assertNotIn("把其中的数据、坐标轴、图例", _DEFAULT_QUESTION) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/look_at_image.py b/tools/look_at_image.py index febbeb1..bced827 100644 --- a/tools/look_at_image.py +++ b/tools/look_at_image.py @@ -21,12 +21,11 @@ from .output import compact_tool_output from .image_ref import load_image_as_data_url from .media_common import ark_chat_with_retry, extract_chat_answer, record_usage_safe -# 不带 question 时的默认提问:全覆盖(描述 + OCR + 图表读数),让模型一次把图里能用的信息都吐出来 +# question 理应由主模型按用户任务具体填写；这里仅作旧调用兼容兜底。 +# 默认只给简洁概览，避免无条件触发「描述 + 全文 OCR + 图表解析」导致长输出和高延迟。 _DEFAULT_QUESTION = ( - "请仔细看这张图并回答:" - "1) 完整描述画面内容(主体/场景/关键元素);" - "2) 如果图中有任何文字,逐字准确 OCR 出来,尽量保留原排版与换行;" - "3) 如果是图表/表格/示意图,把其中的数据、坐标轴、图例、结构关系读出来。" + "请简洁说明这张图片的主要内容，只提供理解图片所必需的信息。" + "不要主动全文 OCR、逐项枚举或展开图表数据；无法确定时明确说明。" ) @@ -36,10 +35,10 @@ class LookAtImageTool(Tool): "Read/understand an image using Doubao Seed 2.0 Lite vision (the main model is text-only). " "Use to OCR text, describe a picture, read charts/tables/diagrams, or identify objects in an " "image the user uploaded (look for a `[用户上传的参考图] ` line in their message) or that " - "was generated/saved in the task. Pass the image path; optionally a specific question " - "(default: describe + OCR everything). SLOW (tens of seconds per call) — only call when you " - "genuinely need the image's actual content to proceed, and ask everything you need in ONE " - "call (bundle all questions into `question`); never re-read the same image. " + "was generated/saved in the task. Pass the image path and a task-specific `question`; " + "only request full-image OCR when the user explicitly needs it. SLOW (tens of seconds per " + "call) — only call when you genuinely need the image's actual content to proceed, and ask " + "everything you need in ONE call; never re-read the same image. " "Returns the model's textual reading of the image." ) parameters = { @@ -55,8 +54,9 @@ class LookAtImageTool(Tool): "question": { "type": "string", "description": ( - "想从图里知道什么(可选)。如「这张图里的表格数据是多少」「图中仪表读数」" - "「把这页文字 OCR 出来」。不传则默认全面描述 + OCR 全部文字。" + "本次任务需要从图里知道什么(强烈建议填写)。如「读出表格中的抗压强度数据」" + "「图中仪表读数是多少」「把这页文字完整 OCR 出来」。只问完成用户任务所需内容；" + "仅当用户明确要求整图识别时才要求全文 OCR。不传则仅返回简洁画面概览。" ), }, },