perf(vision): 收敛看图请求范围
This commit is contained in:
parent
8aa25bbd5a
commit
c4418711ab
|
|
@ -53,7 +53,8 @@ from tools.office_to_pdf import soffice_available
|
|||
# 二选一(seedream=豆包 / gpt_image=unifyllm 网关),同一 run 只挂一个图像工具。
|
||||
_MEDIA_LOOK_SEG = """\
|
||||
- `look_at_image` —— 看图 / 读图(豆包 Seed 2.0 Lite 视觉)。**你(主模型)是纯文本看不见图,要"看"图就调它**:OCR 文字、描述画面、读图表/表格/示意图、识别物体。**很慢(一次要几十秒),谨慎调用** —— 只在确实需要图的实际内容才调。
|
||||
- **何时调**:用户消息里出现 `[用户上传的参考图] <路径>` 且需要据图内容回答(问"这图里写了啥 / 是什么 / 表格数据多少");或要基于 task 内某张图(`figures/xxx.png`)的**实际内容**做事(不是改图,改图走 seedream)。传 `image=<路径>` + 可选 `question`,**把想知道的一次在 `question` 里问全**,别分多次看。
|
||||
- **何时调**:用户消息里出现 `[用户上传的参考图] <路径>` 且需要据图内容回答(问"这图里写了啥 / 是什么 / 表格数据多少");或要基于 task 内某张图(`figures/xxx.png`)的**实际内容**做事(不是改图,改图走 seedream)。传 `image=<路径>`，并且**必须在 `question` 里一次写清完成当前任务所需的具体信息**，别分多次看。
|
||||
- **问题范围**:只问完成用户任务所需内容。例如问仪表读数就只读数，问某列数据就只提取该列；仅当用户明确要求"全文 OCR / 转文字 / 完整识别整张图"时，才要求逐字识别全部文字。不要无条件同时要求画面描述、全文 OCR 和图表解析。
|
||||
- **何时不调**:用户只是要改图(走 seedream i2i)/ 只要文件名不关心内容 / 图是你自己刚生成的且 prompt 已知(无需再读)/ 图的内容对当前任务可有可无。**绝不对同一张图反复看**(每看一次都是几十秒等待)。"""
|
||||
_MEDIA_READDOC_SEG = """\
|
||||
- `read_document` —— 读 PDF(豆包 Seed 2.0 Lite 文档理解),**专治扫描件**:markitdown 对某 PDF 转出**空 / 近空**(纯图页无文本层)→ 用它逐页 OCR 成 markdown。每页约 1-2 厘钱,单次上限 100 页(更长先拆分卷)。
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
"""look_at_image 的问题收敛与兼容兜底测试（不碰网络和数据库）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from core.ark_client import ArkConfig
|
||||
from tools.look_at_image import LookAtImageTool, _DEFAULT_QUESTION
|
||||
|
||||
|
||||
class LookAtImageQuestionTests(unittest.TestCase):
|
||||
def _tool(self) -> LookAtImageTool:
|
||||
return LookAtImageTool(
|
||||
ark_cfg=ArkConfig(api_key="test", base_url="https://example.invalid", raw={}),
|
||||
vision_variant_cfg={
|
||||
"model_id": "vision-test",
|
||||
"request_timeout_s": 1,
|
||||
"timeout_retries": 0,
|
||||
},
|
||||
variant_key="test",
|
||||
working_dir=Path("."),
|
||||
task_id=uuid.uuid4(),
|
||||
user_id=uuid.uuid4(),
|
||||
base_dir=Path("."),
|
||||
user_root=Path("."),
|
||||
)
|
||||
|
||||
def _execute_and_question(self, question=None) -> str:
|
||||
captured = {}
|
||||
|
||||
def fake_chat(_cfg, _endpoint, body, **_kwargs):
|
||||
captured["question"] = body["messages"][0]["content"][0]["text"]
|
||||
return {
|
||||
"choices": [{"finish_reason": "stop", "message": {"content": "ok"}}],
|
||||
"usage": {},
|
||||
}, ""
|
||||
|
||||
with mock.patch(
|
||||
"tools.look_at_image.load_image_as_data_url",
|
||||
return_value=("data:image/png;base64,AA==", "image.png", ""),
|
||||
), mock.patch("tools.look_at_image.ark_chat_with_retry", side_effect=fake_chat), \
|
||||
mock.patch("tools.look_at_image.record_usage_safe", return_value=0):
|
||||
self._tool().execute("image.png", question=question)
|
||||
return captured["question"]
|
||||
|
||||
def test_specific_question_is_forwarded_unchanged(self):
|
||||
question = "只读出仪表盘当前数值，不要描述其他内容。"
|
||||
self.assertEqual(self._execute_and_question(question), question)
|
||||
|
||||
def test_missing_question_uses_concise_compatibility_fallback(self):
|
||||
self.assertEqual(self._execute_and_question(), _DEFAULT_QUESTION)
|
||||
self.assertIn("简洁", _DEFAULT_QUESTION)
|
||||
self.assertIn("不要主动全文 OCR", _DEFAULT_QUESTION)
|
||||
self.assertNotIn("完整描述画面内容", _DEFAULT_QUESTION)
|
||||
self.assertNotIn("把其中的数据、坐标轴、图例", _DEFAULT_QUESTION)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -21,12 +21,11 @@ from .output import compact_tool_output
|
|||
from .image_ref import load_image_as_data_url
|
||||
from .media_common import ark_chat_with_retry, extract_chat_answer, record_usage_safe
|
||||
|
||||
# 不带 question 时的默认提问:全覆盖(描述 + OCR + 图表读数),让模型一次把图里能用的信息都吐出来
|
||||
# question 理应由主模型按用户任务具体填写；这里仅作旧调用兼容兜底。
|
||||
# 默认只给简洁概览，避免无条件触发「描述 + 全文 OCR + 图表解析」导致长输出和高延迟。
|
||||
_DEFAULT_QUESTION = (
|
||||
"请仔细看这张图并回答:"
|
||||
"1) 完整描述画面内容(主体/场景/关键元素);"
|
||||
"2) 如果图中有任何文字,逐字准确 OCR 出来,尽量保留原排版与换行;"
|
||||
"3) 如果是图表/表格/示意图,把其中的数据、坐标轴、图例、结构关系读出来。"
|
||||
"请简洁说明这张图片的主要内容，只提供理解图片所必需的信息。"
|
||||
"不要主动全文 OCR、逐项枚举或展开图表数据；无法确定时明确说明。"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -36,10 +35,10 @@ class LookAtImageTool(Tool):
|
|||
"Read/understand an image using Doubao Seed 2.0 Lite vision (the main model is text-only). "
|
||||
"Use to OCR text, describe a picture, read charts/tables/diagrams, or identify objects in an "
|
||||
"image the user uploaded (look for a `[用户上传的参考图] <path>` line in their message) or that "
|
||||
"was generated/saved in the task. Pass the image path; optionally a specific question "
|
||||
"(default: describe + OCR everything). SLOW (tens of seconds per call) — only call when you "
|
||||
"genuinely need the image's actual content to proceed, and ask everything you need in ONE "
|
||||
"call (bundle all questions into `question`); never re-read the same image. "
|
||||
"was generated/saved in the task. Pass the image path and a task-specific `question`; "
|
||||
"only request full-image OCR when the user explicitly needs it. SLOW (tens of seconds per "
|
||||
"call) — only call when you genuinely need the image's actual content to proceed, and ask "
|
||||
"everything you need in ONE call; never re-read the same image. "
|
||||
"Returns the model's textual reading of the image."
|
||||
)
|
||||
parameters = {
|
||||
|
|
@ -55,8 +54,9 @@ class LookAtImageTool(Tool):
|
|||
"question": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"想从图里知道什么(可选)。如「这张图里的表格数据是多少」「图中仪表读数」"
|
||||
"「把这页文字 OCR 出来」。不传则默认全面描述 + OCR 全部文字。"
|
||||
"本次任务需要从图里知道什么(强烈建议填写)。如「读出表格中的抗压强度数据」"
|
||||
"「图中仪表读数是多少」「把这页文字完整 OCR 出来」。只问完成用户任务所需内容；"
|
||||
"仅当用户明确要求整图识别时才要求全文 OCR。不传则仅返回简洁画面概览。"
|
||||
),
|
||||
},
|
||||
},
|
||||
|
|
|
|||
Loading…
Reference in New Issue