From a00ebfe50ecd6b507c44078ea9c7eb45e8ec6f66 Mon Sep 17 00:00:00 2001 From: caoqianming Date: Mon, 3 Aug 2026 10:11:25 +0800 Subject: [PATCH] fix(markdown): repair nested code fences --- core/markdown_guard.py | 133 +++++++++++++++++++++++++ core/session.py | 22 +++- prompts/system/general_v1.md | 1 + tests/frontend_markdown_guard.test.mjs | 35 +++++++ tests/test_markdown_guard.py | 60 +++++++++++ tests/test_message_payload.py | 67 +++++++++++++ web/static/js/markdown.js | 65 +++++++++++- 7 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 core/markdown_guard.py create mode 100644 tests/frontend_markdown_guard.test.mjs create mode 100644 tests/test_markdown_guard.py diff --git a/core/markdown_guard.py b/core/markdown_guard.py new file mode 100644 index 0000000..aa94cd0 --- /dev/null +++ b/core/markdown_guard.py @@ -0,0 +1,133 @@ +"""Narrow repairs for malformed nested Markdown fence examples. + +Models occasionally wrap a fenced example in an equally long ``markdown`` +fence. CommonMark cannot nest those fences: the inner closing fence closes +the outer block and the intended outer close opens a new, unclosed block. +Only that unambiguous shape is repaired here; arbitrary unclosed fences are +reported but left untouched. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass + +_FENCE_RE = re.compile(r"^( {0,3})(`{3,}|~{3,})([^\r\n]*)(\r?\n)?$") + + +@dataclass(frozen=True) +class MarkdownFenceResult: + text: str + repairs: int = 0 + unclosed_fence: bool = False + + +def _fence(line: str) -> tuple[str, str, int, str] | None: + match = _FENCE_RE.match(line) + if not match: + return None + token = match.group(2) + return match.group(1), token[0], len(token), match.group(3).strip() + + +def _next_nonblank(lines: list[str], start: int) -> int | None: + for idx in range(start, len(lines)): + if lines[idx].strip(): + return idx + return None + + +def _replace_fence(line: str, length: int) -> str: + match = _FENCE_RE.match(line) + if not match: + return line + return ( + match.group(1) + + match.group(2)[0] * length + + match.group(3) + + (match.group(4) or "") + ) + + +def _has_unclosed_fence(lines: list[str]) -> bool: + opened: tuple[str, int] | None = None + for line in lines: + parsed = _fence(line) + if parsed is None: + continue + _indent, char, length, info = parsed + if opened is None: + opened = (char, length) + elif not info and char == opened[0] and length >= opened[1]: + opened = None + return opened is not None + + +def normalize_markdown_fences(text: str) -> MarkdownFenceResult: + """Repair only an equally-long fenced block nested in markdown/md. + + Recognised shape (blank lines between the two closing fences are allowed): + ``markdown opener -> language opener -> language close -> outer close``. + The outer pair is lengthened by one character. Other malformed input is + preserved so the platform never guesses broadly at author intent. + """ + if not text: + return MarkdownFenceResult(text=text) + + lines = text.splitlines(keepends=True) + repairs = 0 + idx = 0 + while idx < len(lines): + outer = _fence(lines[idx]) + if outer is None or outer[3].lower() not in {"markdown", "md"}: + idx += 1 + continue + + inner_idx = _next_nonblank(lines, idx + 1) + inner = _fence(lines[inner_idx]) if inner_idx is not None else None + if ( + inner is None + or not inner[3] + or inner[1] != outer[1] + or inner[2] < outer[2] + ): + idx += 1 + continue + + inner_close_idx = None + for candidate in range(inner_idx + 1, len(lines)): + closing = _fence(lines[candidate]) + if ( + closing is not None + and not closing[3] + and closing[1] == inner[1] + and closing[2] >= inner[2] + ): + inner_close_idx = candidate + break + if inner_close_idx is None: + idx += 1 + continue + + outer_close_idx = _next_nonblank(lines, inner_close_idx + 1) + outer_close = _fence(lines[outer_close_idx]) if outer_close_idx is not None else None + if ( + outer_close is None + or outer_close[3] + or outer_close[1] != outer[1] + or outer_close[2] < outer[2] + ): + idx += 1 + continue + + repaired_length = max(outer[2], inner[2]) + 1 + lines[idx] = _replace_fence(lines[idx], repaired_length) + lines[outer_close_idx] = _replace_fence(lines[outer_close_idx], repaired_length) + repairs += 1 + idx = outer_close_idx + 1 + + normalized = "".join(lines) + return MarkdownFenceResult( + text=normalized, + repairs=repairs, + unclosed_fence=_has_unclosed_fence(lines), + ) diff --git a/core/session.py b/core/session.py index 8e6270c..86abcf6 100644 --- a/core/session.py +++ b/core/session.py @@ -19,6 +19,7 @@ from sqlalchemy import delete, func, select from .storage import session_scope from .storage.models import Message, Task, sanitize_jsonb_nul from .file_store import atomic_write_text +from .markdown_guard import normalize_markdown_fences def _to_dict(msg: Any) -> Any: @@ -72,6 +73,20 @@ class Session: # 与 Message.payload 的 ORM 守门保持一致，使当前 run 的内存上下文和落库值 # 完全相同；外部工具提取文本偶尔会携带 PostgreSQL JSONB 不支持的 NUL。 msg_dict = sanitize_jsonb_nul(_to_dict(msg)) + if msg_dict.get("role") == "assistant" and isinstance(msg_dict.get("content"), str): + fence_result = normalize_markdown_fences(msg_dict["content"]) + msg_dict["content"] = fence_result.text + if fence_result.repairs: + print( + f"[markdown:fence-repair] task={self.task_id} " + f"repairs={fence_result.repairs}", + flush=True, + ) + elif fence_result.unclosed_fence: + print( + f"[markdown:fence-warning] task={self.task_id} unclosed=1", + flush=True, + ) self.messages.append(msg_dict) if msg_dict.get("role") == "system": return None @@ -201,7 +216,12 @@ class Session: .order_by(Message.idx) ).scalars().all() for row in rows: - sess.messages.append(dict(row.payload)) + payload = dict(row.payload) + if payload.get("role") == "assistant" and isinstance(payload.get("content"), str): + # 历史行不回写生产库；只在重建 LLM 上下文时应用同一窄修复， + # 与 Web 展示层保持一致，避免旧坏围栏继续污染后续轮次。 + payload["content"] = normalize_markdown_fences(payload["content"]).text + sess.messages.append(payload) # 真实总条数(含 base 之前的归档历史),保证 append 续号不撞 idx。 sess._db_idx = s.execute( select(func.count()) diff --git a/prompts/system/general_v1.md b/prompts/system/general_v1.md index 203ae6b..35328db 100644 --- a/prompts/system/general_v1.md +++ b/prompts/system/general_v1.md @@ -36,6 +36,7 @@ - 动手前先看: 用 read/grep/glob 摸清现状,再 edit - 改动最小化: edit 工具的 old_str 必须唯一匹配,不够唯一就多带上下文 - 有测试就跑测试验证;没有就用 run_python 写一段最小复现验证 +- Markdown 围栏:展示包含 fenced code block 的 Markdown 源码时,外层统一用 `~~~~` 围栏,内层保留反引号围栏,并确保外层围栏长于任何同字符内层围栏 - 输出简洁: 不复述 diff,只说做了什么、下一步要不要继续 - 工具结果带 `[Error ...]` 时,先想清楚原因再重试,不要盲目重复同一调用 - 不臆造 API、文献、数据 —— 不知道就 read 源码 / 让用户提供 / 明说不知道 diff --git a/tests/frontend_markdown_guard.test.mjs b/tests/frontend_markdown_guard.test.mjs new file mode 100644 index 0000000..f2128cc --- /dev/null +++ b/tests/frontend_markdown_guard.test.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import test from "node:test"; + +const require = createRequire(import.meta.url); +const marked = require("../web/static/vendor/markdown/marked.umd.js"); +globalThis.window = { marked }; + +const { normalizeMarkdownFences, renderMd } = await import("../web/static/js/markdown.js"); + +test("repairs equal-length nested markdown fences", () => { + const broken = [ + "```markdown", + "```mermaid", + "flowchart LR", + " A --> B", + "```", + "```", + "", + "正文 **正常**。", + "", + ].join("\n"); + + const repaired = normalizeMarkdownFences(broken); + + assert.match(repaired, /^````markdown\n```mermaid/); + assert.match(repaired, /```\n````\n\n正文 \*\*正常\*\*。/); + const html = renderMd(broken); + assert.match(html, /

正文 正常<\/strong>。<\/p>/); +}); + +test("preserves unrelated unclosed fences", () => { + const broken = "正文\n```python\nprint('x')\n"; + assert.equal(normalizeMarkdownFences(broken), broken); +}); diff --git a/tests/test_markdown_guard.py b/tests/test_markdown_guard.py new file mode 100644 index 0000000..b874b5e --- /dev/null +++ b/tests/test_markdown_guard.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import unittest + +from core.markdown_guard import normalize_markdown_fences + +BROKEN = """前文 + +```markdown +```mermaid +flowchart LR + A --> B +``` +``` + +后文 **应正常渲染**。 +""" + + +class MarkdownFenceGuardTests(unittest.TestCase): + def test_repairs_equal_length_nested_fence_example(self) -> None: + result = normalize_markdown_fences(BROKEN) + + self.assertEqual(result.repairs, 1) + self.assertFalse(result.unclosed_fence) + self.assertIn("````markdown\n```mermaid", result.text) + self.assertIn("```\n````\n\n后文", result.text) + + def test_preserves_already_valid_longer_outer_fence(self) -> None: + valid = BROKEN.replace("```markdown", "````markdown", 1).replace( + "```\n\n后文", "````\n\n后文", 1 + ) + + result = normalize_markdown_fences(valid) + + self.assertEqual(result.text, valid) + self.assertEqual(result.repairs, 0) + self.assertFalse(result.unclosed_fence) + + def test_preserves_arbitrary_unclosed_fence_and_reports_it(self) -> None: + broken = "正文\n\n```python\nprint('x')\n" + + result = normalize_markdown_fences(broken) + + self.assertEqual(result.text, broken) + self.assertEqual(result.repairs, 0) + self.assertTrue(result.unclosed_fence) + + def test_supports_tilde_fences_and_blank_line_between_closes(self) -> None: + broken = "~~~~md\n~~~~js\nx = 1\n~~~~\n\n~~~~\n正文\n" + + result = normalize_markdown_fences(broken) + + self.assertEqual(result.repairs, 1) + self.assertTrue(result.text.startswith("~~~~~md\n~~~~js")) + self.assertIn("~~~~\n\n~~~~~\n正文", result.text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_message_payload.py b/tests/test_message_payload.py index ce82b91..ec2c008 100644 --- a/tests/test_message_payload.py +++ b/tests/test_message_payload.py @@ -1,5 +1,6 @@ import unittest from contextlib import contextmanager +from types import SimpleNamespace from unittest.mock import patch from uuid import uuid4 @@ -62,6 +63,72 @@ class MessagePayloadSanitizationTests(unittest.TestCase): self.assertEqual(session.messages, [expected]) self.assertEqual(fake_db.row.payload, expected) + def test_session_repairs_assistant_markdown_before_storing_it(self) -> None: + class FakeDbSession: + row: Message + + def add(self, row: Message) -> None: + self.row = row + + def flush(self) -> None: + self.row.message_id = uuid4() + + fake_db = FakeDbSession() + + @contextmanager + def fake_session_scope(): + yield fake_db + + broken = "```markdown\n```mermaid\nA --> B\n```\n```\n正文\n" + session = Session(task_id=uuid4()) + with patch("core.session.session_scope", side_effect=fake_session_scope): + session.append({"role": "assistant", "content": broken}) + + stored = session.messages[0]["content"] + self.assertTrue(stored.startswith("````markdown\n```mermaid")) + self.assertIn("```\n````\n正文", stored) + self.assertEqual(fake_db.row.payload["content"], stored) + + def test_session_repairs_historical_assistant_markdown_in_memory_only(self) -> None: + broken = "```markdown\n```mermaid\nA --> B\n```\n```\n正文\n" + + class FakeResult: + def __init__(self, value): + self.value = value + + def first(self): + return self.value + + def scalars(self): + return self + + def all(self): + return self.value + + def scalar_one(self): + return self.value + + class FakeDbSession: + def __init__(self): + self.results = iter([ + FakeResult(SimpleNamespace(context_base_idx=0, context_summary="")), + FakeResult([SimpleNamespace(payload={"role": "assistant", "content": broken})]), + FakeResult(1), + ]) + + def execute(self, _query): + return next(self.results) + + @contextmanager + def fake_session_scope(): + yield FakeDbSession() + + with patch("core.session.session_scope", side_effect=fake_session_scope): + session = Session.load(uuid4()) + + self.assertTrue(session.messages[0]["content"].startswith("````markdown")) + self.assertEqual(session._db_idx, 1) + if __name__ == "__main__": unittest.main() diff --git a/web/static/js/markdown.js b/web/static/js/markdown.js index 28ab984..2eff8af 100644 --- a/web/static/js/markdown.js +++ b/web/static/js/markdown.js @@ -6,8 +6,71 @@ if (window.marked && window.marked.setOptions) { window.marked.setOptions({ gfm: true, breaks: true, headerIds: false, mangle: false }); } +const FENCE_RE = /^( {0,3})(`{3,}|~{3,})([^\r\n]*)(\r?\n)?$/; + +function parseFence(line) { + const m = String(line || "").match(FENCE_RE); + if (!m) return null; + return { indent: m[1], char: m[2][0], len: m[2].length, info: m[3].trim() }; +} + +function nextNonblank(lines, start) { + for (let i = start; i < lines.length; i++) { + if (lines[i].trim()) return i; + } + return -1; +} + +function replaceFence(line, length) { + const m = String(line || "").match(FENCE_RE); + if (!m) return line; + return m[1] + m[2][0].repeat(length) + m[3] + (m[4] || ""); +} + +// 只修复明确的「markdown 外层与内层语言块使用同长围栏」形态。 +// 其他残缺 Markdown 原样交给 marked，避免猜测作者意图。 +export function normalizeMarkdownFences(text) { + const lines = String(text || "").match(/.*(?:\r\n|\n|$)/g).filter(Boolean); + for (let i = 0; i < lines.length;) { + const outer = parseFence(lines[i]); + if (!outer || !["markdown", "md"].includes(outer.info.toLowerCase())) { + i++; + continue; + } + const innerIdx = nextNonblank(lines, i + 1); + const inner = innerIdx >= 0 ? parseFence(lines[innerIdx]) : null; + if (!inner || !inner.info || inner.char !== outer.char || inner.len < outer.len) { + i++; + continue; + } + let innerCloseIdx = -1; + for (let j = innerIdx + 1; j < lines.length; j++) { + const close = parseFence(lines[j]); + if (close && !close.info && close.char === inner.char && close.len >= inner.len) { + innerCloseIdx = j; + break; + } + } + if (innerCloseIdx < 0) { + i++; + continue; + } + const outerCloseIdx = nextNonblank(lines, innerCloseIdx + 1); + const outerClose = outerCloseIdx >= 0 ? parseFence(lines[outerCloseIdx]) : null; + if (!outerClose || outerClose.info || outerClose.char !== outer.char || outerClose.len < outer.len) { + i++; + continue; + } + const repairedLen = Math.max(outer.len, inner.len) + 1; + lines[i] = replaceFence(lines[i], repairedLen); + lines[outerCloseIdx] = replaceFence(lines[outerCloseIdx], repairedLen); + i = outerCloseIdx + 1; + } + return lines.join(""); +} + export function renderMd(text) { - const raw = String(text || ""); + const raw = normalizeMarkdownFences(text); if (!window.marked || !window.marked.parse) { return `

${escapeHtml(raw)}
`; }