diff --git a/core/markdown_guard.py b/core/markdown_guard.py new file mode 100644 index 0000000..aa94cd0 --- /dev/null +++ b/core/markdown_guard.py @@ -0,0 +1,133 @@ +"""Narrow repairs for malformed nested Markdown fence examples. + +Models occasionally wrap a fenced example in an equally long ``markdown`` +fence. CommonMark cannot nest those fences: the inner closing fence closes +the outer block and the intended outer close opens a new, unclosed block. +Only that unambiguous shape is repaired here; arbitrary unclosed fences are +reported but left untouched. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass + +_FENCE_RE = re.compile(r"^( {0,3})(`{3,}|~{3,})([^\r\n]*)(\r?\n)?$") + + +@dataclass(frozen=True) +class MarkdownFenceResult: + text: str + repairs: int = 0 + unclosed_fence: bool = False + + +def _fence(line: str) -> tuple[str, str, int, str] | None: + match = _FENCE_RE.match(line) + if not match: + return None + token = match.group(2) + return match.group(1), token[0], len(token), match.group(3).strip() + + +def _next_nonblank(lines: list[str], start: int) -> int | None: + for idx in range(start, len(lines)): + if lines[idx].strip(): + return idx + return None + + +def _replace_fence(line: str, length: int) -> str: + match = _FENCE_RE.match(line) + if not match: + return line + return ( + match.group(1) + + match.group(2)[0] * length + + match.group(3) + + (match.group(4) or "") + ) + + +def _has_unclosed_fence(lines: list[str]) -> bool: + opened: tuple[str, int] | None = None + for line in lines: + parsed = _fence(line) + if parsed is None: + continue + _indent, char, length, info = parsed + if opened is None: + opened = (char, length) + elif not info and char == opened[0] and length >= opened[1]: + opened = None + return opened is not None + + +def normalize_markdown_fences(text: str) -> MarkdownFenceResult: + """Repair only an equally-long fenced block nested in markdown/md. + + Recognised shape (blank lines between the two closing fences are allowed): + ``markdown opener -> language opener -> language close -> outer close``. + The outer pair is lengthened by one character. Other malformed input is + preserved so the platform never guesses broadly at author intent. + """ + if not text: + return MarkdownFenceResult(text=text) + + lines = text.splitlines(keepends=True) + repairs = 0 + idx = 0 + while idx < len(lines): + outer = _fence(lines[idx]) + if outer is None or outer[3].lower() not in {"markdown", "md"}: + idx += 1 + continue + + inner_idx = _next_nonblank(lines, idx + 1) + inner = _fence(lines[inner_idx]) if inner_idx is not None else None + if ( + inner is None + or not inner[3] + or inner[1] != outer[1] + or inner[2] < outer[2] + ): + idx += 1 + continue + + inner_close_idx = None + for candidate in range(inner_idx + 1, len(lines)): + closing = _fence(lines[candidate]) + if ( + closing is not None + and not closing[3] + and closing[1] == inner[1] + and closing[2] >= inner[2] + ): + inner_close_idx = candidate + break + if inner_close_idx is None: + idx += 1 + continue + + outer_close_idx = _next_nonblank(lines, inner_close_idx + 1) + outer_close = _fence(lines[outer_close_idx]) if outer_close_idx is not None else None + if ( + outer_close is None + or outer_close[3] + or outer_close[1] != outer[1] + or outer_close[2] < outer[2] + ): + idx += 1 + continue + + repaired_length = max(outer[2], inner[2]) + 1 + lines[idx] = _replace_fence(lines[idx], repaired_length) + lines[outer_close_idx] = _replace_fence(lines[outer_close_idx], repaired_length) + repairs += 1 + idx = outer_close_idx + 1 + + normalized = "".join(lines) + return MarkdownFenceResult( + text=normalized, + repairs=repairs, + unclosed_fence=_has_unclosed_fence(lines), + ) diff --git a/core/session.py b/core/session.py index 8e6270c..86abcf6 100644 --- a/core/session.py +++ b/core/session.py @@ -19,6 +19,7 @@ from sqlalchemy import delete, func, select from .storage import session_scope from .storage.models import Message, Task, sanitize_jsonb_nul from .file_store import atomic_write_text +from .markdown_guard import normalize_markdown_fences def _to_dict(msg: Any) -> Any: @@ -72,6 +73,20 @@ class Session: # 与 Message.payload 的 ORM 守门保持一致,使当前 run 的内存上下文和落库值 # 完全相同;外部工具提取文本偶尔会携带 PostgreSQL JSONB 不支持的 NUL。 msg_dict = sanitize_jsonb_nul(_to_dict(msg)) + if msg_dict.get("role") == "assistant" and isinstance(msg_dict.get("content"), str): + fence_result = normalize_markdown_fences(msg_dict["content"]) + msg_dict["content"] = fence_result.text + if fence_result.repairs: + print( + f"[markdown:fence-repair] task={self.task_id} " + f"repairs={fence_result.repairs}", + flush=True, + ) + elif fence_result.unclosed_fence: + print( + f"[markdown:fence-warning] task={self.task_id} unclosed=1", + flush=True, + ) self.messages.append(msg_dict) if msg_dict.get("role") == "system": return None @@ -201,7 +216,12 @@ class Session: .order_by(Message.idx) ).scalars().all() for row in rows: - sess.messages.append(dict(row.payload)) + payload = dict(row.payload) + if payload.get("role") == "assistant" and isinstance(payload.get("content"), str): + # 历史行不回写生产库;只在重建 LLM 上下文时应用同一窄修复, + # 与 Web 展示层保持一致,避免旧坏围栏继续污染后续轮次。 + payload["content"] = normalize_markdown_fences(payload["content"]).text + sess.messages.append(payload) # 真实总条数(含 base 之前的归档历史),保证 append 续号不撞 idx。 sess._db_idx = s.execute( select(func.count()) diff --git a/prompts/system/general_v1.md b/prompts/system/general_v1.md index 203ae6b..35328db 100644 --- a/prompts/system/general_v1.md +++ b/prompts/system/general_v1.md @@ -36,6 +36,7 @@ - 动手前先看: 用 read/grep/glob 摸清现状,再 edit - 改动最小化: edit 工具的 old_str 必须唯一匹配,不够唯一就多带上下文 - 有测试就跑测试验证;没有就用 run_python 写一段最小复现验证 +- Markdown 围栏:展示包含 fenced code block 的 Markdown 源码时,外层统一用 `~~~~` 围栏,内层保留反引号围栏,并确保外层围栏长于任何同字符内层围栏 - 输出简洁: 不复述 diff,只说做了什么、下一步要不要继续 - 工具结果带 `[Error ...]` 时,先想清楚原因再重试,不要盲目重复同一调用 - 不臆造 API、文献、数据 —— 不知道就 read 源码 / 让用户提供 / 明说不知道 diff --git a/tests/frontend_markdown_guard.test.mjs b/tests/frontend_markdown_guard.test.mjs new file mode 100644 index 0000000..f2128cc --- /dev/null +++ b/tests/frontend_markdown_guard.test.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import test from "node:test"; + +const require = createRequire(import.meta.url); +const marked = require("../web/static/vendor/markdown/marked.umd.js"); +globalThis.window = { marked }; + +const { normalizeMarkdownFences, renderMd } = await import("../web/static/js/markdown.js"); + +test("repairs equal-length nested markdown fences", () => { + const broken = [ + "```markdown", + "```mermaid", + "flowchart LR", + " A --> B", + "```", + "```", + "", + "正文 **正常**。", + "", + ].join("\n"); + + const repaired = normalizeMarkdownFences(broken); + + assert.match(repaired, /^````markdown\n```mermaid/); + assert.match(repaired, /```\n````\n\n正文 \*\*正常\*\*。/); + const html = renderMd(broken); + assert.match(html, /
正文 正常<\/strong>。<\/p>/);
+});
+
+test("preserves unrelated unclosed fences", () => {
+ const broken = "正文\n```python\nprint('x')\n";
+ assert.equal(normalizeMarkdownFences(broken), broken);
+});
diff --git a/tests/test_markdown_guard.py b/tests/test_markdown_guard.py
new file mode 100644
index 0000000..b874b5e
--- /dev/null
+++ b/tests/test_markdown_guard.py
@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import unittest
+
+from core.markdown_guard import normalize_markdown_fences
+
+BROKEN = """前文
+
+```markdown
+```mermaid
+flowchart LR
+ A --> B
+```
+```
+
+后文 **应正常渲染**。
+"""
+
+
+class MarkdownFenceGuardTests(unittest.TestCase):
+ def test_repairs_equal_length_nested_fence_example(self) -> None:
+ result = normalize_markdown_fences(BROKEN)
+
+ self.assertEqual(result.repairs, 1)
+ self.assertFalse(result.unclosed_fence)
+ self.assertIn("````markdown\n```mermaid", result.text)
+ self.assertIn("```\n````\n\n后文", result.text)
+
+ def test_preserves_already_valid_longer_outer_fence(self) -> None:
+ valid = BROKEN.replace("```markdown", "````markdown", 1).replace(
+ "```\n\n后文", "````\n\n后文", 1
+ )
+
+ result = normalize_markdown_fences(valid)
+
+ self.assertEqual(result.text, valid)
+ self.assertEqual(result.repairs, 0)
+ self.assertFalse(result.unclosed_fence)
+
+ def test_preserves_arbitrary_unclosed_fence_and_reports_it(self) -> None:
+ broken = "正文\n\n```python\nprint('x')\n"
+
+ result = normalize_markdown_fences(broken)
+
+ self.assertEqual(result.text, broken)
+ self.assertEqual(result.repairs, 0)
+ self.assertTrue(result.unclosed_fence)
+
+ def test_supports_tilde_fences_and_blank_line_between_closes(self) -> None:
+ broken = "~~~~md\n~~~~js\nx = 1\n~~~~\n\n~~~~\n正文\n"
+
+ result = normalize_markdown_fences(broken)
+
+ self.assertEqual(result.repairs, 1)
+ self.assertTrue(result.text.startswith("~~~~~md\n~~~~js"))
+ self.assertIn("~~~~\n\n~~~~~\n正文", result.text)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_message_payload.py b/tests/test_message_payload.py
index ce82b91..ec2c008 100644
--- a/tests/test_message_payload.py
+++ b/tests/test_message_payload.py
@@ -1,5 +1,6 @@
import unittest
from contextlib import contextmanager
+from types import SimpleNamespace
from unittest.mock import patch
from uuid import uuid4
@@ -62,6 +63,72 @@ class MessagePayloadSanitizationTests(unittest.TestCase):
self.assertEqual(session.messages, [expected])
self.assertEqual(fake_db.row.payload, expected)
+ def test_session_repairs_assistant_markdown_before_storing_it(self) -> None:
+ class FakeDbSession:
+ row: Message
+
+ def add(self, row: Message) -> None:
+ self.row = row
+
+ def flush(self) -> None:
+ self.row.message_id = uuid4()
+
+ fake_db = FakeDbSession()
+
+ @contextmanager
+ def fake_session_scope():
+ yield fake_db
+
+ broken = "```markdown\n```mermaid\nA --> B\n```\n```\n正文\n"
+ session = Session(task_id=uuid4())
+ with patch("core.session.session_scope", side_effect=fake_session_scope):
+ session.append({"role": "assistant", "content": broken})
+
+ stored = session.messages[0]["content"]
+ self.assertTrue(stored.startswith("````markdown\n```mermaid"))
+ self.assertIn("```\n````\n正文", stored)
+ self.assertEqual(fake_db.row.payload["content"], stored)
+
+ def test_session_repairs_historical_assistant_markdown_in_memory_only(self) -> None:
+ broken = "```markdown\n```mermaid\nA --> B\n```\n```\n正文\n"
+
+ class FakeResult:
+ def __init__(self, value):
+ self.value = value
+
+ def first(self):
+ return self.value
+
+ def scalars(self):
+ return self
+
+ def all(self):
+ return self.value
+
+ def scalar_one(self):
+ return self.value
+
+ class FakeDbSession:
+ def __init__(self):
+ self.results = iter([
+ FakeResult(SimpleNamespace(context_base_idx=0, context_summary="")),
+ FakeResult([SimpleNamespace(payload={"role": "assistant", "content": broken})]),
+ FakeResult(1),
+ ])
+
+ def execute(self, _query):
+ return next(self.results)
+
+ @contextmanager
+ def fake_session_scope():
+ yield FakeDbSession()
+
+ with patch("core.session.session_scope", side_effect=fake_session_scope):
+ session = Session.load(uuid4())
+
+ self.assertTrue(session.messages[0]["content"].startswith("````markdown"))
+ self.assertEqual(session._db_idx, 1)
+
if __name__ == "__main__":
unittest.main()
diff --git a/web/static/js/markdown.js b/web/static/js/markdown.js
index 28ab984..2eff8af 100644
--- a/web/static/js/markdown.js
+++ b/web/static/js/markdown.js
@@ -6,8 +6,71 @@ if (window.marked && window.marked.setOptions) {
window.marked.setOptions({ gfm: true, breaks: true, headerIds: false, mangle: false });
}
+const FENCE_RE = /^( {0,3})(`{3,}|~{3,})([^\r\n]*)(\r?\n)?$/;
+
+function parseFence(line) {
+ const m = String(line || "").match(FENCE_RE);
+ if (!m) return null;
+ return { indent: m[1], char: m[2][0], len: m[2].length, info: m[3].trim() };
+}
+
+function nextNonblank(lines, start) {
+ for (let i = start; i < lines.length; i++) {
+ if (lines[i].trim()) return i;
+ }
+ return -1;
+}
+
+function replaceFence(line, length) {
+ const m = String(line || "").match(FENCE_RE);
+ if (!m) return line;
+ return m[1] + m[2][0].repeat(length) + m[3] + (m[4] || "");
+}
+
+// 只修复明确的「markdown 外层与内层语言块使用同长围栏」形态。
+// 其他残缺 Markdown 原样交给 marked,避免猜测作者意图。
+export function normalizeMarkdownFences(text) {
+ const lines = String(text || "").match(/.*(?:\r\n|\n|$)/g).filter(Boolean);
+ for (let i = 0; i < lines.length;) {
+ const outer = parseFence(lines[i]);
+ if (!outer || !["markdown", "md"].includes(outer.info.toLowerCase())) {
+ i++;
+ continue;
+ }
+ const innerIdx = nextNonblank(lines, i + 1);
+ const inner = innerIdx >= 0 ? parseFence(lines[innerIdx]) : null;
+ if (!inner || !inner.info || inner.char !== outer.char || inner.len < outer.len) {
+ i++;
+ continue;
+ }
+ let innerCloseIdx = -1;
+ for (let j = innerIdx + 1; j < lines.length; j++) {
+ const close = parseFence(lines[j]);
+ if (close && !close.info && close.char === inner.char && close.len >= inner.len) {
+ innerCloseIdx = j;
+ break;
+ }
+ }
+ if (innerCloseIdx < 0) {
+ i++;
+ continue;
+ }
+ const outerCloseIdx = nextNonblank(lines, innerCloseIdx + 1);
+ const outerClose = outerCloseIdx >= 0 ? parseFence(lines[outerCloseIdx]) : null;
+ if (!outerClose || outerClose.info || outerClose.char !== outer.char || outerClose.len < outer.len) {
+ i++;
+ continue;
+ }
+ const repairedLen = Math.max(outer.len, inner.len) + 1;
+ lines[i] = replaceFence(lines[i], repairedLen);
+ lines[outerCloseIdx] = replaceFence(lines[outerCloseIdx], repairedLen);
+ i = outerCloseIdx + 1;
+ }
+ return lines.join("");
+}
+
export function renderMd(text) {
- const raw = String(text || "");
+ const raw = normalizeMarkdownFences(text);
if (!window.marked || !window.marked.parse) {
return `${escapeHtml(raw)}`;
}