fix(markdown): repair nested code fences
This commit is contained in:
parent
a8b84b1d6f
commit
a00ebfe50e
|
|
@ -0,0 +1,133 @@
|
|||
"""Narrow repairs for malformed nested Markdown fence examples.
|
||||
|
||||
Models occasionally wrap a fenced example in an equally long ``markdown``
|
||||
fence. CommonMark cannot nest those fences: the inner closing fence closes
|
||||
the outer block and the intended outer close opens a new, unclosed block.
|
||||
Only that unambiguous shape is repaired here; arbitrary unclosed fences are
|
||||
reported but left untouched.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
_FENCE_RE = re.compile(r"^( {0,3})(`{3,}|~{3,})([^\r\n]*)(\r?\n)?$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MarkdownFenceResult:
|
||||
text: str
|
||||
repairs: int = 0
|
||||
unclosed_fence: bool = False
|
||||
|
||||
|
||||
def _fence(line: str) -> tuple[str, str, int, str] | None:
|
||||
match = _FENCE_RE.match(line)
|
||||
if not match:
|
||||
return None
|
||||
token = match.group(2)
|
||||
return match.group(1), token[0], len(token), match.group(3).strip()
|
||||
|
||||
|
||||
def _next_nonblank(lines: list[str], start: int) -> int | None:
|
||||
for idx in range(start, len(lines)):
|
||||
if lines[idx].strip():
|
||||
return idx
|
||||
return None
|
||||
|
||||
|
||||
def _replace_fence(line: str, length: int) -> str:
|
||||
match = _FENCE_RE.match(line)
|
||||
if not match:
|
||||
return line
|
||||
return (
|
||||
match.group(1)
|
||||
+ match.group(2)[0] * length
|
||||
+ match.group(3)
|
||||
+ (match.group(4) or "")
|
||||
)
|
||||
|
||||
|
||||
def _has_unclosed_fence(lines: list[str]) -> bool:
|
||||
opened: tuple[str, int] | None = None
|
||||
for line in lines:
|
||||
parsed = _fence(line)
|
||||
if parsed is None:
|
||||
continue
|
||||
_indent, char, length, info = parsed
|
||||
if opened is None:
|
||||
opened = (char, length)
|
||||
elif not info and char == opened[0] and length >= opened[1]:
|
||||
opened = None
|
||||
return opened is not None
|
||||
|
||||
|
||||
def normalize_markdown_fences(text: str) -> MarkdownFenceResult:
|
||||
"""Repair only an equally-long fenced block nested in markdown/md.
|
||||
|
||||
Recognised shape (blank lines between the two closing fences are allowed):
|
||||
``markdown opener -> language opener -> language close -> outer close``.
|
||||
The outer pair is lengthened by one character. Other malformed input is
|
||||
preserved so the platform never guesses broadly at author intent.
|
||||
"""
|
||||
if not text:
|
||||
return MarkdownFenceResult(text=text)
|
||||
|
||||
lines = text.splitlines(keepends=True)
|
||||
repairs = 0
|
||||
idx = 0
|
||||
while idx < len(lines):
|
||||
outer = _fence(lines[idx])
|
||||
if outer is None or outer[3].lower() not in {"markdown", "md"}:
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
inner_idx = _next_nonblank(lines, idx + 1)
|
||||
inner = _fence(lines[inner_idx]) if inner_idx is not None else None
|
||||
if (
|
||||
inner is None
|
||||
or not inner[3]
|
||||
or inner[1] != outer[1]
|
||||
or inner[2] < outer[2]
|
||||
):
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
inner_close_idx = None
|
||||
for candidate in range(inner_idx + 1, len(lines)):
|
||||
closing = _fence(lines[candidate])
|
||||
if (
|
||||
closing is not None
|
||||
and not closing[3]
|
||||
and closing[1] == inner[1]
|
||||
and closing[2] >= inner[2]
|
||||
):
|
||||
inner_close_idx = candidate
|
||||
break
|
||||
if inner_close_idx is None:
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
outer_close_idx = _next_nonblank(lines, inner_close_idx + 1)
|
||||
outer_close = _fence(lines[outer_close_idx]) if outer_close_idx is not None else None
|
||||
if (
|
||||
outer_close is None
|
||||
or outer_close[3]
|
||||
or outer_close[1] != outer[1]
|
||||
or outer_close[2] < outer[2]
|
||||
):
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
repaired_length = max(outer[2], inner[2]) + 1
|
||||
lines[idx] = _replace_fence(lines[idx], repaired_length)
|
||||
lines[outer_close_idx] = _replace_fence(lines[outer_close_idx], repaired_length)
|
||||
repairs += 1
|
||||
idx = outer_close_idx + 1
|
||||
|
||||
normalized = "".join(lines)
|
||||
return MarkdownFenceResult(
|
||||
text=normalized,
|
||||
repairs=repairs,
|
||||
unclosed_fence=_has_unclosed_fence(lines),
|
||||
)
|
||||
|
|
@ -19,6 +19,7 @@ from sqlalchemy import delete, func, select
|
|||
from .storage import session_scope
|
||||
from .storage.models import Message, Task, sanitize_jsonb_nul
|
||||
from .file_store import atomic_write_text
|
||||
from .markdown_guard import normalize_markdown_fences
|
||||
|
||||
|
||||
def _to_dict(msg: Any) -> Any:
|
||||
|
|
@ -72,6 +73,20 @@ class Session:
|
|||
# 与 Message.payload 的 ORM 守门保持一致，使当前 run 的内存上下文和落库值
|
||||
# 完全相同；外部工具提取文本偶尔会携带 PostgreSQL JSONB 不支持的 NUL。
|
||||
msg_dict = sanitize_jsonb_nul(_to_dict(msg))
|
||||
if msg_dict.get("role") == "assistant" and isinstance(msg_dict.get("content"), str):
|
||||
fence_result = normalize_markdown_fences(msg_dict["content"])
|
||||
msg_dict["content"] = fence_result.text
|
||||
if fence_result.repairs:
|
||||
print(
|
||||
f"[markdown:fence-repair] task={self.task_id} "
|
||||
f"repairs={fence_result.repairs}",
|
||||
flush=True,
|
||||
)
|
||||
elif fence_result.unclosed_fence:
|
||||
print(
|
||||
f"[markdown:fence-warning] task={self.task_id} unclosed=1",
|
||||
flush=True,
|
||||
)
|
||||
self.messages.append(msg_dict)
|
||||
if msg_dict.get("role") == "system":
|
||||
return None
|
||||
|
|
@ -201,7 +216,12 @@ class Session:
|
|||
.order_by(Message.idx)
|
||||
).scalars().all()
|
||||
for row in rows:
|
||||
sess.messages.append(dict(row.payload))
|
||||
payload = dict(row.payload)
|
||||
if payload.get("role") == "assistant" and isinstance(payload.get("content"), str):
|
||||
# 历史行不回写生产库；只在重建 LLM 上下文时应用同一窄修复，
|
||||
# 与 Web 展示层保持一致，避免旧坏围栏继续污染后续轮次。
|
||||
payload["content"] = normalize_markdown_fences(payload["content"]).text
|
||||
sess.messages.append(payload)
|
||||
# 真实总条数(含 base 之前的归档历史),保证 append 续号不撞 idx。
|
||||
sess._db_idx = s.execute(
|
||||
select(func.count())
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@
|
|||
- 动手前先看: 用 read/grep/glob 摸清现状,再 edit
|
||||
- 改动最小化: edit 工具的 old_str 必须唯一匹配,不够唯一就多带上下文
|
||||
- 有测试就跑测试验证;没有就用 run_python 写一段最小复现验证
|
||||
- Markdown 围栏:展示包含 fenced code block 的 Markdown 源码时,外层统一用 `~~~~` 围栏,内层保留反引号围栏,并确保外层围栏长于任何同字符内层围栏
|
||||
- 输出简洁: 不复述 diff,只说做了什么、下一步要不要继续
|
||||
- 工具结果带 `[Error ...]` 时,先想清楚原因再重试,不要盲目重复同一调用
|
||||
- 不臆造 API、文献、数据 —— 不知道就 read 源码 / 让用户提供 / 明说不知道
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { createRequire } from "node:module";
|
||||
import test from "node:test";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const marked = require("../web/static/vendor/markdown/marked.umd.js");
|
||||
globalThis.window = { marked };
|
||||
|
||||
const { normalizeMarkdownFences, renderMd } = await import("../web/static/js/markdown.js");
|
||||
|
||||
test("repairs equal-length nested markdown fences", () => {
|
||||
const broken = [
|
||||
"```markdown",
|
||||
"```mermaid",
|
||||
"flowchart LR",
|
||||
" A --> B",
|
||||
"```",
|
||||
"```",
|
||||
"",
|
||||
"正文 **正常**。",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
const repaired = normalizeMarkdownFences(broken);
|
||||
|
||||
assert.match(repaired, /^````markdown\n```mermaid/);
|
||||
assert.match(repaired, /```\n````\n\n正文 \*\*正常\*\*。/);
|
||||
const html = renderMd(broken);
|
||||
assert.match(html, /<p>正文 <strong>正常<\/strong>。<\/p>/);
|
||||
});
|
||||
|
||||
test("preserves unrelated unclosed fences", () => {
|
||||
const broken = "正文\n```python\nprint('x')\n";
|
||||
assert.equal(normalizeMarkdownFences(broken), broken);
|
||||
});
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from core.markdown_guard import normalize_markdown_fences
|
||||
|
||||
BROKEN = """前文
|
||||
|
||||
```markdown
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A --> B
|
||||
```
|
||||
```
|
||||
|
||||
后文 **应正常渲染**。
|
||||
"""
|
||||
|
||||
|
||||
class MarkdownFenceGuardTests(unittest.TestCase):
|
||||
def test_repairs_equal_length_nested_fence_example(self) -> None:
|
||||
result = normalize_markdown_fences(BROKEN)
|
||||
|
||||
self.assertEqual(result.repairs, 1)
|
||||
self.assertFalse(result.unclosed_fence)
|
||||
self.assertIn("````markdown\n```mermaid", result.text)
|
||||
self.assertIn("```\n````\n\n后文", result.text)
|
||||
|
||||
def test_preserves_already_valid_longer_outer_fence(self) -> None:
|
||||
valid = BROKEN.replace("```markdown", "````markdown", 1).replace(
|
||||
"```\n\n后文", "````\n\n后文", 1
|
||||
)
|
||||
|
||||
result = normalize_markdown_fences(valid)
|
||||
|
||||
self.assertEqual(result.text, valid)
|
||||
self.assertEqual(result.repairs, 0)
|
||||
self.assertFalse(result.unclosed_fence)
|
||||
|
||||
def test_preserves_arbitrary_unclosed_fence_and_reports_it(self) -> None:
|
||||
broken = "正文\n\n```python\nprint('x')\n"
|
||||
|
||||
result = normalize_markdown_fences(broken)
|
||||
|
||||
self.assertEqual(result.text, broken)
|
||||
self.assertEqual(result.repairs, 0)
|
||||
self.assertTrue(result.unclosed_fence)
|
||||
|
||||
def test_supports_tilde_fences_and_blank_line_between_closes(self) -> None:
|
||||
broken = "~~~~md\n~~~~js\nx = 1\n~~~~\n\n~~~~\n正文\n"
|
||||
|
||||
result = normalize_markdown_fences(broken)
|
||||
|
||||
self.assertEqual(result.repairs, 1)
|
||||
self.assertTrue(result.text.startswith("~~~~~md\n~~~~js"))
|
||||
self.assertIn("~~~~\n\n~~~~~\n正文", result.text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
|
|
@ -62,6 +63,72 @@ class MessagePayloadSanitizationTests(unittest.TestCase):
|
|||
self.assertEqual(session.messages, [expected])
|
||||
self.assertEqual(fake_db.row.payload, expected)
|
||||
|
||||
def test_session_repairs_assistant_markdown_before_storing_it(self) -> None:
|
||||
class FakeDbSession:
|
||||
row: Message
|
||||
|
||||
def add(self, row: Message) -> None:
|
||||
self.row = row
|
||||
|
||||
def flush(self) -> None:
|
||||
self.row.message_id = uuid4()
|
||||
|
||||
fake_db = FakeDbSession()
|
||||
|
||||
@contextmanager
|
||||
def fake_session_scope():
|
||||
yield fake_db
|
||||
|
||||
broken = "```markdown\n```mermaid\nA --> B\n```\n```\n正文\n"
|
||||
session = Session(task_id=uuid4())
|
||||
with patch("core.session.session_scope", side_effect=fake_session_scope):
|
||||
session.append({"role": "assistant", "content": broken})
|
||||
|
||||
stored = session.messages[0]["content"]
|
||||
self.assertTrue(stored.startswith("````markdown\n```mermaid"))
|
||||
self.assertIn("```\n````\n正文", stored)
|
||||
self.assertEqual(fake_db.row.payload["content"], stored)
|
||||
|
||||
def test_session_repairs_historical_assistant_markdown_in_memory_only(self) -> None:
|
||||
broken = "```markdown\n```mermaid\nA --> B\n```\n```\n正文\n"
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def first(self):
|
||||
return self.value
|
||||
|
||||
def scalars(self):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return self.value
|
||||
|
||||
def scalar_one(self):
|
||||
return self.value
|
||||
|
||||
class FakeDbSession:
|
||||
def __init__(self):
|
||||
self.results = iter([
|
||||
FakeResult(SimpleNamespace(context_base_idx=0, context_summary="")),
|
||||
FakeResult([SimpleNamespace(payload={"role": "assistant", "content": broken})]),
|
||||
FakeResult(1),
|
||||
])
|
||||
|
||||
def execute(self, _query):
|
||||
return next(self.results)
|
||||
|
||||
@contextmanager
|
||||
def fake_session_scope():
|
||||
yield FakeDbSession()
|
||||
|
||||
with patch("core.session.session_scope", side_effect=fake_session_scope):
|
||||
session = Session.load(uuid4())
|
||||
|
||||
self.assertTrue(session.messages[0]["content"].startswith("````markdown"))
|
||||
self.assertEqual(session._db_idx, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -6,8 +6,71 @@ if (window.marked && window.marked.setOptions) {
|
|||
window.marked.setOptions({ gfm: true, breaks: true, headerIds: false, mangle: false });
|
||||
}
|
||||
|
||||
const FENCE_RE = /^( {0,3})(`{3,}|~{3,})([^\r\n]*)(\r?\n)?$/;
|
||||
|
||||
function parseFence(line) {
|
||||
const m = String(line || "").match(FENCE_RE);
|
||||
if (!m) return null;
|
||||
return { indent: m[1], char: m[2][0], len: m[2].length, info: m[3].trim() };
|
||||
}
|
||||
|
||||
function nextNonblank(lines, start) {
|
||||
for (let i = start; i < lines.length; i++) {
|
||||
if (lines[i].trim()) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function replaceFence(line, length) {
|
||||
const m = String(line || "").match(FENCE_RE);
|
||||
if (!m) return line;
|
||||
return m[1] + m[2][0].repeat(length) + m[3] + (m[4] || "");
|
||||
}
|
||||
|
||||
// 只修复明确的「markdown 外层与内层语言块使用同长围栏」形态。
|
||||
// 其他残缺 Markdown 原样交给 marked，避免猜测作者意图。
|
||||
export function normalizeMarkdownFences(text) {
|
||||
const lines = String(text || "").match(/.*(?:\r\n|\n|$)/g).filter(Boolean);
|
||||
for (let i = 0; i < lines.length;) {
|
||||
const outer = parseFence(lines[i]);
|
||||
if (!outer || !["markdown", "md"].includes(outer.info.toLowerCase())) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
const innerIdx = nextNonblank(lines, i + 1);
|
||||
const inner = innerIdx >= 0 ? parseFence(lines[innerIdx]) : null;
|
||||
if (!inner || !inner.info || inner.char !== outer.char || inner.len < outer.len) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
let innerCloseIdx = -1;
|
||||
for (let j = innerIdx + 1; j < lines.length; j++) {
|
||||
const close = parseFence(lines[j]);
|
||||
if (close && !close.info && close.char === inner.char && close.len >= inner.len) {
|
||||
innerCloseIdx = j;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (innerCloseIdx < 0) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
const outerCloseIdx = nextNonblank(lines, innerCloseIdx + 1);
|
||||
const outerClose = outerCloseIdx >= 0 ? parseFence(lines[outerCloseIdx]) : null;
|
||||
if (!outerClose || outerClose.info || outerClose.char !== outer.char || outerClose.len < outer.len) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
const repairedLen = Math.max(outer.len, inner.len) + 1;
|
||||
lines[i] = replaceFence(lines[i], repairedLen);
|
||||
lines[outerCloseIdx] = replaceFence(lines[outerCloseIdx], repairedLen);
|
||||
i = outerCloseIdx + 1;
|
||||
}
|
||||
return lines.join("");
|
||||
}
|
||||
|
||||
export function renderMd(text) {
|
||||
const raw = String(text || "");
|
||||
const raw = normalizeMarkdownFences(text);
|
||||
if (!window.marked || !window.marked.parse) {
|
||||
return `<pre style="white-space:pre-wrap;word-break:break-word;font-family:inherit;margin:0;">${escapeHtml(raw)}</pre>`;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue