From c6232d832551a399fc4e8d9050e6b9031ee1ab59 Mon Sep 17 00:00:00 2001 From: caoqianming Date: Wed, 22 Jul 2026 15:20:43 +0800 Subject: [PATCH] =?UTF-8?q?feat(kb):=20=E4=B8=AA=E4=BA=BA=E7=9F=A5?= =?UTF-8?q?=E8=AF=86=E5=BA=93=E2=80=94=E2=80=94.kb/=20=E7=BA=AF=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=9C=BA=E5=88=B6+/v1/kb*=20API+=E5=89=8D=E7=AB=AF=20?= =?UTF-8?q?modal+=E6=B3=A8=E5=85=A5=E5=A5=91=E7=BA=A6(bump=200.59.0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - core/kb.py: 状态/视图层(INDEX 单行格式 parse/format、已入库判据=INDEX 有条目、 删单篇连带原件与索引行、kb_block 注入——有库才注) - core/kb_ingest.py: 入库管线 markitdown→扫描件 OCR 兜底(方舟)→flash 摘要(失败降级 不阻塞)→追加 INDEX;编排 to_thread+per-(user,库) 锁,幂等可续跑 - 0022 迁移: usage_events.task_id 放宽可 NULL(kb 入库无 task 上下文); 记账溯源 kind="kb_ingest"/"vision" + units {"kb","source"} - web/app.py: /v1/kb* 8 端点(列/建/删库、详情带进度、上传即入库、手动 ingest、看/删单篇), 不设 HTTP 检索端点(agent 走 fs 工具) - 前端: kb.js 两栏 modal(上传 XHR/进度轮询/删除确认);rail 左下新增知识库入口, 四按钮改图标+小字两行布局 - agent_builder 在 memory_block 后注 kb_block(docker/host 路径换算同 .memory); documents skill「何时不用」补自建资料路由行 - DESIGN §3.8 机制小节 / CHANGELOG 0.59.0 / PROGRESS Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 5 + DESIGN.md | 10 + PROGRESS.md | 7 +- core/__init__.py | 2 +- core/agent_builder.py | 6 + core/kb.py | 288 +++++++++++++++++ core/kb_ingest.py | 299 ++++++++++++++++++ core/storage/models.py | 6 +- core/storage/usage.py | 6 +- ...1000_0022_usage_events_task_id_nullable.py | 38 +++ skills/documents/SKILL.md | 1 + web/app.py | 130 ++++++++ web/static/dev.html | 71 ++++- web/static/js/kb.js | 239 ++++++++++++++ web/static/js/main.js | 2 + 15 files changed, 1100 insertions(+), 10 deletions(-) create mode 100644 core/kb.py create mode 100644 core/kb_ingest.py create mode 100644 db/migrations/versions/20260722_1000_0022_usage_events_task_id_nullable.py create mode 100644 web/static/js/kb.js diff --git a/CHANGELOG.md b/CHANGELOG.md index a4281a4..840c1e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ > 所以不是每个版本号都有条目。条目格式 `## <版本> — <日期>`,新条目加在最上面。 > 工程口径的完整记录见 `PROGRESS.md` / git log。 +## 0.59.0 — 2026-07-22 + +- 新增「个人知识库」:左下角新入口,可以建多个库(如「行业标准」「项目资料」),把常用的 PDF / Word / PPT / Excel / 网页 / 文本资料上传进去,系统自动转成文字版、写好摘要索引;之后在对话里直接提问,我会按需查你的库作答并标注来源。扫描件 PDF 也会自动识别文字入库。库和文档可随时在面板里查看、删除。 +- 左下角资源入口(技能 / 记忆 / 知识库 / 定时)改为图标在上、小字在下的紧凑样式。 + ## 0.58.55 — 2026-07-21 - 扫描版 PDF 现在也能读了:拍照或扫描生成的 PDF(老标准、检测报告、红头文件等,以前解析出来是空白)会自动逐页识别成文字,表格还原成表格、公式还原成公式,之后写申报书、编标准、做 PPT 都能直接引用其中内容。 diff --git a/DESIGN.md b/DESIGN.md index 16a81ac..da9919f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -85,6 +85,16 @@ Session = 消息列表,ORM 直写 PG `messages`(append-only,jsonb 存 LiteLLM - **前端记忆面板只读,"改"全走对话**:看全貌是读、直读 FS 才是地面真相;改走 agent 自管 = 单一写入口、不写坏 frontmatter。故意零写/删 API;将来若"删一条"摩擦大再单加 delete(唯一廉价确定性 mutation)。路径穿越校验收口在 `core/memory.py`。 - **快捷指令 ≠ memory**(`core/shortcuts.py`):触发词→完整指令,存 `.memory/shortcuts.md` 但**内容永不注上下文**——入口层(渠道核心 + web post_message 共用)整条精确匹配确定性替换,0 额外 token、渠道无关;maintenance 蹭 memory 心智(对话让模型写)。若反过来塞 core.md 靠模型概率召回:既不确定又每轮烧 token,正是要绕开的坑。 +### 3.8 个人知识库(`core/kb.py` + `core/kb_ingest.py`,✅ 2026-07-22) + +用户自建资料(规范/报告/标准/内部文档)的长期查阅层,与 §3.7 记忆同范式:**纯文件 + prompt 注入契约,无向量无 DB**(判据同"真实文件为准":个人库几十~百余文件,agentic search 足够;索引若引入只能是可重建派生缓存)。**两层格局**:zcbot 内建 `.kb/` 私有小库(本节)+ 院检索服务共享大库(document_search,zcbot 只当客户端)——分工标准 = 文件数 × 查询频次,路由规则写进注入契约(学科文献→院库 / 自建资料→`.kb`)。 + +- **做成机制而非 skill**(判据:有独立于会话的持久状态需用户管理 → 机制):落盘 `user_root/.kb/<库名>/`(INDEX.md + docs/ 转换后 md + sources/ 原件)。**"已入库"判据 = INDEX.md 有条目**,sources 有而 INDEX 无 = 待入库 → 入库幂等、崩溃可恢复、零 migration。dotfile 命名同 `.memory` 双向防呆,GET /v1/files 天然隐藏。 +- **入库管线**(`core/kb_ingest.py`,上传即触发 + 手动兜底):markitdown Python API 转 md → 扫描件 PDF(文本近零)走方舟文档理解 OCR 兜底(§8.13 同通道)→ deepseek flash 单次 chat 写 标题/摘要/关键词(失败降级文件名+正文开头,不阻塞)→ 追加 INDEX 行。编排照定时执行器:create_task + to_thread + per-(user,库) 内存锁去重;进度存内存供前端轮询,崩了靠 FS 判据续跑。 +- **agent 侧零新工具**:`kb_block`(照 memory_block)把 INDEX 全文 + 契约(路由规则 / INDEX 行格式 / 答题标来源)注 prompt,**用户有库才注入**;fs 工具在 user_root 内可读写、docker 沙箱整 user_root bind → `.kb` 天然可达。INDEX 行格式是对话内手动入库与后台产出的同一契约。 +- **API 薄壳**(`/v1/kb*` 8 端点):列/建/删库、详情(带入库进度)、上传即入库、手动 ingest、看/删单篇。**不设 HTTP 检索端点**——检索是 agent 的事。前端两栏 modal(kb.js)管上传/删除,查询全走对话。 +- **记账**:`usage_events` kind="kb_ingest"(OCR 那笔走 kind="vision"),无 task 上下文 → 0022 放宽 task_id 可 NULL,溯源靠 units JSONB `{"kb", "source"}`。 + --- ## 4. 模型路由 diff --git a/PROGRESS.md b/PROGRESS.md index 3bbc74a..88776e9 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -23,6 +23,7 @@ ### 2026-07 +- **07-22 / 0.59.0 / 个人知识库(纯文件 .kb/ 机制,照记忆范式)**:方案 07-22 对齐后落地(DESIGN §3.8)。**core**:`core/kb.py`(状态/视图层:库名与文件名校验、INDEX 单行格式 `- [标题](docs/x.md)｜来源 sources/x.pdf｜摘要：…｜关键词：…` 的 parse/format、"已入库判据 = INDEX 有条目"派生 pending、删单篇连带原件与 INDEX 行、`kb_block` 注入)+ `core/kb_ingest.py`(入库管线:markitdown Python API → 扫描件 PDF 文本近零走方舟 OCR 兜底(§8.13 同通道)→ flash 单次 chat 写标题/摘要/关键词(失败降级不阻塞)→ 追加 INDEX;编排 create_task+to_thread+per-(user,库) 锁,进度内存态供轮询)。**记账**:0022 迁移放宽 `usage_events.task_id` 可 NULL(kb 入库无 task 上下文),溯源 = kind="kb_ingest"/"vision" + units `{"kb","source"}`(record_chat/vision_usage 签名同步放宽)。**API**:`/v1/kb*` 8 端点(列/建/删库、详情带入库进度、上传即入库、手动 ingest、看/删单篇),不设 HTTP 检索端点。**agent 侧零新工具**:agent_builder 在 memory_block 后注 `kb_block`(有库才注,docker/host 路径换算同 .memory);documents skill「何时不用」补自建资料路由行。**前端**:`kb.js` 两栏 modal(建/删库、上传 XHR、入库进度 3s 轮询、看/删单篇),rail 底部四按钮改图标+小字两行布局(用户 07-22 定)。冒烟:core 层 28 项全过(真实 markitdown + flash 摘要),TestClient API 15 项全过(usage 行实测 task_id=NULL + units 溯源),受影响存量测试(usage_accounting / system_prompt_paths)绿。 - **07-21 / 0.58.55 / 扫描件 PDF 直读(read_document,方舟文档理解)**:markitdown 只抽文本层,扫描件(老标准/检测报告/红头指南)转出为空=死路。探针(`scripts/probe_ark_doc.py`)验证方舟 chat file 内容块直读 PDF:格式 `{"type":"file","file":{filename,file_data}}` + `data:application/pdf;base64,` 前缀、base64 内联 17MB 可用(免 TOS)、单页栅格化 3600 万像素硬限(PIL 存 PDF 需标对 dpi)、~1300 输入 token/页(约 1 厘/页)、100 页魔术串全覆盖。落地 `tools/read_document.py`(seed_2_lite 同 variant 同 key,记账走 record_vision_usage):体积/页数双闸(30MB/100 页,pdfminer 软探页数免白付撞上下文)+ `finish_reason=length` 截断提示 + **多页 OCR `save_md` 全文落盘只返 1500 字预览**(防上下文爆);`image_ref.py` 抽 `load_pdf_as_data_url` 复用三形态路径解析与 user_root 边界。agent_builder 注册 + 系统提示 `_MEDIA_READDOC_SEG`(何时调/何时不调防重复花钱);六 skill(paper/patent/standard/proposal/rebuttal/ppt)摄取段加扫描件兜底一行。冒烟 `scripts/smoke_read_document.py` 全过(3 页 ¥0.0066,表格→md 表、公式→LaTeX)。选型对比(外部解析 API=新增第三方数据面 / 本地 OCR=过度投资)见 DESIGN §8.13。host 侧工具,**无需重建沙箱镜像**。 - **07-21 / 0.58.55 / 长对话点目录圆点首次跳不到位修复**:根因链=loadMessagesAround 后 renderMessages 尾部无条件滚底钉到窗口末尾,底部 sentinel 入视口立刻触发 loadNewerMessages 整窗重渲染删掉平滑滚动目标。修:renderMessages 加 stickBottom 参数(三个调窗口路径传 false);jumpToMessage 重建窗口后瞬时定位(auto)不留动画窗口期;`_msgScrollObserver` 在 `_outlineJumpLock` 期间不补载、解锁时对 sentinel 重投交叉状态。 - **07-21 / 0.58.54 / 手机端文件预览改悬浮卡片(四边留边距、不压输入区)**:用户反馈移动端预览弹框全屏贴边(100vw×100dvh、直角)观感差且糊在消息输入框上。改 `dev.html` ≤640px 媒体查询:遮罩层 `padding:10px` 四边留呼吸边距、底部 `calc(--preview-bottom-inset + 10px)` 在 chat-form 让位之上再加间隙;卡片尺寸改 `100%`(相对遮罩内容区,弃 100dvh,旧 WebView 兼容更稳),恢复默认圆角。输入区仍走既有 `body.fp-open #chat-form` z-index 抬升保持可用;桌面端不动。 @@ -226,6 +227,8 @@ core/session.py 153 ← ORM core/task.py 82 ← PG-backed TaskState core/skills.py 180 ← 多来源 registry(SkillSource)+ source 标记 + 覆盖感知(user wins)+ load_errors + container_dir core/memory.py 81 ← per-user `.memory/` dotfile +core/kb.py ~260 ← per-user `.kb/` 个人知识库(状态/视图/kb_block 注入) +core/kb_ingest.py ~280 ← kb 入库管线(markitdown→OCR 兜底→flash 摘要→INDEX) core/export_docx.py 383 core/storage/{__init__,engine,models,usage,utils}.py ← 4 表(0004-0007 演进);record_chat/image_usage core/ark_client.py 105 ← 火山方舟 HTTP 客户端 @@ -235,11 +238,11 @@ core/agent_builder.py 340 ← 装配 lib(有 ARK_API_KEY 才挂 SeedreamT core/executor.py / sandbox/{network,pool}.py / executor_docker.py ← Executor ABC + Docker per-user 容器池 tools/{base,fs,shell,run_python,skill_tool,skill_authoring,seedream,seedance,look_at_image,read_document,image_ref,web_search,web_fetch,documents,materials_project,transcribe_audio}.py ← read_document=扫描件 PDF OCR(方舟文档理解);image_ref=图/PDF 路径解析+base64 共享 main.py ~210 ← 入口:web / db / probe / user / sandbox check -db/migrations/versions/ 0001-0008 +db/migrations/versions/ 0001-0022 web/app.py ~1360 ← /v1 JSON API + user_id 隔离 + run lock + cancel + files + pptx 预览 + skills(列表/正文/删) web/auth.py ~190 ← 邮箱密码 + platform_key → JWT web/broker.py / sinks.py / pptx_render.py -web/static/dev.html + js/*.js ← dev SPA 拆 15 个零构建 ES module(main.js 入口;skills.js=技能查看 modal) +web/static/dev.html + js/*.js ← dev SPA 零构建 ES module(main.js 入口;skills.js=技能 modal;kb.js=知识库 modal) web/static/vendor/ ~1 MB ← jszip / docx-preview / xlsx ───────────────────────────────── Python 合计 ~3400 行(+ dev SPA + vendor 1MB);加 skills 脚本 + 配置,总仓库约 3800 行 diff --git a/core/__init__.py b/core/__init__.py index 5d2798f..3a09de4 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,3 +1,3 @@ # zcbot 版本号单一事实源:web/app.py 的 FastAPI version、/healthz 返回、前端展示都引这里。 # 改版本只动这一行。 -__version__ = "0.58.55" +__version__ = "0.59.0" diff --git a/core/agent_builder.py b/core/agent_builder.py index 7e539fa..f7da170 100644 --- a/core/agent_builder.py +++ b/core/agent_builder.py @@ -29,6 +29,7 @@ from rich.console import Console from core.capabilities import ModelCapabilities from core.executor_docker import DockerExecutor from core.executor_host import HostExecutor +from core.kb import kb_block from core.llm import LLM from core.loop import AgentLoop from core.memory import memory_block @@ -361,6 +362,11 @@ def _build_system_prompt( user_root(workspace_dir, user_id) / ".memory" ) prompt += memory_block(workspace_dir, user_id, mem_dir_display) + # 个人知识库 .kb/:路径换算同 .memory(docker 给容器路径);用户没建库时注空串零成本。 + kb_dir_display = "/workspace/.kb" if is_docker else str( + user_root(workspace_dir, user_id) / ".kb" + ) + prompt += kb_block(workspace_dir, user_id, kb_dir_display) if media_block: prompt += "\n\n" + media_block wd_abs = working_dir.resolve() diff --git a/core/kb.py b/core/kb.py new file mode 100644 index 0000000..2776af1 --- /dev/null +++ b/core/kb.py @@ -0,0 +1,288 @@ +"""个人知识库: `workspace/users//.kb/<库名>/` —— 纯文件,无向量无 DB。 + +照「记忆」机制范式(core/memory.py):真实文件为准 + agentic search,索引只是 +可重建的派生视图。每个库一个目录: + + .kb/<库名>/ + INDEX.md —— 库目录(单行一条,格式见 INDEX_LINE_FORMAT),注 prompt 的召回依据 + docs/.md —— 转换后的 markdown 正文(agent 用 read/grep 按需拉) + sources/ —— 原始文件(pdf/docx/...),留档 + 待入库队列 + +**「已入库」判据 = INDEX.md 有指向该 doc 的条目**;sources 有而 INDEX 无对应 doc += 待入库。由此入库天然幂等(重跑只补缺口)、崩溃可恢复(半截转换重来即可)、 +零 DB migration。入库编排在 core/kb_ingest.py,本模块只管状态读写与视图。 + +.kb 是 dotfile:被 GET /v1/files 天然隐藏、validate_task_name 拒 `.` 起头 —— +与 .memory/.skills 同款防呆。agent 无需新工具:fs 工具 user_root 内可读写, +docker 沙箱把整个 user_root bind 到 /workspace,.kb 随之可见。 +""" +from __future__ import annotations + +import re +import shutil +from pathlib import Path +from typing import Any, Dict, List, Optional +from uuid import UUID + +# INDEX 单行格式(全角 ｜ 分隔,摘要/关键词内允许半角标点)。agent 对话内手动入库 +# 与后台 ingest 产出同一格式 —— 契约文本(kb_block)里原样给出。 +INDEX_LINE_FORMAT = "- [标题](docs/<文件名>.md)｜来源 sources/<原件名>｜摘要：<两三句>｜关键词：<逗号分隔>" + +_INDEX_LINE_RE = re.compile( + r"^-\s*\[(?P[^\]]*)\]\((?P<doc>docs/[^)]+)\)" + r"\s*｜\s*来源\s*(?P<source>sources/[^｜]+?)" + r"\s*｜\s*摘要[:：]\s*(?P<summary>[^｜]*)" + r"(?:\s*｜\s*关键词[:：]\s*(?P<keywords>.*))?\s*$" +) + +# 库名:中文/字母/数字/-/_,拒 dotfile、路径分隔、Windows 保留字符。 +_KB_NAME_RE = re.compile(r"^[\w一-鿿][\w一-鿿\-. ]{0,39}$") +# 库内文件名(docs/ 与 sources/ 下的扁平文件):拒斜杠 / `..` / dotfile。 +_FILE_NAME_RE = re.compile(r"^[^/\\]{1,200}$") + + +def kb_root(workspace_dir: Path, user_id: UUID) -> Path: + return workspace_dir / "users" / str(user_id) / ".kb" + + +def is_safe_kb_name(name: str) -> bool: + if not name or name != name.strip() or name.startswith("."): + return False + if ".." in name or any(c in name for c in '/\\:*?"<>|'): + return False + return bool(_KB_NAME_RE.match(name)) + + +def is_safe_file_name(name: str) -> bool: + if not name or name.startswith(".") or ".." in name: + return False + return bool(_FILE_NAME_RE.match(name)) + + +def kb_dir(workspace_dir: Path, user_id: UUID, name: str) -> Optional[Path]: + """库目录(校验名字合法 + 落在 .kb 子树内);非法返回 None(调用方转 4xx)。""" + if not is_safe_kb_name(name): + return None + root = kb_root(workspace_dir, user_id).resolve() + d = (root / name).resolve() + if d.parent != root: + return None + return d + + +def parse_index(text: str) -> List[Dict[str, str]]: + """解析 INDEX.md → [{title, doc, source, summary, keywords}];不合格式的行忽略。""" + out: List[Dict[str, str]] = [] + for raw in text.splitlines(): + m = _INDEX_LINE_RE.match(raw.strip()) + if not m: + continue + out.append({ + "title": m.group("title").strip(), + "doc": m.group("doc").strip(), + "source": (m.group("source") or "").strip(), + "summary": (m.group("summary") or "").strip(), + "keywords": (m.group("keywords") or "").strip(), + }) + return out + + +def format_index_line(*, title: str, doc: str, source: str, summary: str, keywords: str) -> str: + """产出与 INDEX_LINE_FORMAT 一致的单行(后台 ingest 用;agent 侧照契约手写)。""" + def clean(s: str) -> str: + return " ".join((s or "").split()).replace("｜", "|") + return ( + f"- [{clean(title)}]({doc})｜来源 {source}" + f"｜摘要：{clean(summary)}｜关键词：{clean(keywords)}" + ) + + +def _read_index(d: Path) -> List[Dict[str, str]]: + p = d / "INDEX.md" + if not p.is_file(): + return [] + try: + return parse_index(p.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError): + return [] + + +def _list_sources(d: Path) -> List[str]: + src = d / "sources" + if not src.is_dir(): + return [] + return sorted(p.name for p in src.iterdir() if p.is_file() and not p.name.startswith(".")) + + +def pending_sources(d: Path) -> List[str]: + """sources 有而 INDEX 无 = 待入库(判据即幂等性来源,见模块注释)。""" + indexed = {e["source"].removeprefix("sources/") for e in _read_index(d)} + return [n for n in _list_sources(d) if n not in indexed] + + +def list_kbs(workspace_dir: Path, user_id: UUID) -> List[Dict[str, Any]]: + """所有库概览:[{name, doc_count, pending_count}],按名排序。""" + root = kb_root(workspace_dir, user_id) + if not root.is_dir(): + return [] + out: List[Dict[str, Any]] = [] + for d in sorted(root.iterdir()): + if not d.is_dir() or d.name.startswith("."): + continue + entries = _read_index(d) + out.append({ + "name": d.name, + "doc_count": len(entries), + "pending_count": len(pending_sources(d)), + }) + return out + + +def create_kb(workspace_dir: Path, user_id: UUID, name: str) -> Optional[Path]: + """建库(幂等):目录 + 空 INDEX.md + docs/ + sources/。名字非法返回 None。""" + d = kb_dir(workspace_dir, user_id, name) + if d is None: + return None + (d / "docs").mkdir(parents=True, exist_ok=True) + (d / "sources").mkdir(parents=True, exist_ok=True) + idx = d / "INDEX.md" + if not idx.exists(): + idx.write_text(f"# {name}\n\n", encoding="utf-8") + return d + + +def delete_kb(workspace_dir: Path, user_id: UUID, name: str) -> bool: + """整库删除(原件 + docs + INDEX 一起没,前端已二次确认)。""" + d = kb_dir(workspace_dir, user_id, name) + if d is None or not d.is_dir(): + return False + shutil.rmtree(d) + return True + + +def kb_detail(workspace_dir: Path, user_id: UUID, name: str) -> Optional[Dict[str, Any]]: + """单库全貌:INDEX 条目 + 待入库 sources 列表。库不存在返回 None。""" + d = kb_dir(workspace_dir, user_id, name) + if d is None or not d.is_dir(): + return None + return { + "name": name, + "entries": _read_index(d), + "pending": pending_sources(d), + } + + +def read_doc(workspace_dir: Path, user_id: UUID, name: str, filename: str) -> Optional[str]: + """读单篇 docs/<filename> 原文;非法 / 不存在 → None(调用方转 404)。""" + d = kb_dir(workspace_dir, user_id, name) + if d is None or not is_safe_file_name(filename) or not filename.endswith(".md"): + return None + target = (d / "docs" / filename).resolve() + if target.parent != (d / "docs").resolve() or not target.is_file(): + return None + try: + return target.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return None + + +def delete_doc(workspace_dir: Path, user_id: UUID, name: str, filename: str) -> bool: + """删单篇:docs 文件 + INDEX 对应行 + 对应 source 原件一起删(否则原件会被当 + 待入库重新转一遍 —— 判据使然)。""" + d = kb_dir(workspace_dir, user_id, name) + if d is None or not is_safe_file_name(filename) or not filename.endswith(".md"): + return False + doc_rel = f"docs/{filename}" + entries = _read_index(d) + hit = next((e for e in entries if e["doc"] == doc_rel), None) + target = (d / "docs" / filename).resolve() + if target.parent != (d / "docs").resolve(): + return False + if not target.is_file() and hit is None: + return False + if target.is_file(): + target.unlink() + if hit is not None: + src_name = hit["source"].removeprefix("sources/") + if is_safe_file_name(src_name): + src = d / "sources" / src_name + if src.is_file(): + src.unlink() + idx = d / "INDEX.md" + try: + lines = idx.read_text(encoding="utf-8").splitlines() + kept = [ + ln for ln in lines + if not (_INDEX_LINE_RE.match(ln.strip()) and f"({doc_rel})" in ln) + ] + idx.write_text("\n".join(kept).rstrip() + "\n", encoding="utf-8") + except (OSError, UnicodeDecodeError): + pass + return True + + +def save_source(workspace_dir: Path, user_id: UUID, name: str, filename: str, data: bytes) -> Optional[str]: + """上传原件落 sources/(同名覆盖 —— 重传即重新入库的自然语义)。 + 返回落盘文件名;库不存在 / 文件名非法 → None。""" + d = kb_dir(workspace_dir, user_id, name) + if d is None or not d.is_dir() or not is_safe_file_name(filename): + return None + src_dir = d / "sources" + src_dir.mkdir(parents=True, exist_ok=True) + # 覆盖旧 doc 判据:同名 source 若已在 INDEX,删掉旧条目让它重新排队入库 + doc_rel_hits = [e for e in _read_index(d) if e["source"] == f"sources/{filename}"] + for e in doc_rel_hits: + delete_doc(workspace_dir, user_id, name, e["doc"].removeprefix("docs/")) + (src_dir / filename).write_bytes(data) + return filename + + +# ── prompt 注入(照 memory_block 范式) ──────────────────────────────── + +_KB_CONTRACT = """\ +用法规矩: +- **路由**:找**材料学科学术文献**优先 `document_search`(院内共享大库);查**用户自建 + 资料**(个人上传的规范 / 报告 / 标准 / 内部文档)用下面的知识库 —— 先看各库 INDEX + 条目定位,再 `read` 对应 `docs/*.md` 正文,量大时 `grep` 先缩范围。 +- **答题标来源**:引用了哪个库哪篇就在回答里注明(标题或文件名)。 +- **对话内入库**:用户在对话里给了值得长期留的资料时,可直接写入:原件放 + `sources/`(没有原件就跳过)、正文转成 markdown 写 `docs/<slug>.md`、再往该库 + INDEX.md 追加一行,**格式必须是**: + `{fmt}` + (与后台自动入库产出一致;摘要写准 —— 它是下次召回的依据)。""" + + +def kb_block( + workspace_dir: Path, + user_id: UUID, + kb_dir_display: Optional[str] = None, +) -> str: + """构造注入 system prompt 的知识库段;用户没有任何库时返回空串(零成本)。 + + kb_dir_display: `.kb/` 在 agent 视角下的路径前缀(docker 传 `/workspace/.kb`, + host 传 None ⇒ 宿主绝对路径)—— 与 memory_block 的 mem_dir_display 同款约定。 + 注 INDEX 全文而非只注库名:INDEX 就是召回索引(照 memory extended 的 + description 逻辑),几十篇的个人库体量注得起;正文仍按需 read。 + """ + kbs = list_kbs(workspace_dir, user_id) + if not kbs: + return "" + root = kb_root(workspace_dir, user_id) + base = (kb_dir_display if kb_dir_display is not None else str(root)).rstrip("/") + + parts = ["\n\n## 个人知识库 (user 级,跨 task 共享)\n"] + parts.append(_KB_CONTRACT.replace("{fmt}", INDEX_LINE_FORMAT)) + for kb in kbs: + d = root / kb["name"] + parts.append(f"\n\n### 库「{kb['name']}」(`{base}/{kb['name']}/`)\n") + entries = _read_index(d) + if not entries: + parts.append("(空库,尚无已入库文档)\n") + continue + for e in entries: + kw = f"｜关键词：{e['keywords']}" if e["keywords"] else "" + parts.append( + f"- [{e['title']}](`{base}/{kb['name']}/{e['doc']}`)" + f"｜摘要：{e['summary']}{kw}\n" + ) + return "".join(parts) diff --git a/core/kb_ingest.py b/core/kb_ingest.py new file mode 100644 index 0000000..0731a2d --- /dev/null +++ b/core/kb_ingest.py @@ -0,0 +1,299 @@ +"""kb 入库后台任务:sources/ 待入库原件 → docs/*.md + 摘要 + INDEX 行。 + +流水线(每份原件独立,失败不连坐): + 1. markitdown Python API 抽文本(pdf/docx/pptx/xlsx/html/txt...) + 2. PDF 文本近零(扫描件)→ 方舟 seed-2.0-lite 文档理解 OCR 兜底 + (照 tools/read_document.py 同款请求体;ARK_API_KEY 未配则该件报错留待) + 3. deepseek flash 单次 chat 写 标题/摘要/关键词(照 core/context_fold.py 范式; + LLM 失败降级为文件名 + 正文开头,入库不阻塞) + 4. 追加 INDEX 行(core.kb.format_index_line)——写入即"已入库" + 5. record_chat_usage(kind="kb_ingest", task_id=None, units 带 {"kb","source"}) + +编排照定时执行器范式:web 层 asyncio.create_task(asyncio.to_thread(run_ingest, ...)), +本模块全同步;per-(user,库) threading.Lock 非阻塞抢占 —— 抢不到 = 已在入库,直接返回 +(上传即触发 + 手动触发天然去重)。进度状态存内存 dict 供前端轮询;崩溃丢状态无妨, +"待入库"判据在文件系统(INDEX 缺口),重触发即续跑。 +""" +from __future__ import annotations + +import base64 +import hashlib +import re +import threading +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +from uuid import UUID + +from core.ark_client import ArkClient, ArkError +from core.capabilities import ModelCapabilities +from core.kb import format_index_line, kb_dir, parse_index, pending_sources +from core.llm import LLM +from core.storage.usage import record_chat_usage, record_vision_usage + +# 摘要模型固定走最便宜档(与 web FALLBACK_MODEL_PROFILE 同值;单次几千 token,不随任务模型) +SUMMARY_PROFILE = "deepseek_v4.flash" +# 判定"文本近零"(扫描件)的阈值:markitdown 抽出的字符数低于此值即认为无文本层 +_MIN_TEXT_CHARS = 120 +# 喂给摘要模型的正文截断长度(开头段落通常含标题/摘要/目录,足够定性) +_SUMMARY_INPUT_CHARS = 5000 + +_OCR_QUESTION = ( + "这是一份多页 PDF 文档。请逐页把其中的文字完整 OCR 成 markdown:" + "每页以「== 第N页 ==」开头;表格转成 markdown 表格;保留标题层级与段落换行;" + "公式尽量用 LaTeX;不要总结、不要遗漏、不要自行补充原文没有的内容。" +) + +_SUMMARY_PROMPT = """\ +下面是文档「{filename}」的正文开头(可能被截断)。请为它写入库索引条目,严格按以下三行格式输出,不要输出任何其他文字: +标题：<文档标题,没有明确标题就概括一个,不超过 40 字> +摘要：<两三句话说清这份文档讲什么、有什么用,不超过 120 字> +关键词：<3-6 个检索关键词,用逗号分隔> + +正文开头: +{content}""" + +# ── 进度状态(内存,供 API 轮询) ───────────────────────────────────── +_guard = threading.Lock() +_locks: Dict[Tuple[str, str], threading.Lock] = {} +_status: Dict[Tuple[str, str], Dict[str, Any]] = {} + + +def _lock_for(key: Tuple[str, str]) -> threading.Lock: + with _guard: + return _locks.setdefault(key, threading.Lock()) + + +def ingest_status(user_id: UUID, name: str) -> Dict[str, Any]: + """当前/最近一次入库进度(无记录返回 idle);前端轮询 + 库详情附带。""" + st = _status.get((str(user_id), name)) + return dict(st) if st else {"running": False, "total": 0, "done": 0, "errors": []} + + +# ── 转换 ───────────────────────────────────────────────────────────── + +def _extract_text(src: Path) -> str: + """markitdown 抽文本;不支持的格式 / 解析失败抛 ValueError(留待重试)。""" + from markitdown import MarkItDown + try: + result = MarkItDown(enable_plugins=False).convert(str(src)) + except Exception as e: + raise ValueError(f"markitdown 转换失败: {type(e).__name__}: {e}") + return (result.text_content or "").strip() + + +def _ocr_pdf(src: Path, *, user_id: UUID, kb_name: str) -> str: + """扫描件 PDF 走方舟文档理解 OCR(照 tools/read_document.py 请求体)。 + + 失败抛 ValueError。记账 kind=vision、task_id=None、units 带 kb/source 溯源。 + """ + from core.ark_client import ArkConfig + ark_cfg = ArkConfig.load() + vision_cfg = (ark_cfg.raw.get("vision") or {}) if ark_cfg else {} + vis_key, cfg = "", None + for k, v in vision_cfg.items(): + if isinstance(v, dict): + vis_key, cfg = k, v + break + if ark_cfg is None or cfg is None: + raise ValueError("扫描件 PDF 需 OCR,但方舟(ARK_API_KEY / vision 段)未配置") + + max_bytes = int(float(cfg.get("max_pdf_mb", 30)) * 1024 * 1024) + size = src.stat().st_size + if size > max_bytes: + raise ValueError(f"PDF {size / 1e6:.1f}MB 超过 OCR 上限 {max_bytes / 1e6:.0f}MB") + + data_url = "data:application/pdf;base64," + base64.b64encode(src.read_bytes()).decode() + body = { + "model": cfg["model_id"], + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": _OCR_QUESTION}, + {"type": "file", "file": {"filename": src.name, "file_data": data_url}}, + ], + }], + } + timeout_s = float(cfg.get("doc_request_timeout_s", 600)) + try: + with ArkClient(ark_cfg, timeout_s=timeout_s) as client: + resp = client.post_json(cfg.get("endpoint", "/chat/completions"), body, timeout_s=timeout_s) + except ArkError as e: + raise ValueError(f"OCR 调用失败: {e}") + + choices = resp.get("choices") or [] + msg = (choices[0].get("message") if choices and isinstance(choices[0], dict) else None) or {} + content = msg.get("content") + if isinstance(content, list): + content = "\n".join( + c.get("text", "") for c in content if isinstance(c, dict) and c.get("type") == "text" + ) + text = (content or "").strip() if isinstance(content, (str,)) else "" + if not text: + raise ValueError("OCR 响应无文本(PDF 损坏或页面超像素上限)") + + usage = resp.get("usage") or {} + try: + record_vision_usage( + task_id=None, + user_id=user_id, + model_profile=f"doubao.{vis_key}", + prompt_tokens=int(usage.get("prompt_tokens", 0) or 0), + completion_tokens=int(usage.get("completion_tokens", 0) or 0), + input_cny_per_mtoken=float(cfg.get("price_cny_per_mtoken_input", 0)), + output_cny_per_mtoken=float(cfg.get("price_cny_per_mtoken_output", 0)), + extra_units={"kb": kb_name, "source": src.name}, + ) + except Exception: + pass # 记账失败不阻塞入库(与 loop 同纪律) + return text + + +# ── 摘要 ───────────────────────────────────────────────────────────── + +def _parse_summary(raw: str) -> Tuple[str, str, str]: + """解析三行格式;缺行返回空串由调用方兜底。""" + title = summary = keywords = "" + for line in raw.splitlines(): + line = line.strip().lstrip("*# ") + m = re.match(r"^(标题|摘要|关键词)[:：]\s*(.*)$", line) + if not m: + continue + key, val = m.group(1), m.group(2).strip() + if key == "标题" and not title: + title = val + elif key == "摘要" and not summary: + summary = val + elif key == "关键词" and not keywords: + keywords = val + return title, summary, keywords + + +def _summarize( + llm: LLM, caps: ModelCapabilities, *, text: str, filename: str, + user_id: UUID, kb_name: str, +) -> Tuple[str, str, str]: + """单次 chat 产 (标题, 摘要, 关键词);失败降级文件名 + 正文开头,不阻塞入库。""" + fallback = (Path(filename).stem, text[:100].replace("\n", " "), "") + try: + response = llm.chat(messages=[{ + "role": "user", + "content": _SUMMARY_PROMPT.format(filename=filename, content=text[:_SUMMARY_INPUT_CHARS]), + }], tools=None) + choices = getattr(response, "choices", None) or [] + raw = ((choices[0].message.content if choices else "") or "").strip() + except Exception as e: + print(f"[kb_ingest] summary llm failed ({filename}): {type(e).__name__}: {e}", flush=True) + return fallback + usage = getattr(response, "usage", None) + try: + record_chat_usage( + task_id=None, + user_id=user_id, + message_id=None, + model_profile=f"{caps.family}.{caps.variant}", + prompt_tokens=getattr(usage, "prompt_tokens", 0) or 0, + completion_tokens=getattr(usage, "completion_tokens", 0) or 0, + input_cny_per_mtoken=caps.input_cny_per_mtoken, + output_cny_per_mtoken=caps.output_cny_per_mtoken, + response=response, + kind="kb_ingest", + extra_units={"kb": kb_name, "source": filename}, + ) + except Exception: + pass + title, summary, keywords = _parse_summary(raw) + return (title or fallback[0], summary or fallback[1], keywords) + + +# ── 主流程 ─────────────────────────────────────────────────────────── + +def _doc_name_for(d: Path, source_name: str) -> str: + """source 原件名 → docs/ 下的 md 文件名;与既有他源文档撞名时加短 hash 后缀。""" + stem = Path(source_name).stem + stem = re.sub(r'[\\/:*?"<>|｜\[\]()]+', "_", stem).strip("._ ") or "doc" + stem = stem[:80] + entries = parse_index((d / "INDEX.md").read_text(encoding="utf-8")) if (d / "INDEX.md").is_file() else [] + doc_rel = f"docs/{stem}.md" + for e in entries: + if e["doc"] == doc_rel and e["source"] != f"sources/{source_name}": + suffix = hashlib.md5(source_name.encode("utf-8")).hexdigest()[:6] + return f"{stem}_{suffix}.md" + return f"{stem}.md" + + +def _append_index(d: Path, line: str) -> None: + idx = d / "INDEX.md" + text = idx.read_text(encoding="utf-8") if idx.is_file() else "" + if text and not text.endswith("\n"): + text += "\n" + idx.write_text(text + line + "\n", encoding="utf-8") + + +def run_ingest(workspace_dir: Path, user_id: UUID, kb_name: str, models_dir: Path) -> bool: + """同步跑一轮入库(web 层用 asyncio.to_thread 下沉)。 + + 返回 False = 没跑(锁被占,已有入库在进行 / 库不存在);True = 跑完(含空转)。 + 幂等:只处理 INDEX 缺口,单件失败记入 status.errors 并保持待入库,下次重触发续跑。 + """ + d = kb_dir(workspace_dir, user_id, kb_name) + if d is None or not d.is_dir(): + return False + key = (str(user_id), kb_name) + lock = _lock_for(key) + if not lock.acquire(blocking=False): + return False + try: + pending = pending_sources(d) + st: Dict[str, Any] = { + "running": True, "total": len(pending), "done": 0, + "current": None, "errors": [], + "started_at": datetime.now().isoformat(timespec="seconds"), + } + _status[key] = st + if not pending: + return True + + llm: Optional[LLM] = None + caps: Optional[ModelCapabilities] = None + try: + caps = ModelCapabilities.load(SUMMARY_PROFILE, models_dir) + llm = LLM(caps) + except Exception as e: + print(f"[kb_ingest] summary model load failed: {type(e).__name__}: {e}", flush=True) + + for name in pending: + st["current"] = name + src = d / "sources" / name + try: + text = _extract_text(src) + if len(text) < _MIN_TEXT_CHARS and src.suffix.lower() == ".pdf": + text = _ocr_pdf(src, user_id=user_id, kb_name=kb_name) + if len(text) < _MIN_TEXT_CHARS: + raise ValueError(f"抽出文本过少({len(text)} 字符),无法入库") + doc_name = _doc_name_for(d, name) + (d / "docs").mkdir(parents=True, exist_ok=True) + (d / "docs" / doc_name).write_text(text, encoding="utf-8") + if llm is not None and caps is not None: + title, summary, keywords = _summarize( + llm, caps, text=text, filename=name, user_id=user_id, kb_name=kb_name + ) + else: + title, summary, keywords = Path(name).stem, text[:100].replace("\n", " "), "" + _append_index(d, format_index_line( + title=title, doc=f"docs/{doc_name}", source=f"sources/{name}", + summary=summary, keywords=keywords, + )) + st["done"] += 1 + except Exception as e: + msg = str(e) if isinstance(e, ValueError) else f"{type(e).__name__}: {e}" + st["errors"].append({"source": name, "error": msg}) + print(f"[kb_ingest] {kb_name}/{name} failed: {msg}", flush=True) + return True + finally: + st = _status.get(key) + if st is not None: + st["running"] = False + st["current"] = None + st["finished_at"] = datetime.now().isoformat(timespec="seconds") + lock.release() diff --git a/core/storage/models.py b/core/storage/models.py index b8fc5cb..a36b364 100644 --- a/core/storage/models.py +++ b/core/storage/models.py @@ -163,10 +163,12 @@ class UsageEvent(Base): user_id: Mapped[UUID] = mapped_column( PG_UUID(as_uuid=True), ForeignKey("users.user_id"), nullable=False ) - task_id: Mapped[UUID] = mapped_column( + # NULL = 非 task 维度的用量(0022 放宽)。目前仅 kind="kb_ingest"(知识库入库, + # 溯源在 units JSONB:{"kb": 库名, "source": 原件名}),对账按 kind 聚合即可区分。 + task_id: Mapped[Optional[UUID]] = mapped_column( PG_UUID(as_uuid=True), ForeignKey("tasks.task_id", ondelete="CASCADE"), - nullable=False, + nullable=True, ) message_id: Mapped[Optional[UUID]] = mapped_column( PG_UUID(as_uuid=True), diff --git a/core/storage/usage.py b/core/storage/usage.py index 4b14054..3faa180 100644 --- a/core/storage/usage.py +++ b/core/storage/usage.py @@ -78,7 +78,7 @@ def _fallback_chat_cost_cny( def record_chat_usage( *, - task_id: UUID, + task_id: Optional[UUID], user_id: UUID, message_id: Optional[UUID], model_profile: str, @@ -99,6 +99,8 @@ def record_chat_usage( `model_profile` 形如 `"deepseek_v4.pro"`(family.variant)。 `kind` 默认 "chat";chat 形态的辅助调用可传别的标签(如 "context_fold"), 按 kind GROUP BY 对账 + 派生统计(折叠次数 = count)—— prompt_optimize 同范式。 + `task_id` 可为 None(0022):非 task 维度的调用(如 kind="kb_ingest")没有 task + 可挂,溯源靠 kind + extra_units(kb_ingest 带 {"kb": 库名, "source": 原件名})。 返回算出的 cost_cny(已落库),调用方可用作 SSE 显示。 """ cost_usd = _safe_chat_cost_usd(response) @@ -395,7 +397,7 @@ def record_video_usage( def record_vision_usage( *, - task_id: UUID, + task_id: Optional[UUID], user_id: UUID, model_profile: str, prompt_tokens: int, diff --git a/db/migrations/versions/20260722_1000_0022_usage_events_task_id_nullable.py b/db/migrations/versions/20260722_1000_0022_usage_events_task_id_nullable.py new file mode 100644 index 0000000..2383594 --- /dev/null +++ b/db/migrations/versions/20260722_1000_0022_usage_events_task_id_nullable.py @@ -0,0 +1,38 @@ +"""usage_events.task_id 放宽为 NULL(知识库入库记账). + +Revision ID: 0022 +Revises: 0021 +Create Date: 2026-07-22 + +kb 入库(kind="kb_ingest")在任何 task 之外跑(上传/手动触发的后台任务), +没有 task 上下文可挂 —— task_id NOT NULL 挡住这类事件。放宽为 NULL: +纯放宽约束,存量行全部满足,零数据迁移;NULL = "非 task 维度的用量事件"。 +统计侧不受影响(按 user/kind 聚合的 query 不依赖 task_id 非空)。 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +revision: str = "0022" +down_revision: Union[str, None] = "0021" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "usage_events", "task_id", + existing_type=sa.dialects.postgresql.UUID(as_uuid=True), + nullable=True, + ) + + +def downgrade() -> None: + # 回滚前提:已无 task_id IS NULL 的行(kb_ingest 事件需先清理),否则失败。 + op.alter_column( + "usage_events", "task_id", + existing_type=sa.dialects.postgresql.UUID(as_uuid=True), + nullable=False, + ) diff --git a/skills/documents/SKILL.md b/skills/documents/SKILL.md index 3dcd215..7215a63 100644 --- a/skills/documents/SKILL.md +++ b/skills/documents/SKILL.md @@ -20,6 +20,7 @@ description: 查内部材料学科知识库(document_search API,7 个学科:胶 - 用户只问通识(直接答) - 用户已经给了具体内部文档路径(直接读,不要二次校验) +- 用户问的是**自己上传的自建资料**(规范 / 报告 / 内部文档)→ 走个人知识库(system prompt「个人知识库」段注入的 `.kb/` 索引,fs 工具直接 read/grep),不在本库搜 ## 三个 tool diff --git a/web/app.py b/web/app.py index 031ecb6..2b15c55 100644 --- a/web/app.py +++ b/web/app.py @@ -866,6 +866,10 @@ class ChangePasswordRequest(BaseModel): new_password: str +class KbCreateRequest(BaseModel): + name: str # 库名(中文/字母/数字/-/_,拒 dotfile 与路径字符,≤40 字符) + + # ────────────────────── App 工厂 ────────────────────── # web/static 目录路径 — /static 静态挂载用,dev.html 也放这 @@ -2520,6 +2524,132 @@ def create_app() -> FastAPI: raise HTTPException(404, f"memory file not found: {filename!r}") return {"filename": filename, "content": content} + # ───────────── 个人知识库(纯文件 .kb/,照记忆机制范式)───────────── + # 状态全在 FS(core/kb.py),端点只是薄壳;入库后台任务照定时执行器范式 + # (create_task + to_thread,per-(user,库) 锁在 core/kb_ingest.py 内去重)。 + # 不设 HTTP 检索端点 —— agent 侧走 fs 工具 read/grep(kb_block 注入契约)。 + + _kb_ingest_tasks: set = set() # 持引用防 GC(fire-and-forget task 的惯例) + + def _spawn_kb_ingest(user_id: UUID, name: str) -> None: + from core.agent_builder import load_config, resolve_workspace + from core.kb_ingest import run_ingest + cfg = load_config() + ws = resolve_workspace(None) + t = asyncio.create_task( + asyncio.to_thread(run_ingest, ws, user_id, name, ROOT / cfg["models_dir"]) + ) + _kb_ingest_tasks.add(t) + t.add_done_callback(_kb_ingest_tasks.discard) + + @app.get("/v1/kb", tags=["kb"]) + def kb_list(user_id: UUID = Depends(require_user)): + """所有库概览:[{name, doc_count, pending_count}]。前端入口一次拉满。""" + from core.agent_builder import resolve_workspace + from core.kb import list_kbs + return {"results": list_kbs(resolve_workspace(None), user_id)} + + @app.post("/v1/kb", tags=["kb"]) + def kb_create(body: KbCreateRequest, user_id: UUID = Depends(require_user)): + """建库(幂等)。名字非法 → 400。""" + from core.agent_builder import resolve_workspace + from core.kb import create_kb + name = (body.name or "").strip() + if create_kb(resolve_workspace(None), user_id, name) is None: + raise HTTPException(400, f"invalid kb name: {body.name!r}") + return {"name": name} + + @app.get("/v1/kb/{name}", tags=["kb"]) + def kb_detail_ep(name: str, user_id: UUID = Depends(require_user)): + """单库全貌:INDEX 条目 + 待入库列表 + 入库进度(前端轮询这里)。""" + from core.agent_builder import resolve_workspace + from core.kb import kb_detail + from core.kb_ingest import ingest_status + detail = kb_detail(resolve_workspace(None), user_id, name) + if detail is None: + raise HTTPException(404, f"kb not found: {name!r}") + detail["ingest"] = ingest_status(user_id, name) + return detail + + @app.delete("/v1/kb/{name}", tags=["kb"]) + def kb_delete(name: str, user_id: UUID = Depends(require_user)): + """整库删除(原件 + docs + INDEX)。入库进行中 → 409(避免半截写盘)。""" + from core.agent_builder import resolve_workspace + from core.kb import delete_kb + from core.kb_ingest import ingest_status + if ingest_status(user_id, name).get("running"): + raise HTTPException(409, "该库正在入库,等入库结束再删除") + if not delete_kb(resolve_workspace(None), user_id, name): + raise HTTPException(404, f"kb not found: {name!r}") + return {"deleted": name} + + @app.post("/v1/kb/{name}/upload", tags=["kb"]) + async def kb_upload( + name: str, + files: list[UploadFile] = File(...), + user_id: UUID = Depends(require_user), + ): + """上传原件到 sources/ 并立即触发入库(上传即入库)。 + + 同名覆盖 = 重新入库(core/kb.py::save_source 会摘掉旧 INDEX 条目重排队)。 + 磁盘配额 gate 与 /v1/files/upload 同款。 + """ + from core.agent_builder import load_config as _load_cfg, resolve_workspace + from core.kb import kb_dir, save_source + from core.storage.disk_quota import check_disk_quota, parse_bytes + _quotas_cfg = (_load_cfg().get("quotas") or {}) + _limit = parse_bytes(_quotas_cfg.get("disk_bytes_per_user")) + if _limit is not None and _limit > 0: + _err = check_disk_quota(user_id, _limit) + if _err is not None: + raise HTTPException(413, _err) + + ws = resolve_workspace(None) + d = kb_dir(ws, user_id, name) + if d is None or not d.is_dir(): + raise HTTPException(404, f"kb not found: {name!r}") + saved: list[dict] = [] + for up in files or []: + raw_name = up.filename or "" + data = await up.read() + ok = save_source(ws, user_id, name, raw_name, data) + if ok is None: + raise HTTPException(400, f"invalid filename: {raw_name!r}") + saved.append({"name": ok, "size": len(data)}) + if not saved: + raise HTTPException(400, "no files uploaded") + _spawn_kb_ingest(user_id, name) + return {"count": len(saved), "saved": saved} + + @app.post("/v1/kb/{name}/ingest", tags=["kb"]) + async def kb_ingest_trigger(name: str, user_id: UUID = Depends(require_user)): + """手动触发入库(兜待入库缺口:上传时崩溃 / 单件失败重试)。幂等,已在跑则空转。""" + from core.agent_builder import resolve_workspace + from core.kb import kb_dir + if kb_dir(resolve_workspace(None), user_id, name) is None: + raise HTTPException(404, f"kb not found: {name!r}") + _spawn_kb_ingest(user_id, name) + return {"started": True} + + @app.get("/v1/kb/{name}/docs/{filename}", tags=["kb"]) + def kb_doc_read(name: str, filename: str, user_id: UUID = Depends(require_user)): + """读单篇转换后 markdown(点开列表项时拉)。穿越校验收口 core/kb.py::read_doc。""" + from core.agent_builder import resolve_workspace + from core.kb import read_doc + content = read_doc(resolve_workspace(None), user_id, name, filename) + if content is None: + raise HTTPException(404, f"doc not found: {filename!r}") + return {"filename": filename, "content": content} + + @app.delete("/v1/kb/{name}/docs/{filename}", tags=["kb"]) + def kb_doc_delete(name: str, filename: str, user_id: UUID = Depends(require_user)): + """删单篇(docs 文件 + INDEX 行 + source 原件,防原件被重新入库)。""" + from core.agent_builder import resolve_workspace + from core.kb import delete_doc + if not delete_doc(resolve_workspace(None), user_id, name, filename): + raise HTTPException(404, f"doc not found: {filename!r}") + return {"deleted": filename} + # ───────────── 定时任务(DESIGN §8.5)───────────── # 前端只读展示 + 停用/删除两个便捷动作;建/改全走对话(schedule_* 工具)。 # 与对话工具共用 core.scheduler 服务层,两条路径不漂移。 diff --git a/web/static/dev.html b/web/static/dev.html index d0980f1..c9d31db 100644 --- a/web/static/dev.html +++ b/web/static/dev.html @@ -248,14 +248,15 @@ .app-msg.error::before { content: "\2715"; color: var(--c-red); } .app-msg.info::before { content: "\2139"; color: var(--c-blue); } - /* ───── 左侧 rail 底部「我的资源」入口(技能,后续可加记忆)───── */ + /* ───── 左侧 rail 底部「我的资源」入口(技能 / 记忆 / 知识库 / 定时)───── + 四个并列后横排放不下 → 图标在上、小字在下两行布局 */ #rail-resources { flex-shrink: 0; border-top: 1px solid var(--border); padding: 8px; display: flex; gap: 6px; } #rail-resources > button { - flex: 1; font-size: 13px; - display: inline-flex; align-items: center; justify-content: center; gap: 6px; + flex: 1; font-size: 11px; padding: 6px 2px 4px; + display: inline-flex; flex-direction: column; align-items: center; justify-content: center; gap: 3px; } #rail-resources > button svg { flex-shrink: 0; opacity: .85; } /* 版本号:钉在右侧文件面板底部存储条最左,带细分隔线;点击弹更新日志, @@ -474,6 +475,48 @@ #mem-cols { flex-direction: column; } } + /* ───── 个人知识库 modal(两栏;上传/删除在 GUI,检索走对话)───── */ + #kb-modal { z-index: 112; } + #kb-modal .card { + width: 880px; max-width: 94vw; height: 80vh; max-height: 80vh; + display: flex; flex-direction: column; + } + #kb-modal h3 { + margin: 0; padding: 12px 16px; font-size: 16px; + border-bottom: 1px solid var(--border); + display: flex; align-items: center; gap: 8px; + } + #kb-modal h3 .spacer { flex: 1; } + #kb-modal h3 svg { opacity: .85; } + #kb-modal .sk-x { + border: none; background: transparent; font-size: 16px; + cursor: pointer; color: var(--muted); padding: 2px 6px; + } + #kb-hint { + padding: 8px 16px; font-size: 12px; color: var(--muted); + border-bottom: 1px solid var(--border); background: #fafafa; + } + #kb-cols { flex: 1; display: flex; min-height: 0; } + #kb-detail { flex: 1; min-width: 0; overflow: auto; padding: 16px 20px; } + #kb-detail .sk-d-head { display: flex; align-items: center; gap: 8px; } + #kb-detail .sk-d-head .spacer { flex: 1; } + .kb-ingest-bar { + margin-top: 10px; padding: 6px 10px; font-size: 12px; border-radius: 6px; + background: color-mix(in srgb, var(--accent, #2563eb) 10%, transparent); + } + .kb-ingest-err { margin-top: 8px; font-size: 12px; color: #b45309; } + .kb-pending { padding: 3px 0; font-size: 12px; color: var(--muted); } + .kb-entry { display: flex; align-items: flex-start; gap: 6px; padding: 6px 0; border-bottom: 1px dashed var(--border); } + .kb-entry-main { flex: 1; min-width: 0; cursor: pointer; } + .kb-entry-main:hover .sk-name { text-decoration: underline; } + .kb-entry .kb-doc-del { flex-shrink: 0; opacity: .55; } + .kb-entry:hover .kb-doc-del { opacity: 1; } + @media (max-width: 760px) { + #kb-modal .card { width: 96vw; height: 88vh; max-height: 88vh; } + #kb-cols { flex-direction: column; } + #kb-list { width: auto; max-height: 30vh; border-right: none; border-bottom: 1px solid var(--border); } + } + /* ───── 3-pane layout ───── */ #app { display: none; height: 100vh; } #app.ready { @@ -1627,6 +1670,24 @@ </div> </div> +<!-- ───── 个人知识库 modal(纯文件 .kb/;上传即入库,检索走对话)───── --> +<div id="kb-modal" class="modal"> + <div class="card"> + <h3> + <svg viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><ellipse cx="12" cy="5" rx="9" ry="3"></ellipse><path d="M21 12c0 1.66-4 3-9 3s-9-1.34-9-3"></path><path d="M3 5v14c0 1.66 4 3 9 3s9-1.34 9-3V5"></path></svg> + <span>个人知识库</span> + <span class="spacer"></span> + <button id="kb-close" class="sk-x" title="关闭">✕</button> + </h3> + <div id="kb-hint">上传资料(PDF / Word / PPT / Excel / 网页 / 文本)自动转成 markdown 并写摘要入库,之后<b>在对话里直接问</b>,我会按索引查你的库并标注来源。扫描件 PDF 自动 OCR。</div> + <div id="kb-cols"> + <div id="kb-list" class="sk-pane"><div class="muted" style="padding:8px;">加载中…</div></div> + <div id="kb-detail"><div class="sk-empty">← 选一个库查看,或「+ 新建库」</div></div> + </div> + </div> +</div> +<input type="file" id="kb-upload-input" multiple style="display:none;" /> + <!-- ───── embed-mode waiting overlay (token 握手中) ───── --> <div id="embed-waiting"> <div class="spinner"></div> @@ -1706,6 +1767,10 @@ <svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M4 19.5A2.5 2.5 0 0 1 6.5 17H20"></path><path d="M6.5 2H20v20H6.5A2.5 2.5 0 0 1 4 19.5v-15A2.5 2.5 0 0 1 6.5 2z"></path></svg> <span>记忆</span> </button> + <button id="hd-kb" title="个人知识库(上传资料自动入库,对话可查)"> + <svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><ellipse cx="12" cy="5" rx="9" ry="3"></ellipse><path d="M21 12c0 1.66-4 3-9 3s-9-1.34-9-3"></path><path d="M3 5v14c0 1.66 4 3 9 3s9-1.34 9-3V5"></path></svg> + <span>知识库</span> + </button> <button id="hd-crons" title="查看定时任务(建 / 改请在对话里说)"> <svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="9"></circle><path d="M12 7v5l3 2"></path></svg> <span>定时</span> diff --git a/web/static/js/kb.js b/web/static/js/kb.js new file mode 100644 index 0000000..1d6969a --- /dev/null +++ b/web/static/js/kb.js @@ -0,0 +1,239 @@ +// 个人知识库 modal:左栏列库,右栏库详情(已入库条目 + 待入库 + 入库进度)/ 单篇正文。 +// 照 memory.js master-detail 范式;删除走 dialogConfirm(crons.js 同款);上传用 XHR +// FormData(files.js 同款,带 Bearer);入库进行中每 3s 轮询库详情(procs.js 自适应启停)。 +// 后端:/v1/kb*(core/kb.py 纯文件,真实文件为准);检索侧 agent 走 fs 工具,前端只当"眼睛+搬运工"。 +import { $ } from "./dom.js"; +import { api } from "./api.js"; +import { state } from "./state.js"; +import { escapeHtml } from "./format.js"; +import { renderMd, highlightIn } from "./markdown.js"; +import { dialogConfirm, dialogPrompt, message } from "./dialog.js"; + +const PLACEHOLDER = '<div class="sk-empty">← 选一个库查看,或「+ 新建库」</div>'; +const POLL_MS = 3000; +let _current = null; // 当前选中库名 +let _pollTimer = null; + +function openKbModal() { + $("kb-modal").classList.add("show"); + $("kb-detail").innerHTML = PLACEHOLDER; + _current = null; + renderList(); +} +export function closeKbModal() { + $("kb-modal").classList.remove("show"); + stopPolling(); +} + +function startPolling() { + if (!_pollTimer) _pollTimer = setInterval(() => { if (_current) showKb(_current, null, true); }, POLL_MS); +} +function stopPolling() { + if (_pollTimer) { clearInterval(_pollTimer); _pollTimer = null; } +} + +// ───── 左栏:库列表 ───── +async function renderList() { + const list = $("kb-list"); + let data; + try { + data = await api("GET", "/v1/kb"); + } catch (e) { + list.innerHTML = `<div class="err" style="padding:8px;">加载失败: ${escapeHtml(e.message)}</div>`; + return; + } + const kbs = data.results || []; + let html = '<div class="sk-group-title">我的库</div>'; + html += kbs.length + ? kbs.map((k) => `<div class="sk-item${k.name === _current ? " active" : ""}" data-kb="${escapeHtml(k.name)}"> + <div class="sk-name">${escapeHtml(k.name)}${k.pending_count ? ` <span class="sk-badge">待入库 ${k.pending_count}</span>` : ""}</div> + <div class="sk-desc">${k.doc_count} 篇文档</div> + </div>`).join("") + : '<div class="muted" style="padding:4px 8px;font-size:12px;">还没有库。建一个,把常用资料(规范 / 报告 / 标准…)传进来,以后对话我能直接查。</div>'; + html += '<div style="padding:8px;"><button id="kb-new" class="small" style="width:100%;">+ 新建库</button></div>'; + list.innerHTML = html; +} + +async function createKb() { + const name = await dialogPrompt({ + title: "新建知识库", + message: "库名(中文 / 字母 / 数字,如「行业标准」「项目资料」):", + okText: "创建", + }); + if (!name || !name.trim()) return; + try { + await api("POST", "/v1/kb", { name: name.trim() }); + } catch (e) { message("创建失败: " + e.message, "error"); return; } + await renderList(); + showKb(name.trim(), null); +} + +// ───── 右栏:库详情 ───── +function highlightSel(name) { + $("kb-list").querySelectorAll(".sk-item").forEach((el) => { + el.classList.toggle("active", el.getAttribute("data-kb") === name); + }); +} + +async function showKb(name, itemEl, isPoll = false) { + _current = name; + if (!isPoll) { + highlightSel(name); + $("kb-detail").innerHTML = '<div class="muted" style="padding:8px;">加载中…</div>'; + } + let d; + try { + d = await api("GET", "/v1/kb/" + encodeURIComponent(name)); + } catch (e) { + if (!isPoll) $("kb-detail").innerHTML = `<div class="err" style="padding:8px;">加载失败: ${escapeHtml(e.message)}</div>`; + stopPolling(); + return; + } + if (_current !== name) return; // 轮询回来时已切走 + renderDetail(d); + if (d.ingest && d.ingest.running) startPolling(); else stopPolling(); +} + +function renderDetail(d) { + const ing = d.ingest || {}; + const entries = d.entries || []; + const pending = d.pending || []; + let html = `<div class="sk-d-head"> + <span class="sk-d-name">${escapeHtml(d.name)}</span> + <span class="sk-badge">${entries.length} 篇</span> + <span class="spacer"></span> + <button id="kb-upload-btn" class="small primary">上传文件</button> + <button id="kb-del" class="small danger" data-kb="${escapeHtml(d.name)}">删除库</button> + </div>`; + + if (ing.running) { + html += `<div class="kb-ingest-bar">入库中… ${ing.done}/${ing.total}${ing.current ? " · " + escapeHtml(ing.current) : ""}</div>`; + } + if (ing.errors && ing.errors.length) { + html += `<div class="kb-ingest-err">${ing.errors.map((x) => + `<div>⚠ ${escapeHtml(x.source)}:${escapeHtml(x.error)}</div>`).join("")}</div>`; + } + if (pending.length) { + html += `<div class="sk-group-title" style="margin-top:10px;">待入库 (${pending.length})</div>`; + html += pending.map((n) => `<div class="kb-pending">${escapeHtml(n)}</div>`).join(""); + if (!ing.running) { + html += `<div style="padding:6px 0;"><button id="kb-reingest" class="small">重试入库</button></div>`; + } + } + html += '<div class="sk-group-title" style="margin-top:10px;">已入库</div>'; + html += entries.length + ? entries.map((e) => `<div class="kb-entry"> + <div class="kb-entry-main" data-doc="${escapeHtml(e.doc.replace(/^docs\//, ""))}"> + <div class="sk-name">${escapeHtml(e.title)}</div> + <div class="sk-desc">${escapeHtml(e.summary)}${e.keywords ? " ｜ " + escapeHtml(e.keywords) : ""}</div> + </div> + <button class="small danger kb-doc-del" data-doc="${escapeHtml(e.doc.replace(/^docs\//, ""))}" title="删除该篇(含原件)">✕</button> + </div>`).join("") + : '<div class="muted" style="padding:4px 0;font-size:12px;">空库。点「上传文件」把资料传进来,自动转 markdown 并写摘要。</div>'; + $("kb-detail").innerHTML = html; +} + +// ───── 单篇查看 ───── +async function showDoc(filename) { + const detail = $("kb-detail"); + detail.innerHTML = '<div class="muted" style="padding:8px;">加载中…</div>'; + let data; + try { + data = await api("GET", "/v1/kb/" + encodeURIComponent(_current) + "/docs/" + encodeURIComponent(filename)); + } catch (e) { + detail.innerHTML = `<div class="err" style="padding:8px;">加载失败: ${escapeHtml(e.message)}</div>`; + return; + } + detail.innerHTML = + `<div class="sk-d-head"><button id="kb-back" class="small">← 返回</button> + <span class="sk-d-name">${escapeHtml(filename)}</span></div>` + + `<div class="sk-detail-md">${renderMd(data.content)}</div>`; + highlightIn(detail); +} + +// ───── 上传(files.js 同款 XHR,带进度)───── +function uploadTo(name, fileList) { + const files = Array.from(fileList || []); + if (!files.length) return; + const fd = new FormData(); + for (const f of files) fd.append("files", f); + const xhr = new XMLHttpRequest(); + xhr.open("POST", "/v1/kb/" + encodeURIComponent(name) + "/upload"); + xhr.setRequestHeader("Authorization", "Bearer " + state.token); + xhr.onload = () => { + let payload = null; + try { payload = JSON.parse(xhr.responseText); } catch (e) {} + if (xhr.status >= 200 && xhr.status < 300) { + message(`已上传 ${payload && payload.count || files.length} 个文件,开始入库…`); + showKb(name, null); // 立即刷详情;running 会自动开轮询 + } else { + const msg = (payload && payload.detail) || (xhr.status + " " + xhr.statusText); + message("上传失败: " + msg, "error"); + } + }; + xhr.onerror = () => message("上传失败: 网络错误", "error"); + xhr.send(fd); + message("上传中…"); +} + +// ───── 顶层绑定 ───── +$("hd-kb").onclick = openKbModal; +$("kb-close").onclick = closeKbModal; +$("kb-modal").addEventListener("click", (e) => { + if (e.target.id === "kb-modal") closeKbModal(); // 点遮罩关闭 +}); +$("kb-list").addEventListener("click", (e) => { + if (e.target.id === "kb-new" || e.target.closest("#kb-new")) { createKb(); return; } + const item = e.target.closest(".sk-item"); + if (item) showKb(item.getAttribute("data-kb"), item); +}); +$("kb-upload-input").addEventListener("change", () => { + const inp = $("kb-upload-input"); + if (_current) uploadTo(_current, inp.files); + inp.value = ""; +}); +$("kb-detail").addEventListener("click", async (e) => { + if (e.target.id === "kb-upload-btn") { $("kb-upload-input").click(); return; } + if (e.target.id === "kb-back") { showKb(_current, null); return; } + if (e.target.id === "kb-reingest") { + try { await api("POST", "/v1/kb/" + encodeURIComponent(_current) + "/ingest"); } catch (err) { + message("触发失败: " + err.message, "error"); return; + } + showKb(_current, null); + return; + } + const del = e.target.closest("#kb-del"); + if (del) { + const name = del.getAttribute("data-kb"); + if (!await dialogConfirm({ + title: "删除知识库", + message: `删除库「${name}」?其中所有文档与原件将一并删除,不可恢复。`, + okText: "删除", danger: true, + })) return; + try { await api("DELETE", "/v1/kb/" + encodeURIComponent(name)); } catch (err) { + message("删除失败: " + err.message, "error"); return; + } + _current = null; + stopPolling(); + $("kb-detail").innerHTML = PLACEHOLDER; + await renderList(); + return; + } + const docDel = e.target.closest(".kb-doc-del"); + if (docDel) { + const doc = docDel.getAttribute("data-doc"); + if (!await dialogConfirm({ + title: "删除文档", + message: `删除「${doc}」?对应原件与索引条目一并删除,不可恢复。`, + okText: "删除", danger: true, + })) return; + try { + await api("DELETE", "/v1/kb/" + encodeURIComponent(_current) + "/docs/" + encodeURIComponent(doc)); + } catch (err) { message("删除失败: " + err.message, "error"); return; } + showKb(_current, null); + renderList(); // 左栏计数同步 + return; + } + const main = e.target.closest(".kb-entry-main"); + if (main) showDoc(main.getAttribute("data-doc")); +}); diff --git a/web/static/js/main.js b/web/static/js/main.js index 3c1068d..b225cb1 100644 --- a/web/static/js/main.js +++ b/web/static/js/main.js @@ -8,6 +8,7 @@ import { api } from "./api.js"; import { closeChpwModal } from "./auth.js"; import { closeSkillsModal } from "./skills.js"; import { closeMemoryModal } from "./memory.js"; +import { closeKbModal } from "./kb.js"; import { closeCronsModal } from "./crons.js"; import { closeWechatModal } from "./wechat.js"; import { closeChangelogModal, markVersion } from "./changelog.js"; @@ -110,6 +111,7 @@ document.addEventListener("keydown", (e) => { if ($("chpw-modal").classList.contains("show")) { closeChpwModal(); return; } if ($("skills-modal").classList.contains("show")) { closeSkillsModal(); return; } if ($("memory-modal").classList.contains("show")) { closeMemoryModal(); return; } + if ($("kb-modal").classList.contains("show")) { closeKbModal(); return; } if ($("crons-modal").classList.contains("show")) { closeCronsModal(); return; } if ($("wechat-modal").classList.contains("show")) { closeWechatModal(); return; } if ($("changelog-modal").classList.contains("show")) { closeChangelogModal(); return; }