diff --git a/ai_knowledge_base_v2/apps/backend/app/services/feishu_async_service.py b/ai_knowledge_base_v2/apps/backend/app/services/feishu_async_service.py index cec5612..09f31b3 100644 --- a/ai_knowledge_base_v2/apps/backend/app/services/feishu_async_service.py +++ b/ai_knowledge_base_v2/apps/backend/app/services/feishu_async_service.py @@ -365,8 +365,19 @@ async def _feishu_download_async(path: str, config: FeishuRetrievalConfig) -> by async def _get_docx_content_async(doc_token: str, config: FeishuRetrievalConfig) -> str: - data = await _feishu_get_async(f"/open-apis/docx/v1/documents/{doc_token}/blocks", config=config) - items = data.get("data", {}).get("items", []) + path = f"/open-apis/docx/v1/documents/{doc_token}/blocks" + items: list[dict] = [] + page_token = "" + while True: + params: dict[str, Any] = {"page_size": 500} + if page_token: + params["page_token"] = page_token + data = await _feishu_get_async(path, params=params, config=config) + page = data.get("data", {}) + items.extend(page.get("items", [])) + page_token = str(page.get("page_token") or "") + if not page.get("has_more") or not page_token: + break return _parse_docx_content(items) diff --git a/ai_knowledge_base_v2/apps/backend/app/services/feishu_service.py b/ai_knowledge_base_v2/apps/backend/app/services/feishu_service.py index 075d0bd..364afb7 100644 --- a/ai_knowledge_base_v2/apps/backend/app/services/feishu_service.py +++ b/ai_knowledge_base_v2/apps/backend/app/services/feishu_service.py @@ -234,8 +234,19 @@ def _parse_docx_content(blocks: list[dict]) -> str: def _get_docx_content(doc_token: str, config: FeishuRetrievalConfig) -> str: """读取单个飞书云文档内容""" - data = _feishu_get(f"/open-apis/docx/v1/documents/{doc_token}/blocks", config=config) - items = data.get("data", {}).get("items", []) + path = f"/open-apis/docx/v1/documents/{doc_token}/blocks" + items: list[dict] = [] + page_token = "" + while True: + params: dict[str, Any] = {"page_size": 500} + if page_token: + params["page_token"] = page_token + data = _feishu_get(path, params=params, config=config) + page = data.get("data", {}) + items.extend(page.get("items", [])) + page_token = str(page.get("page_token") or "") + if not page.get("has_more") or not page_token: + break return _parse_docx_content(items) diff --git a/ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py b/ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py index 4ca3489..400422f 100644 --- a/ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py +++ b/ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py @@ -32,6 +32,7 @@ from app.services.rag_service import PromptService, RagResult, RetrievedChunk SAFETY_RULE_VERSION = "minimum-safety-v1" MAX_LEXICAL_CANDIDATES = 12 MAX_SELECTED_SECTIONS = 4 +MAX_OVERVIEW_SELECTED_SECTIONS = 8 BUSINESS_MARKERS = {"课程", "大本营", "训练营", "老师", "卢慧", "功课", "学员", "课堂", "练习", "觉察", "内在"} @@ -107,13 +108,22 @@ class KnowledgeAgentService: candidates = cls.search_knowledge(db, terms, selected_ids, catalog) trace.append(cls._trace("search_knowledge", len(trace) + 1, {"queryTerms": terms, "knowledgeIds": selected_ids}, {"candidateCount": len(candidates), "candidates": [cls._candidate_trace(x) for x in candidates]}, started)) await cls._rerank(db, retrieval_question, candidates, trace, started) - selected = cls._select_sections(candidates) + practice_overview = cls._is_practice_overview(retrieval_question) + selected = cls._select_sections( + candidates, + limit=cls._selection_limit(retrieval_question), + prefer_numbered_practice=practice_overview, + ) + selected_contents = { + item.section.id: cls._protect_content(cls._read_complete_section(db, item.section)) + for item in selected + } chunks = [ RetrievedChunk( knowledge_id=item.knowledge.id, knowledge_name=item.knowledge.name, title=item.section.title, - content=cls._protect_content(item.section.content), + content=selected_contents[item.section.id], version_id=item.version.id, section_id=item.section.id, chunk_id=item.chunk.id, @@ -121,7 +131,7 @@ class KnowledgeAgentService: ) for item in selected ] - trace.append(cls._trace("read_knowledge_sections", len(trace) + 1, {"sectionIds": [item.section.id for item in selected]}, {"count": len(selected), "sections": [{"knowledgeId": x.knowledge.id, "knowledgeName": x.knowledge.name, "sectionId": x.section.id, "title": x.section.title, "content": cls._protect_content(x.section.content), "adopted": True} for x in selected]}, started)) + trace.append(cls._trace("read_knowledge_sections", len(trace) + 1, {"sectionIds": [item.section.id for item in selected]}, {"count": len(selected), "sections": [{"knowledgeId": x.knowledge.id, "knowledgeName": x.knowledge.name, "sectionId": x.section.id, "title": x.section.title, "content": selected_contents[x.section.id], "adopted": True} for x in selected]}, started)) cls._persist_candidates(db, log.id, candidates) log.knowledge_called = 1 log.selected_knowledge_ids = ",".join(str(item) for item in selected_ids) @@ -328,13 +338,14 @@ class KnowledgeAgentService: section, knowledge, version = sections.get(chunk.section_id), knowledge_rows.get(chunk.knowledge_id), versions.get(chunk.version_id) if not section or not knowledge or not version: continue - haystack = f"{chunk.title} {chunk.normalized_text} {chunk.content}".lower() + haystack = f"{knowledge.name} {chunk.title} {chunk.normalized_text} {chunk.content}".lower() score = 0.0 for term in terms: count = haystack.count(term) if count: idf = math.log((len(chunks) + 1) / (frequencies.get(term, 0) + 1)) + 1 score += (count / (count + 1.2)) * idf * (2.4 if term in chunk.title.lower() else 1.0) + score += cls._title_intent_boost(chunk.title, terms) if score > 0: candidates.append(Candidate(chunk, section, knowledge, version, score)) candidates.sort(key=lambda item: (item.lexical_score, item.version.published_at or item.version.created_at), reverse=True) @@ -367,7 +378,21 @@ class KnowledgeAgentService: trace.append(cls._trace("Rerank", len(trace) + 1, {"candidateCount": len(candidates)}, {"count": len(candidates), "mode": "lexical_fallback"}, started, status="fallback", error=str(exc)[:300])) @staticmethod - def _select_sections(candidates: list[Candidate]) -> list[Candidate]: + def _select_sections( + candidates: list[Candidate], + *, + limit: int = MAX_SELECTED_SECTIONS, + prefer_numbered_practice: bool = False, + ) -> list[Candidate]: + if prefer_numbered_practice: + candidates = sorted( + candidates, + key=lambda item: ( + KnowledgeAgentService._is_numbered_practice_title(item.chunk.title), + item.rerank_score if item.rerank_score is not None else item.lexical_score, + ), + reverse=True, + ) selected: list[Candidate] = [] seen: set[int] = set() for item in candidates: @@ -376,7 +401,7 @@ class KnowledgeAgentService: item.discard_reason = "相关性不足" elif item.section.id in seen: item.discard_reason = "同一父章节已有更高分候选" - elif len(selected) >= MAX_SELECTED_SECTIONS: + elif len(selected) >= limit: item.discard_reason = "超过本轮章节数量限制" else: item.selected = True @@ -384,6 +409,54 @@ class KnowledgeAgentService: seen.add(item.section.id) return selected + @staticmethod + def _selection_limit(question: str) -> int: + if KnowledgeAgentService._is_practice_overview(question): + return MAX_OVERVIEW_SELECTED_SECTIONS + return MAX_SELECTED_SECTIONS + + @staticmethod + def _is_practice_overview(question: str) -> bool: + overview_markers = ("有哪些", "是什么", "包括什么", "都有什么", "列出", "汇总", "总结") + practice_markers = ("作业", "功课", "练习") + return any(marker in question for marker in practice_markers) and any(marker in question for marker in overview_markers) + + @staticmethod + def _is_numbered_practice_title(title: str) -> bool: + return bool(re.search(r"(?:练习|作业|功课)\s*[一二三四五六七八九十百\d]+", title)) + + @staticmethod + def _title_intent_boost(title: str, terms: list[str]) -> float: + if not {"作业", "功课", "练习"}.intersection(terms): + return 0.0 + if KnowledgeAgentService._is_numbered_practice_title(title): + return 20.0 + if any(marker in title for marker in ("练习", "作业", "功课")): + return 3.0 + return 0.0 + + @staticmethod + def _read_complete_section(db: Session, section: KnowledgeSection) -> str: + """Read a heading together with its lower-level child sections.""" + level = _heading_level(section.content) + if level is None: + return section.content + rows = db.scalars( + select(KnowledgeSection) + .where( + KnowledgeSection.version_id == section.version_id, + KnowledgeSection.sort_order >= section.sort_order, + ) + .order_by(KnowledgeSection.sort_order) + ).all() + parts: list[str] = [] + for row in rows: + row_level = _heading_level(row.content) + if row.id != section.id and row_level is not None and row_level <= level: + break + parts.append(row.content) + return "\n\n".join(parts) + @staticmethod def _persist_candidates(db: Session, log_id: int, candidates: list[Candidate]) -> None: for item in candidates: @@ -443,6 +516,11 @@ def _json_trace_value(value): return str(value) +def _heading_level(content: str) -> int | None: + match = re.match(r"^(#{1,6})\s+", content.lstrip()) + return len(match.group(1)) if match else None + + def _extract_json(value: str) -> str: match = re.search(r"\{[\s\S]*\}", value) if not match: diff --git a/ai_knowledge_base_v2/apps/backend/app/services/knowledge_pipeline_service.py b/ai_knowledge_base_v2/apps/backend/app/services/knowledge_pipeline_service.py index 18df29f..b639449 100644 --- a/ai_knowledge_base_v2/apps/backend/app/services/knowledge_pipeline_service.py +++ b/ai_knowledge_base_v2/apps/backend/app/services/knowledge_pipeline_service.py @@ -455,6 +455,9 @@ _SYNONYMS = { "焦虑": ["担心", "紧张", "害怕"], "沟通": ["交流", "对话", "聊天"], "课程": ["训练营", "大本营", "课堂"], + "作业": ["功课", "练习"], + "功课": ["作业", "练习"], + "练习": ["作业", "功课"], } diff --git a/ai_knowledge_base_v2/apps/backend/app/services/rag_service.py b/ai_knowledge_base_v2/apps/backend/app/services/rag_service.py index c734a5c..9a025c3 100644 --- a/ai_knowledge_base_v2/apps/backend/app/services/rag_service.py +++ b/ai_knowledge_base_v2/apps/backend/app/services/rag_service.py @@ -121,7 +121,7 @@ class PromptService: # 知识库上下文 context = "\n\n".join( - f"[已回读完整章节 {index}] {chunk.title}\n{chunk.content}" + f"[已回读完整章节 {index}] {chunk.title}\n来源知识库：{chunk.knowledge_name}\n{chunk.content}" for index, chunk in enumerate(chunks, start=1) ) if not context: @@ -152,7 +152,21 @@ class PromptService: if content and message.role in {"user", "assistant"}: messages.append({"role": message.role, "content": content}) - messages.append({"role": "system", "content": f"[本轮可靠知识上下文]\n{context}"}) + messages.append({ + "role": "system", + "content": ( + "[本轮可靠知识上下文]\n" + "以下章节均来自当前正式开放、已发布的知识库，可以作为本轮回答的可靠依据。" + "如果其中已经明确包含用户询问的课程、作业、功课或练习，不得再以‘没有资料’、" + "‘无法确认’或要求用户补充课程名称来回避回答；应直接根据章节标题和正文归纳。" + "用户问‘某课程的作业是什么’时，含义是询问该课程包含哪些作业或练习，" + "不要求知识库中必须存在一个与‘某课程作业’完全同名的章节。" + "回答作业清单类问题时，应直接给出作业名称，并概括每项作业的用途或核心方向；" + "正文已经提供基本步骤时可以简要说明，不要把本轮已经取到的内容留到下一轮再问。" + "章节标题可以作为作业或练习名称，标题下的子章节是对应的正式说明。\n\n" + f"{context}" + ), + }) messages.append({"role": "user", "content": question.strip()}) return messages diff --git a/ai_knowledge_base_v2/apps/backend/tests/test_feishu_pagination.py b/ai_knowledge_base_v2/apps/backend/tests/test_feishu_pagination.py new file mode 100644 index 0000000..c542bc3 --- /dev/null +++ b/ai_knowledge_base_v2/apps/backend/tests/test_feishu_pagination.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, patch + +from app.services.feishu_async_service import _get_docx_content_async +from app.services.feishu_service import FeishuRetrievalConfig, _get_docx_content + + +def _config() -> FeishuRetrievalConfig: + return FeishuRetrievalConfig( + search_url="", + app_id="app-id", + app_secret="secret", + timeout_seconds=10, + retry_count=1, + ) + + +def _heading(text: str) -> dict: + return { + "block_type": 3, + "heading1": {"elements": [{"text_run": {"content": text}}]}, + } + + +def test_docx_content_reads_all_block_pages(): + pages = [ + {"data": {"items": [_heading("第一页")], "has_more": True, "page_token": "next-page"}}, + {"data": {"items": [_heading("第二页作业")], "has_more": False, "page_token": ""}}, + ] + with patch("app.services.feishu_service._feishu_get", side_effect=pages) as feishu_get: + content = _get_docx_content("doc-token", _config()) + + assert "第一页" in content + assert "第二页作业" in content + assert feishu_get.call_count == 2 + assert feishu_get.call_args_list[0].kwargs["params"] == {"page_size": 500} + assert feishu_get.call_args_list[1].kwargs["params"] == {"page_size": 500, "page_token": "next-page"} + + +def test_async_docx_content_reads_all_block_pages(): + pages = [ + {"data": {"items": [_heading("第一页")], "has_more": True, "page_token": "next-page"}}, + {"data": {"items": [_heading("第二页作业")], "has_more": False, "page_token": ""}}, + ] + feishu_get = AsyncMock(side_effect=pages) + with patch("app.services.feishu_async_service._feishu_get_async", feishu_get): + content = asyncio.run(_get_docx_content_async("doc-token", _config())) + + assert "第一页" in content + assert "第二页作业" in content + assert feishu_get.await_count == 2 + assert feishu_get.await_args_list[1].kwargs["params"] == {"page_size": 500, "page_token": "next-page"} diff --git a/ai_knowledge_base_v2/apps/backend/tests/test_knowledge_agent.py b/ai_knowledge_base_v2/apps/backend/tests/test_knowledge_agent.py index e6ed151..50f1076 100644 --- a/ai_knowledge_base_v2/apps/backend/tests/test_knowledge_agent.py +++ b/ai_knowledge_base_v2/apps/backend/tests/test_knowledge_agent.py @@ -4,7 +4,7 @@ import asyncio import json from datetime import datetime -from sqlalchemy import create_engine +from sqlalchemy import create_engine, select from sqlalchemy.orm import Session from sqlalchemy.pool import StaticPool @@ -141,6 +141,13 @@ def test_course_question_searches_chunk_and_reads_parent_section(): assert len(result.chunks) == 1 assert "先稳定自己的焦虑" in result.chunks[0].content assert any(item["tool"] == "read_knowledge_sections" for item in result.tool_trace) + knowledge_context = next( + message["content"] for message in result.messages if message["content"].startswith("[本轮可靠知识上下文]") + ) + assert "不得再以‘没有资料’" in knowledge_context + assert "不要求知识库中必须存在一个" in knowledge_context + assert "不要把本轮已经取到的内容留到下一轮再问" in knowledge_context + assert "来源知识库：亲子课程" in knowledge_context def test_tool_trace_serializes_published_datetime(): @@ -173,3 +180,57 @@ def test_contextual_follow_up_is_rewritten_before_agent_decision(): assert rewrite["rewrittenQuestion"] == "关于“课程退款条件有哪些？”的追问：那第二种情况呢？" assert decision["request"]["question"] == rewrite["rewrittenQuestion"] + + +def test_homework_overview_expands_practice_terms_and_section_limit(): + terms = KnowledgeAgentService._query_terms("合一的作业是什么？") + + assert "功课" in terms + assert "练习" in terms + assert KnowledgeAgentService._selection_limit("合一的作业是什么？") == 8 + assert KnowledgeAgentService._selection_limit("这个练习怎么做？") == 4 + assert KnowledgeAgentService._title_intent_boost("十七、练习一：风铃式静心", terms) == 20.0 + assert KnowledgeAgentService._title_intent_boost("完整练习的方向", terms) == 3.0 + assert KnowledgeAgentService._title_intent_boost("课程定位", terms) == 0.0 + + +def test_complete_section_includes_child_headings_but_stops_at_next_peer(): + with _database() as db: + knowledge = _add_published_knowledge(db, knowledge_id=1, name="合一课程") + version_id = knowledge.current_version_id + parent = db.scalar(select(KnowledgeSection).where(KnowledgeSection.version_id == version_id)) + assert parent is not None + parent.title = "练习一：风铃式静心" + parent.content = "# 练习一：风铃式静心" + parent.sort_order = 1 + db.add_all([ + parent, + KnowledgeSection( + knowledge_id=knowledge.id, + version_id=version_id, + section_key="S0002", + title="基本操作", + content="## 基本操作\n鼻吸鼻呼，呼气时发出连续蜂鸣声。", + source_start=10, + source_end=40, + sort_order=2, + content_hash="child", + ), + KnowledgeSection( + knowledge_id=knowledge.id, + version_id=version_id, + section_key="S0003", + title="练习二：内感知建模", + content="# 练习二：内感知建模\n这是下一项作业。", + source_start=41, + source_end=70, + sort_order=3, + content_hash="next", + ), + ]) + db.commit() + + content = KnowledgeAgentService._read_complete_section(db, parent) + + assert "鼻吸鼻呼" in content + assert "下一项作业" not in content