diff --git a/ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py b/ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py index 22522df..933616d 100644 --- a/ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py +++ b/ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py @@ -23,7 +23,13 @@ from app.models.knowledge import ( KnowledgeSection, KnowledgeVersion, ) -from app.services.knowledge_pipeline_service import expand_synonyms, extract_terms +from app.services.knowledge_pipeline_service import ( + expand_synonyms, + extract_terms, + is_numbered_heading_title, + is_practice_anchor_title, + practice_titles_related, +) from app.services.knowledge_catalog_cache_service import KnowledgeCatalogCacheService from app.services.knowledge_service import KnowledgeScope from app.services.model_service import _call_configured_model, _system_config_bool @@ -125,7 +131,10 @@ class KnowledgeAgentService: if practice_overview: selected.sort(key=lambda item: (item.knowledge.name, item.section.sort_order)) selected_contents = { - item.section.id: cls._protect_content(cls._read_complete_section(db, item.section)) + item.section.id: cls._protect_content( + cls._read_complete_section(db, item.section), + max_chars=900 if practice_overview else 6000, + ) for item in selected } chunks = [ @@ -450,6 +459,10 @@ class KnowledgeAgentService: @staticmethod def _is_practice_overview(question: str) -> bool: + if "追问：" in question: + follow_up = question.rsplit("追问：", 1)[-1] + if any(marker in follow_up for marker in ("具体", "详细", "怎么做", "步骤", "方法", "注意", "内容")): + return False overview_markers = ("有哪些", "是什么", "包括什么", "都有什么", "列出", "汇总", "总结") practice_markers = ("作业", "功课", "练习") return any(marker in question for marker in practice_markers) and any(marker in question for marker in overview_markers) @@ -471,19 +484,36 @@ class KnowledgeAgentService: @staticmethod def _read_complete_section(db: Session, section: KnowledgeSection) -> str: """Read a heading together with its lower-level child sections.""" - level = _heading_level(section.content) - if level is None: - return section.content rows = db.scalars( select(KnowledgeSection) .where( KnowledgeSection.version_id == section.version_id, - KnowledgeSection.sort_order >= section.sort_order, ) .order_by(KnowledgeSection.sort_order) ).all() + selected_index = next((index for index, row in enumerate(rows) if row.id == section.id), None) + if selected_index is None: + return section.content + + for anchor_index, row in enumerate(rows): + if not is_practice_anchor_title(row.title): + continue + end_index = anchor_index + 1 + while end_index < len(rows): + next_title = rows[end_index].title + if is_practice_anchor_title(next_title): + break + if is_numbered_heading_title(next_title) and not practice_titles_related(row.title, next_title): + break + end_index += 1 + if anchor_index <= selected_index < end_index: + return "\n\n".join(item.content for item in rows[anchor_index:end_index]) + + level = _heading_level(section.content) + if level is None: + return section.content parts: list[str] = [] - for row in rows: + for row in rows[selected_index:]: row_level = _heading_level(row.content) if row.id != section.id and row_level is not None and row_level <= level: break @@ -533,8 +563,8 @@ class KnowledgeAgentService: return list(dict.fromkeys(base + expand_synonyms(base)))[:40] @staticmethod - def _protect_content(content: str) -> str: - return content if len(content) <= 3200 else content[:3200].rstrip() + "\n[章节内容已按保护规则截断]" + def _protect_content(content: str, *, max_chars: int = 6000) -> str: + return content if len(content) <= max_chars else content[:max_chars].rstrip() + "\n[章节内容已按保护规则截断]" def _dump_trace(trace: list[dict]) -> str: diff --git a/ai_knowledge_base_v2/apps/backend/app/services/knowledge_pipeline_service.py b/ai_knowledge_base_v2/apps/backend/app/services/knowledge_pipeline_service.py index 655c2f3..d6ecbba 100644 --- a/ai_knowledge_base_v2/apps/backend/app/services/knowledge_pipeline_service.py +++ b/ai_knowledge_base_v2/apps/backend/app/services/knowledge_pipeline_service.py @@ -25,10 +25,15 @@ from app.services.external_errors import ExternalServiceError from app.services.feishu_async_service import AsyncFeishuKnowledgeService from app.services.feishu_service import _feishu_retrieval_config -PROCESSING_RULE_VERSION = "knowledge-pipeline-v3-latest-only" -MAX_SECTION_CHARS = 2800 +PROCESSING_RULE_VERSION = "knowledge-pipeline-v4-semantic-practice" +MAX_SECTION_CHARS = 8000 MAX_CHUNK_CHARS = 650 +_PRACTICE_ANCHOR = re.compile( + r"(?:^|[、\s])(?:练习|作业|功课)\s*[一二三四五六七八九十百零〇\d]+(?:[：:、.．\s]|$)" +) +_NUMBERED_HEADING = re.compile(r"^(?:第)?[一二三四五六七八九十百零〇\d]+[、.．:：]\s*") + @dataclass(frozen=True) class ParsedSection: @@ -388,9 +393,25 @@ def parse_sections(content: str) -> list[ParsedSection]: boundaries.append((match.start(), raw)) if not boundaries or boundaries[0][0] > 0: boundaries.insert(0, (0, "正文")) + spans: list[tuple[int, int, str]] = [] + index = 0 + while index < len(boundaries): + start, title = boundaries[index] + next_index = index + 1 + if is_practice_anchor_title(title): + while next_index < len(boundaries): + next_title = boundaries[next_index][1] + if is_practice_anchor_title(next_title): + break + if is_numbered_heading_title(next_title) and not practice_titles_related(title, next_title): + break + next_index += 1 + end = boundaries[next_index][0] if next_index < len(boundaries) else len(content) + spans.append((start, end, title)) + index = next_index + sections: list[ParsedSection] = [] - for index, (start, title) in enumerate(boundaries): - end = boundaries[index + 1][0] if index + 1 < len(boundaries) else len(content) + for start, end, title in spans: block = content[start:end].strip() if not block: continue @@ -408,6 +429,29 @@ def parse_sections(content: str) -> list[ParsedSection]: return sections or [ParsedSection("正文", content, 0, len(content))] +def is_practice_anchor_title(title: str) -> bool: + return bool(_PRACTICE_ANCHOR.search(title.strip())) + + +def is_numbered_heading_title(title: str) -> bool: + return bool(_NUMBERED_HEADING.match(title.strip())) + + +def practice_titles_related(anchor: str, candidate: str) -> bool: + """Recognize numbered safety/boundary headings that still belong to a practice.""" + anchor_topic = _PRACTICE_ANCHOR.sub("", _NUMBERED_HEADING.sub("", anchor)).strip(" ：:、") + candidate_text = _NUMBERED_HEADING.sub("", candidate) + normalized = re.sub(r"[\s·・，。、“”《》：:（）()\-]", "", anchor_topic) + candidate_normalized = re.sub(r"[\s·・，。、“”《》：:（）()\-]", "", candidate_text) + ignored = {"练习", "作业", "功课", "静心", "方法", "步骤", "课程"} + pairs = { + normalized[index : index + 2] + for index in range(max(0, len(normalized) - 1)) + if normalized[index : index + 2] not in ignored + } + return any(pair in candidate_normalized for pair in pairs) + + def split_chunks(section: ParsedSection) -> list[tuple[str, int, int]]: chunks: list[tuple[str, int, int]] = [] cursor = 0 diff --git a/ai_knowledge_base_v2/apps/backend/tests/test_knowledge_agent.py b/ai_knowledge_base_v2/apps/backend/tests/test_knowledge_agent.py index 9791df6..b7335fe 100644 --- a/ai_knowledge_base_v2/apps/backend/tests/test_knowledge_agent.py +++ b/ai_knowledge_base_v2/apps/backend/tests/test_knowledge_agent.py @@ -21,6 +21,7 @@ from app.models.knowledge import ( ) from app.models.chat import ChatMessage from app.services.knowledge_agent_service import Candidate, KnowledgeAgentService +from app.services.knowledge_pipeline_service import parse_sections def _database() -> Session: @@ -192,6 +193,10 @@ def test_homework_overview_expands_practice_terms_and_section_limit(): assert KnowledgeAgentService._title_intent_boost("十七、练习一：风铃式静心", terms) == 20.0 assert KnowledgeAgentService._title_intent_boost("完整练习的方向", terms) == 3.0 assert KnowledgeAgentService._title_intent_boost("课程定位", terms) == 0.0 + assert KnowledgeAgentService._is_practice_overview("原生里的作业内容都有什么") is True + assert KnowledgeAgentService._is_practice_overview( + "关于“心光里都有什么功课”的追问：金色光练习具体内容是什么" + ) is False def test_homework_overview_keeps_all_numbered_practices_before_selection(): @@ -264,3 +269,110 @@ def test_complete_section_includes_child_headings_but_stops_at_next_peer(): assert "鼻吸鼻呼" in content assert "下一项作业" not in content + + +def test_same_level_practice_headings_are_read_as_one_semantic_unit(): + with _database() as db: + knowledge = _add_published_knowledge(db, knowledge_id=1, name="心光课程") + version_id = knowledge.current_version_id + parent = db.scalar(select(KnowledgeSection).where(KnowledgeSection.version_id == version_id)) + assert parent is not None + parent.title = "二十九、练习一：金色光欧姆静心" + parent.content = "# 二十九、练习一：金色光欧姆静心" + parent.sort_order = 1 + parent_chunk = db.scalar(select(KnowledgeChunk).where(KnowledgeChunk.section_id == parent.id)) + assert parent_chunk is not None + parent_chunk.title = parent.title + parent_chunk.content = parent.content + parent_chunk.normalized_text = "金色光 欧姆 静心 练习" + detail = KnowledgeSection( + knowledge_id=knowledge.id, + version_id=version_id, + section_key="S0002", + title="基础原则", + content="# 基础原则\n练习时保持自然呼吸，不追求必须看见金色光。", + source_start=10, + source_end=50, + sort_order=2, + content_hash="detail", + ) + db.add_all([ + parent, + parent_chunk, + detail, + KnowledgeSection( + knowledge_id=knowledge.id, + version_id=version_id, + section_key="S0003", + title="三十、金色光练习的现实边界", + content="# 三十、金色光练习的现实边界\n不能替代医疗诊断。", + source_start=51, + source_end=80, + sort_order=3, + content_hash="boundary", + ), + KnowledgeSection( + knowledge_id=knowledge.id, + version_id=version_id, + section_key="S0004", + title="三十一、练习二：纠缠之心静心", + content="# 三十一、练习二：纠缠之心静心\n下一项练习内容。", + source_start=81, + source_end=120, + sort_order=4, + content_hash="next", + ), + ]) + db.flush() + db.add( + KnowledgeChunk( + knowledge_id=knowledge.id, + version_id=version_id, + section_id=detail.id, + title=detail.title, + content=detail.content, + normalized_text="金色光 基础原则 自然呼吸 画面", + keywords='["金色光","自然呼吸"]', + synonyms="[]", + source_start=detail.source_start, + source_end=detail.source_end, + sort_order=1, + content_hash="detail-chunk", + ) + ) + db.commit() + + content = KnowledgeAgentService._read_complete_section(db, detail) + result = asyncio.run( + KnowledgeAgentService.build_result(db, question="金色光欧姆静心具体怎么练习？") + ) + + assert "金色光欧姆静心" in content + assert "保持自然呼吸" in content + assert "不能替代医疗诊断" in content + assert "下一项练习内容" not in content + assert result.chunks + assert "保持自然呼吸" in result.chunks[0].content + + +def test_parser_merges_practice_title_and_same_level_detail_headings(): + source = """# 二十九、练习一：金色光欧姆静心 +# 练习定位 +帮助学员稳定注意力。 +# 基础原则 +保持自然呼吸，不追求画面。 +# 三十、金色光练习的现实边界 +不能替代医疗诊断。 +# 三十一、练习二：纠缠之心静心 +这是下一项练习。 +""" + + sections = parse_sections(source) + + assert [section.title for section in sections] == [ + "二十九、练习一：金色光欧姆静心", + "三十一、练习二：纠缠之心静心", + ] + assert "基础原则" in sections[0].content + assert "现实边界" in sections[0].content + assert "下一项练习" not in sections[0].content