fix(agent): enforce complete homework answers
This commit is contained in:
@@ -122,6 +122,8 @@ class KnowledgeAgentService:
|
|||||||
limit=cls._selection_limit(retrieval_question),
|
limit=cls._selection_limit(retrieval_question),
|
||||||
prefer_numbered_practice=practice_overview,
|
prefer_numbered_practice=practice_overview,
|
||||||
)
|
)
|
||||||
|
if practice_overview:
|
||||||
|
selected.sort(key=lambda item: (item.knowledge.name, item.section.sort_order))
|
||||||
selected_contents = {
|
selected_contents = {
|
||||||
item.section.id: cls._protect_content(cls._read_complete_section(db, item.section))
|
item.section.id: cls._protect_content(cls._read_complete_section(db, item.section))
|
||||||
for item in selected
|
for item in selected
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ class ModelClientService:
|
|||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"message": "Agent 调试完成",
|
"message": "Agent 调试完成",
|
||||||
"answer": answer[:4000],
|
"answer": answer,
|
||||||
"prompt": rag_result.prompt,
|
"prompt": rag_result.prompt,
|
||||||
"modelName": model.model_name,
|
"modelName": model.model_name,
|
||||||
"retrieveCount": len(rag_result.chunks),
|
"retrieveCount": len(rag_result.chunks),
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
@@ -127,6 +128,8 @@ class PromptService:
|
|||||||
if not context:
|
if not context:
|
||||||
context = "本轮没有可靠的正式知识章节。对于一般常识可以谨慎回答;涉及课程、老师观点或公司业务时必须说明依据不足,不得编造。"
|
context = "本轮没有可靠的正式知识章节。对于一般常识可以谨慎回答;涉及课程、老师观点或公司业务时必须说明依据不足,不得编造。"
|
||||||
|
|
||||||
|
completeness_constraint = cls._list_completeness_constraint(question, chunks)
|
||||||
|
|
||||||
messages: list[dict[str, str]] = [
|
messages: list[dict[str, str]] = [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -163,13 +166,38 @@ class PromptService:
|
|||||||
"不要求知识库中必须存在一个与‘某课程作业’完全同名的章节。"
|
"不要求知识库中必须存在一个与‘某课程作业’完全同名的章节。"
|
||||||
"回答作业清单类问题时,应直接给出作业名称,并概括每项作业的用途或核心方向;"
|
"回答作业清单类问题时,应直接给出作业名称,并概括每项作业的用途或核心方向;"
|
||||||
"正文已经提供基本步骤时可以简要说明,不要把本轮已经取到的内容留到下一轮再问。"
|
"正文已经提供基本步骤时可以简要说明,不要把本轮已经取到的内容留到下一轮再问。"
|
||||||
"章节标题可以作为作业或练习名称,标题下的子章节是对应的正式说明。\n\n"
|
"章节标题可以作为作业或练习名称,标题下的子章节是对应的正式说明。"
|
||||||
|
f"{completeness_constraint}\n\n"
|
||||||
f"{context}"
|
f"{context}"
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
messages.append({"role": "user", "content": question.strip()})
|
messages.append({"role": "user", "content": question.strip()})
|
||||||
return messages
|
return messages
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _list_completeness_constraint(question: str, chunks: list[RetrievedChunk]) -> str:
|
||||||
|
is_practice_list = (
|
||||||
|
any(marker in question for marker in ("作业", "功课", "练习"))
|
||||||
|
and any(marker in question for marker in ("有哪些", "是什么", "包括什么", "都有什么", "列出", "汇总", "总结"))
|
||||||
|
)
|
||||||
|
numbered = [
|
||||||
|
chunk for chunk in chunks
|
||||||
|
if re.search(r"(?:练习|作业|功课)\s*[一二三四五六七八九十百\d]+", chunk.title)
|
||||||
|
]
|
||||||
|
if not is_practice_list or not numbered:
|
||||||
|
return ""
|
||||||
|
titles = "\n".join(
|
||||||
|
f"{index}. {chunk.knowledge_name} / {chunk.title}"
|
||||||
|
for index, chunk in enumerate(numbered, start=1)
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
"\n\n[作业清单完整性约束]\n"
|
||||||
|
f"本轮共召回 {len(numbered)} 个正式编号练习。回答必须覆盖下面全部 {len(numbered)} 项,"
|
||||||
|
"不得只挑选部分项目,不得合并成少数概念,不得新增清单外的作业。"
|
||||||
|
"应按知识库分组,逐项保留正式练习名称,并根据对应正文简要概括;即使回答较长也必须列完。\n"
|
||||||
|
f"必须覆盖的标题清单:\n{titles}"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def render_messages(messages: list[dict[str, str]]) -> str:
|
def render_messages(messages: list[dict[str, str]]) -> str:
|
||||||
role_labels = {"system": "系统", "user": "用户", "assistant": "大本营答疑助手"}
|
role_labels = {"system": "系统", "user": "用户", "assistant": "大本营答疑助手"}
|
||||||
@@ -180,8 +208,6 @@ class PromptService:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _clean_history_content(content: str) -> str:
|
def _clean_history_content(content: str) -> str:
|
||||||
import re
|
|
||||||
|
|
||||||
return re.sub(r"<think[\s\S]*?</think\s*>", "", content, flags=re.IGNORECASE).strip()
|
return re.sub(r"<think[\s\S]*?</think\s*>", "", content, flags=re.IGNORECASE).strip()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from app.models.admin import Admin
|
|||||||
from app.models.ai_config import ModelConfig
|
from app.models.ai_config import ModelConfig
|
||||||
from app.schemas.admin import AgentRuntimeConfigSaveRequest
|
from app.schemas.admin import AgentRuntimeConfigSaveRequest
|
||||||
from app.services.model_stream_service import _openai_stream_payload
|
from app.services.model_stream_service import _openai_stream_payload
|
||||||
from app.services.model_service import _max_output_tokens
|
from app.services.model_service import ModelClientService, _max_output_tokens
|
||||||
from app.services.rag_service import RagResult
|
from app.services.rag_service import RagResult
|
||||||
|
|
||||||
|
|
||||||
@@ -99,3 +99,18 @@ def test_saved_runtime_config_is_persisted_on_enabled_model():
|
|||||||
assert payload["top_p"] == 0.9
|
assert payload["top_p"] == 0.9
|
||||||
assert payload["presence_penalty"] == 0.2
|
assert payload["presence_penalty"] == 0.2
|
||||||
assert payload["frequency_penalty"] == 0.4
|
assert payload["frequency_penalty"] == 0.4
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_debug_does_not_truncate_long_answer(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.model_service._call_configured_model",
|
||||||
|
lambda *_args, **_kwargs: "长" * 6000,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = ModelClientService.debug_model(
|
||||||
|
_model(),
|
||||||
|
RagResult(question="测试", knowledge_scopes=[], chunks=[], prompt="测试", allow_general_knowledge=True),
|
||||||
|
{"max_token": 8192},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(result["answer"]) == 6000
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from app.models.knowledge import (
|
|||||||
)
|
)
|
||||||
from app.models.chat import ChatMessage
|
from app.models.chat import ChatMessage
|
||||||
from app.services.knowledge_agent_service import Candidate, KnowledgeAgentService
|
from app.services.knowledge_agent_service import Candidate, KnowledgeAgentService
|
||||||
|
from app.services.rag_service import PromptService, RetrievedChunk
|
||||||
|
|
||||||
|
|
||||||
def _database() -> Session:
|
def _database() -> Session:
|
||||||
@@ -227,6 +228,26 @@ def test_homework_overview_keeps_all_numbered_practices_before_selection():
|
|||||||
assert all(item.chunk.title.startswith("练习") for item in selected)
|
assert all(item.chunk.title.startswith("练习") for item in selected)
|
||||||
|
|
||||||
|
|
||||||
|
def test_homework_overview_prompt_requires_every_recalled_title():
|
||||||
|
chunks = [
|
||||||
|
RetrievedChunk(
|
||||||
|
knowledge_id=1 if index <= 6 else 2,
|
||||||
|
knowledge_name="《原生·上》" if index <= 6 else "《原生·下》",
|
||||||
|
title=f"练习{index}:作业{index}",
|
||||||
|
content=f"作业{index}的正式说明",
|
||||||
|
)
|
||||||
|
for index in range(1, 15)
|
||||||
|
]
|
||||||
|
|
||||||
|
constraint = PromptService._list_completeness_constraint("原生里的作业都有哪些", chunks)
|
||||||
|
|
||||||
|
assert "本轮共召回 14 个正式编号练习" in constraint
|
||||||
|
assert "必须覆盖下面全部 14 项" in constraint
|
||||||
|
assert "不得新增清单外的作业" in constraint
|
||||||
|
assert "练习1:作业1" in constraint
|
||||||
|
assert "练习14:作业14" in constraint
|
||||||
|
|
||||||
|
|
||||||
def test_complete_section_includes_child_headings_but_stops_at_next_peer():
|
def test_complete_section_includes_child_headings_but_stops_at_next_peer():
|
||||||
with _database() as db:
|
with _database() as db:
|
||||||
knowledge = _add_published_knowledge(db, knowledge_id=1, name="合一课程")
|
knowledge = _add_published_knowledge(db, knowledge_id=1, name="合一课程")
|
||||||
|
|||||||
Reference in New Issue
Block a user