feat: v1.2.0 - 图片去重与管理、微信机器人优化、搜索设置可配置
主要功能: - 图片上传时 OCR 内容去重(3个上传端点统一使用公共函数 _check_ocr_duplicate) - 图片管理 Tab:展示所有图片、手动删除、一键去重 - 搜索结果详情弹窗增加删除按钮(带确认弹窗) - 图片管理卡片点击查看详情(复用 showOcrDetailModal) - 搜索限制和 LLM 批量判断数量可通过网站设置 - MiniMax API 调用添加 reasoning_split=True - 企业微信机器人:WebSocket 长连接、图片搜索、配置化搜索数量 - 版本号升级至 1.2.0
This commit is contained in:
17
app/services/__init__.py
Normal file
17
app/services/__init__.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""
|
||||
业务逻辑服务层包
|
||||
"""
|
||||
|
||||
from app.services.embedding_service import EmbeddingService
|
||||
from app.services.ocr_service import OCRService
|
||||
from app.services.search_service import SearchService
|
||||
from app.services.import_service import ImportService
|
||||
from app.services.page_service import PageService
|
||||
|
||||
__all__ = [
|
||||
"EmbeddingService",
|
||||
"OCRService",
|
||||
"SearchService",
|
||||
"ImportService",
|
||||
"PageService",
|
||||
]
|
||||
328
app/services/embedding_service.py
Normal file
328
app/services/embedding_service.py
Normal file
@@ -0,0 +1,328 @@
|
||||
"""
|
||||
嵌入服务模块
|
||||
支持多种嵌入模型提供商:OpenAI / 智谱 / 阿里云 DashScope / 本地 BGE
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.config import EmbeddingProvider, settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ──────────────────────────── 抽象基类 ────────────────────────────
|
||||
|
||||
class EmbeddingProviderBase(abc.ABC):
|
||||
"""嵌入模型提供商抽象基类"""
|
||||
|
||||
@abc.abstractmethod
|
||||
async def embed_batch(self, texts: List[str]) -> List[List[float]]:
|
||||
"""
|
||||
批量生成文本嵌入向量。
|
||||
|
||||
Args:
|
||||
texts: 待嵌入的文本列表
|
||||
|
||||
Returns:
|
||||
嵌入向量列表,每个向量是 float 的列表
|
||||
|
||||
Raises:
|
||||
Exception: 嵌入生成失败时抛出异常
|
||||
"""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
async def embed_single(self, text: str) -> List[float]:
|
||||
"""生成单条文本的嵌入向量"""
|
||||
...
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def dimension(self) -> int:
|
||||
"""嵌入向量维度"""
|
||||
...
|
||||
|
||||
|
||||
# ──────────────────────────── OpenAI 实现 ────────────────────────────
|
||||
|
||||
class OpenAIEmbedding(EmbeddingProviderBase):
|
||||
"""OpenAI 嵌入模型(兼容 OpenAI API 格式的服务)"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=settings.OPENAI_API_KEY or "EMPTY",
|
||||
base_url=settings.OPENAI_BASE_URL,
|
||||
)
|
||||
self._model = settings.EMBEDDING_MODEL or "text-embedding-3-small"
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
return settings.EMBEDDING_DIMENSIONS
|
||||
|
||||
async def embed_batch(self, texts: List[str]) -> List[List[float]]:
|
||||
"""调用 OpenAI 兼容接口批量生成嵌入"""
|
||||
# OpenAI 接口单次最多 2048 条,自动分批
|
||||
batch_size = 2048
|
||||
all_embeddings: List[List[float]] = []
|
||||
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch = texts[i : i + batch_size]
|
||||
response = await self._client.embeddings.create(
|
||||
input=batch,
|
||||
model=self._model,
|
||||
dimensions=self.dimension if self.dimension <= 3072 else None,
|
||||
)
|
||||
batch_embeddings = [item.embedding for item in response.data]
|
||||
all_embeddings.extend(batch_embeddings)
|
||||
|
||||
return all_embeddings
|
||||
|
||||
async def embed_single(self, text: str) -> List[float]:
|
||||
results = await self.embed_batch([text])
|
||||
return results[0]
|
||||
|
||||
|
||||
# ──────────────────────────── 智谱 AI 实现 ────────────────────────────
|
||||
|
||||
class ZhipuEmbedding(EmbeddingProviderBase):
|
||||
"""智谱 AI 嵌入模型"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
from zhipuai import ZhipuAI
|
||||
|
||||
self._client = ZhipuAI(api_key=settings.ZHIPU_API_KEY)
|
||||
self._model = settings.EMBEDDING_MODEL or "embedding-3"
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
return settings.EMBEDDING_DIMENSIONS
|
||||
|
||||
async def embed_batch(self, texts: List[str]) -> List[List[float]]:
|
||||
"""调用智谱 AI 接口批量生成嵌入"""
|
||||
import asyncio
|
||||
|
||||
# 智谱 API 单次最多 16 条
|
||||
batch_size = 16
|
||||
all_embeddings: List[List[float]] = []
|
||||
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch = texts[i : i + batch_size]
|
||||
# 智谱 SDK 是同步的,用线程池包装
|
||||
loop = asyncio.get_event_loop()
|
||||
response = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._client.embeddings.create(
|
||||
input=batch,
|
||||
model=self._model,
|
||||
),
|
||||
)
|
||||
batch_embeddings = [item.embedding for item in response.data]
|
||||
all_embeddings.extend(batch_embeddings)
|
||||
|
||||
return all_embeddings
|
||||
|
||||
async def embed_single(self, text: str) -> List[float]:
|
||||
results = await self.embed_batch([text])
|
||||
return results[0]
|
||||
|
||||
|
||||
# ──────────────────────────── 阿里云 DashScope 实现 ────────────────────────────
|
||||
|
||||
class DashscopeEmbedding(EmbeddingProviderBase):
|
||||
"""阿里云 DashScope 嵌入模型"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
import dashscope
|
||||
dashscope.api_key = settings.DASHSCOPE_API_KEY
|
||||
self._model = settings.EMBEDDING_MODEL or "text-embedding-v3"
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
return settings.EMBEDDING_DIMENSIONS
|
||||
|
||||
async def embed_batch(self, texts: List[str]) -> List[List[float]]:
|
||||
"""调用 DashScope 接口批量生成嵌入"""
|
||||
import asyncio
|
||||
from dashscope import TextEmbedding
|
||||
|
||||
batch_size = 25 # DashScope 单次最多 25 条
|
||||
all_embeddings: List[List[float]] = []
|
||||
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch = texts[i : i + batch_size]
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _call() -> list:
|
||||
resp = TextEmbedding.call(
|
||||
model=self._model,
|
||||
input=batch,
|
||||
dimension=self.dimension,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"DashScope 嵌入失败: {resp.code} - {resp.message}")
|
||||
return [item["embedding"] for item in resp.output["embeddings"]]
|
||||
|
||||
batch_embeddings = await loop.run_in_executor(None, _call)
|
||||
all_embeddings.extend(batch_embeddings)
|
||||
|
||||
return all_embeddings
|
||||
|
||||
async def embed_single(self, text: str) -> List[float]:
|
||||
results = await self.embed_batch([text])
|
||||
return results[0]
|
||||
|
||||
|
||||
# ──────────────────────────── 本地 BGE 实现 ────────────────────────────
|
||||
|
||||
class LocalBGEEmbedding(EmbeddingProviderBase):
|
||||
"""
|
||||
本地 BGE 嵌入模型
|
||||
使用 sentence-transformers 加载模型,在 CPU/GPU 上本地推理
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._model = None
|
||||
self._dim = settings.EMBEDDING_DIMENSIONS
|
||||
|
||||
def _load_model(self):
|
||||
"""延迟加载模型(首次调用时初始化)"""
|
||||
if self._model is not None:
|
||||
return
|
||||
|
||||
logger.info("正在加载本地 BGE 嵌入模型: %s", settings.EMBEDDING_MODEL)
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
model_path = settings.LOCAL_BGE_MODEL_PATH or settings.EMBEDDING_MODEL
|
||||
self._model = SentenceTransformer(
|
||||
model_path,
|
||||
trust_remote_code=True,
|
||||
)
|
||||
# 获取实际维度
|
||||
test_embedding = self._model.encode(["测试"])
|
||||
self._dim = len(test_embedding[0])
|
||||
logger.info("BGE 模型加载完成,维度: %d", self._dim)
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
return self._dim
|
||||
|
||||
async def embed_batch(self, texts: List[str]) -> List[List[float]]:
|
||||
"""使用本地模型批量生成嵌入"""
|
||||
import asyncio
|
||||
|
||||
self._load_model()
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _encode() -> List[List[float]]:
|
||||
embeddings = self._model.encode(texts, normalize_embeddings=True)
|
||||
return embeddings.tolist()
|
||||
|
||||
return await loop.run_in_executor(None, _encode)
|
||||
|
||||
async def embed_single(self, text: str) -> List[float]:
|
||||
results = await self.embed_batch([text])
|
||||
return results[0]
|
||||
|
||||
|
||||
# ──────────────────────────── MiniMax 实现 ────────────────────────────
|
||||
|
||||
class MiniMaxEmbedding(EmbeddingProviderBase):
|
||||
"""
|
||||
MiniMax 嵌入模型
|
||||
通过 MiniMax 的 OpenAI 兼容接口调用嵌入 API
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=settings.MINIMAX_API_KEY or "EMPTY",
|
||||
base_url=settings.MINIMAX_BASE_URL,
|
||||
)
|
||||
self._model = settings.MINIMAX_EMBEDDING_MODEL
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
return settings.EMBEDDING_DIMENSIONS
|
||||
|
||||
async def embed_batch(self, texts: List[str]) -> List[List[float]]:
|
||||
"""调用 MiniMax 兼容接口批量生成嵌入"""
|
||||
batch_size = 64
|
||||
all_embeddings: List[List[float]] = []
|
||||
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch = texts[i : i + batch_size]
|
||||
response = await self._client.embeddings.create(
|
||||
input=batch,
|
||||
model=self._model,
|
||||
)
|
||||
batch_embeddings = [item.embedding for item in response.data]
|
||||
all_embeddings.extend(batch_embeddings)
|
||||
|
||||
return all_embeddings
|
||||
|
||||
async def embed_single(self, text: str) -> List[float]:
|
||||
results = await self.embed_batch([text])
|
||||
return results[0]
|
||||
|
||||
|
||||
# ──────────────────────────── 工厂类 ────────────────────────────
|
||||
|
||||
class EmbeddingService:
|
||||
"""
|
||||
嵌入服务工厂类
|
||||
根据配置自动选择嵌入模型提供商,提供全局单例访问
|
||||
"""
|
||||
|
||||
_instance: Optional[EmbeddingProviderBase] = None
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> EmbeddingProviderBase:
|
||||
"""获取嵌入服务单例实例"""
|
||||
if cls._instance is not None:
|
||||
return cls._instance
|
||||
|
||||
provider_map = {
|
||||
EmbeddingProvider.OPENAI: OpenAIEmbedding,
|
||||
EmbeddingProvider.ZHIPU: ZhipuEmbedding,
|
||||
EmbeddingProvider.DASHSCOPE: DashscopeEmbedding,
|
||||
EmbeddingProvider.LOCAL_BGE: LocalBGEEmbedding,
|
||||
EmbeddingProvider.MINIMAX: MiniMaxEmbedding,
|
||||
}
|
||||
|
||||
provider_cls = provider_map.get(settings.EMBEDDING_PROVIDER)
|
||||
if provider_cls is None:
|
||||
raise ValueError(f"不支持的嵌入模型提供商: {settings.EMBEDDING_PROVIDER}")
|
||||
|
||||
cls._instance = provider_cls()
|
||||
logger.info(
|
||||
"嵌入服务初始化完成: provider=%s, model=%s",
|
||||
settings.EMBEDDING_PROVIDER.value,
|
||||
settings.EMBEDDING_MODEL,
|
||||
)
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
async def embed_batch(cls, texts: List[str]) -> List[List[float]]:
|
||||
"""便捷方法:批量生成嵌入"""
|
||||
instance = cls.get_instance()
|
||||
return await instance.embed_batch(texts)
|
||||
|
||||
@classmethod
|
||||
async def embed_single(cls, text: str) -> List[float]:
|
||||
"""便捷方法:生成单条嵌入"""
|
||||
instance = cls.get_instance()
|
||||
return await instance.embed_single(text)
|
||||
|
||||
@classmethod
|
||||
def reset(cls) -> None:
|
||||
"""重置单例(用于测试或切换配置)"""
|
||||
cls._instance = None
|
||||
452
app/services/import_service.py
Normal file
452
app/services/import_service.py
Normal file
@@ -0,0 +1,452 @@
|
||||
"""
|
||||
导入服务
|
||||
负责从 Markdown / 纯文本 / Word 文件导入知识页面,并进行文本分块和向量化
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.database import IS_SQLITE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _parse_frontmatter(content: str) -> tuple[dict, str]:
|
||||
"""
|
||||
手动解析 YAML frontmatter。
|
||||
格式:--- 开头和结尾包围的 YAML 块。
|
||||
"""
|
||||
import yaml
|
||||
|
||||
if not content.startswith("---"):
|
||||
return {}, content
|
||||
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
return {}, content
|
||||
|
||||
try:
|
||||
metadata = yaml.safe_load(parts[1]) or {}
|
||||
except Exception:
|
||||
metadata = {}
|
||||
|
||||
body = parts[2].strip()
|
||||
return metadata, body
|
||||
|
||||
|
||||
class ImportService:
|
||||
"""知识导入服务"""
|
||||
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
async def import_file(
|
||||
self,
|
||||
file_path: str,
|
||||
course_name: Optional[str] = None,
|
||||
teacher_name: Optional[str] = None,
|
||||
live_date: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
从文件导入知识内容。
|
||||
|
||||
支持的格式:
|
||||
- Markdown (.md): 解析 frontmatter 元数据,按标题分页
|
||||
- 纯文本 (.txt): 整体作为一个页面
|
||||
- Word 文档 (.docx): 提取文本,按标题分页
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
course_name: 课程名称(覆盖 frontmatter)
|
||||
teacher_name: 讲师名称(覆盖 frontmatter)
|
||||
live_date: 直播日期(覆盖 frontmatter)
|
||||
|
||||
Returns:
|
||||
导入结果统计
|
||||
"""
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"文件不存在: {file_path}")
|
||||
|
||||
suffix = path.suffix.lower()
|
||||
|
||||
if suffix == ".md":
|
||||
content = path.read_text(encoding="utf-8")
|
||||
return await self._import_markdown(
|
||||
content, path.name, course_name, teacher_name, live_date
|
||||
)
|
||||
elif suffix == ".txt":
|
||||
content = path.read_text(encoding="utf-8")
|
||||
return await self._import_text(
|
||||
content, path.name, course_name, teacher_name, live_date
|
||||
)
|
||||
elif suffix == ".docx":
|
||||
return await self._import_docx(
|
||||
str(path), path.name, course_name, teacher_name, live_date
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"不支持的文件格式: {suffix},仅支持 .md、.txt 和 .docx")
|
||||
|
||||
async def _import_markdown(
|
||||
self,
|
||||
content: str,
|
||||
filename: str,
|
||||
course_name: Optional[str] = None,
|
||||
teacher_name: Optional[str] = None,
|
||||
live_date: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""导入 Markdown 文件,解析 frontmatter 并按标题分页"""
|
||||
# 解析 frontmatter
|
||||
fm_data, body = _parse_frontmatter(content)
|
||||
|
||||
# frontmatter 中的元数据可作为默认值
|
||||
fm_course = fm_data.get("course", fm_data.get("course_name"))
|
||||
fm_teacher = fm_data.get("teacher", fm_data.get("teacher_name"))
|
||||
fm_date = fm_data.get("date", fm_data.get("live_date"))
|
||||
|
||||
final_course = course_name or fm_course
|
||||
final_teacher = teacher_name or fm_teacher
|
||||
final_date = live_date or (str(fm_date) if fm_date else None)
|
||||
|
||||
# 按二级标题(##)拆分页面
|
||||
sections = self._split_markdown_sections(body)
|
||||
|
||||
if not sections:
|
||||
# 没有二级标题,整体作为一个页面
|
||||
sections = [{"title": Path(filename).stem, "content": body}]
|
||||
|
||||
imported_pages = 0
|
||||
imported_chunks = 0
|
||||
|
||||
for idx, section in enumerate(sections):
|
||||
page_number = idx + 1
|
||||
|
||||
# 插入知识页面
|
||||
page_id = await self._insert_page(
|
||||
title=section["title"],
|
||||
content=section["content"],
|
||||
source_file=filename,
|
||||
course_name=final_course,
|
||||
teacher_name=final_teacher,
|
||||
live_date=final_date,
|
||||
page_number=page_number,
|
||||
)
|
||||
imported_pages += 1
|
||||
|
||||
# 分块并向量化
|
||||
chunks = self._chunk_text(section["content"])
|
||||
chunk_count = await self._insert_chunks(page_id, chunks)
|
||||
imported_chunks += chunk_count
|
||||
|
||||
logger.info(
|
||||
"Markdown 导入完成: file=%s, pages=%d, chunks=%d",
|
||||
filename, imported_pages, imported_chunks,
|
||||
)
|
||||
|
||||
return {
|
||||
"file": filename,
|
||||
"pages": imported_pages,
|
||||
"chunks": imported_chunks,
|
||||
}
|
||||
|
||||
async def _import_text(
|
||||
self,
|
||||
content: str,
|
||||
filename: str,
|
||||
course_name: Optional[str] = None,
|
||||
teacher_name: Optional[str] = None,
|
||||
live_date: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""导入纯文本文件,整体作为一个页面"""
|
||||
title = Path(filename).stem
|
||||
|
||||
page_id = await self._insert_page(
|
||||
title=title,
|
||||
content=content,
|
||||
source_file=filename,
|
||||
course_name=course_name,
|
||||
teacher_name=teacher_name,
|
||||
live_date=live_date,
|
||||
page_number=1,
|
||||
)
|
||||
|
||||
chunks = self._chunk_text(content)
|
||||
chunk_count = await self._insert_chunks(page_id, chunks)
|
||||
|
||||
logger.info(
|
||||
"文本导入完成: file=%s, chunks=%d",
|
||||
filename, chunk_count,
|
||||
)
|
||||
|
||||
return {
|
||||
"file": filename,
|
||||
"pages": 1,
|
||||
"chunks": chunk_count,
|
||||
}
|
||||
|
||||
async def _import_docx(
|
||||
self,
|
||||
file_path: str,
|
||||
filename: str,
|
||||
course_name: Optional[str] = None,
|
||||
teacher_name: Optional[str] = None,
|
||||
live_date: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
导入 Word 文档(.docx)。
|
||||
提取所有段落文本,按标题(Heading 1/2)拆分为多个知识页面。
|
||||
"""
|
||||
from docx import Document
|
||||
|
||||
doc = Document(file_path)
|
||||
|
||||
# 提取所有段落,保留标题层级信息
|
||||
sections: List[dict] = [] # [{"title": "...", "content": "...", "level": int}]
|
||||
current_title = Path(filename).stem
|
||||
current_content: List[str] = []
|
||||
current_level = 0
|
||||
|
||||
for para in doc.paragraphs:
|
||||
style_name = (para.style.name or "").lower()
|
||||
|
||||
# 判断是否为标题段落
|
||||
if style_name.startswith("heading"):
|
||||
try:
|
||||
level = int(style_name.replace("heading", "").strip())
|
||||
except ValueError:
|
||||
level = 1
|
||||
|
||||
# 保存上一个段落
|
||||
if current_content:
|
||||
text = "\n\n".join(current_content).strip()
|
||||
if text:
|
||||
sections.append({
|
||||
"title": current_title,
|
||||
"content": text,
|
||||
"level": current_level,
|
||||
})
|
||||
|
||||
# 开始新段落
|
||||
current_title = para.text.strip() or f"第 {len(sections) + 1} 节"
|
||||
current_content = []
|
||||
current_level = level
|
||||
else:
|
||||
text = para.text.strip()
|
||||
if text:
|
||||
current_content.append(text)
|
||||
|
||||
# 保存最后一个段落
|
||||
if current_content:
|
||||
text = "\n\n".join(current_content).strip()
|
||||
if text:
|
||||
sections.append({
|
||||
"title": current_title,
|
||||
"content": text,
|
||||
"level": current_level,
|
||||
})
|
||||
|
||||
if not sections:
|
||||
raise ValueError(f"Word 文档内容为空: {filename}")
|
||||
|
||||
# 如果只有一个段落且没有标题,整体作为一个页面
|
||||
if len(sections) == 1 and sections[0]["level"] == 0:
|
||||
sections[0]["title"] = Path(filename).stem
|
||||
|
||||
imported_pages = 0
|
||||
imported_chunks = 0
|
||||
|
||||
for idx, section in enumerate(sections):
|
||||
page_id = await self._insert_page(
|
||||
title=section["title"],
|
||||
content=section["content"],
|
||||
source_file=filename,
|
||||
course_name=course_name,
|
||||
teacher_name=teacher_name,
|
||||
live_date=live_date,
|
||||
page_number=idx + 1,
|
||||
)
|
||||
imported_pages += 1
|
||||
|
||||
chunks = self._chunk_text(section["content"])
|
||||
chunk_count = await self._insert_chunks(page_id, chunks)
|
||||
imported_chunks += chunk_count
|
||||
|
||||
logger.info(
|
||||
"Word 文档导入完成: file=%s, pages=%d, chunks=%d",
|
||||
filename, imported_pages, imported_chunks,
|
||||
)
|
||||
|
||||
return {
|
||||
"file": filename,
|
||||
"pages": imported_pages,
|
||||
"chunks": imported_chunks,
|
||||
}
|
||||
|
||||
async def _insert_page(
|
||||
self,
|
||||
title: str,
|
||||
content: str,
|
||||
source_file: str,
|
||||
course_name: Optional[str] = None,
|
||||
teacher_name: Optional[str] = None,
|
||||
live_date: Optional[str] = None,
|
||||
page_number: Optional[int] = None,
|
||||
) -> int:
|
||||
"""插入知识页面记录,返回页面 ID"""
|
||||
sql = text("""
|
||||
INSERT INTO knowledge_pages (title, content, source_file, course_name, teacher_name, live_date, page_number)
|
||||
VALUES (:title, :content, :source_file, :course_name, :teacher_name, :live_date, :page_number)
|
||||
RETURNING id
|
||||
""")
|
||||
result = await self.db.execute(sql, {
|
||||
"title": title,
|
||||
"content": content,
|
||||
"source_file": source_file,
|
||||
"course_name": course_name,
|
||||
"teacher_name": teacher_name,
|
||||
"live_date": live_date,
|
||||
"page_number": page_number,
|
||||
})
|
||||
row = result.fetchone()
|
||||
await self.db.flush()
|
||||
return row.id
|
||||
|
||||
async def _insert_chunks(self, page_id: int, chunks: List[str]) -> int:
|
||||
"""
|
||||
批量插入分块记录并生成嵌入向量。
|
||||
|
||||
Args:
|
||||
page_id: 关联的知识页面 ID
|
||||
chunks: 分块文本列表
|
||||
|
||||
Returns:
|
||||
插入的分块数量
|
||||
"""
|
||||
if not chunks:
|
||||
return 0
|
||||
|
||||
# 批量生成嵌入向量
|
||||
try:
|
||||
from app.services.embedding_service import EmbeddingService
|
||||
embeddings = await EmbeddingService.embed_batch(chunks)
|
||||
except Exception as exc:
|
||||
logger.error("嵌入向量生成失败,分块将不包含向量: %s", exc)
|
||||
embeddings = [None] * len(chunks)
|
||||
|
||||
# 批量插入
|
||||
for idx, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):
|
||||
if IS_SQLITE:
|
||||
# SQLite: 向量存为 JSON 文本
|
||||
import json as _json
|
||||
embedding_str = _json.dumps(embedding) if embedding else None
|
||||
sql = text("""
|
||||
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
|
||||
VALUES (:page_id, :chunk_index, :content, :embedding)
|
||||
""")
|
||||
else:
|
||||
# PostgreSQL: 使用 pgvector 类型
|
||||
embedding_str = "[" + ",".join(str(x) for x in embedding) + "]" if embedding else None
|
||||
sql = text("""
|
||||
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
|
||||
VALUES (:page_id, :chunk_index, :content, :embedding::vector)
|
||||
""")
|
||||
await self.db.execute(sql, {
|
||||
"page_id": page_id,
|
||||
"chunk_index": idx,
|
||||
"content": chunk_text,
|
||||
"embedding": embedding_str,
|
||||
})
|
||||
|
||||
# 更新全文搜索向量(仅 PostgreSQL)
|
||||
if not IS_SQLITE:
|
||||
await self.db.execute(text("""
|
||||
UPDATE knowledge_chunks
|
||||
SET search_vector = to_tsvector('chinese_zh', content)
|
||||
WHERE page_id = :page_id
|
||||
"""), {"page_id": page_id})
|
||||
|
||||
await self.db.flush()
|
||||
return len(chunks)
|
||||
|
||||
@staticmethod
|
||||
def _split_markdown_sections(body: str) -> List[dict]:
|
||||
"""
|
||||
按二级标题(##)拆分 Markdown 内容为多个段落。
|
||||
|
||||
Returns:
|
||||
[{"title": "...", "content": "..."}, ...]
|
||||
"""
|
||||
sections: List[dict] = []
|
||||
# 匹配 ## 开头的标题
|
||||
pattern = re.compile(r"^##\s+(.+)$", re.MULTILINE)
|
||||
matches = list(pattern.finditer(body))
|
||||
|
||||
if not matches:
|
||||
return []
|
||||
|
||||
for i, match in enumerate(matches):
|
||||
title = match.group(1).strip()
|
||||
start = match.end()
|
||||
end = matches[i + 1].start() if i + 1 < len(matches) else len(body)
|
||||
content = body[start:end].strip()
|
||||
if content:
|
||||
sections.append({"title": title, "content": content})
|
||||
|
||||
return sections
|
||||
|
||||
@staticmethod
|
||||
def _chunk_text(text: str) -> List[str]:
|
||||
"""
|
||||
将文本按固定大小分块,保留重叠部分。
|
||||
|
||||
使用简单的字符数分块策略,按段落边界切分以保持语义完整性。
|
||||
"""
|
||||
chunk_size = settings.CHUNK_SIZE
|
||||
overlap = settings.CHUNK_OVERLAP
|
||||
|
||||
if not text or len(text) <= chunk_size:
|
||||
return [text] if text.strip() else []
|
||||
|
||||
# 按段落分割
|
||||
paragraphs = re.split(r"\n{2,}", text)
|
||||
chunks: List[str] = []
|
||||
current_chunk = ""
|
||||
|
||||
for para in paragraphs:
|
||||
para = para.strip()
|
||||
if not para:
|
||||
continue
|
||||
|
||||
if len(current_chunk) + len(para) + 2 <= chunk_size:
|
||||
# 当前块还能容纳这个段落
|
||||
if current_chunk:
|
||||
current_chunk += "\n\n" + para
|
||||
else:
|
||||
current_chunk = para
|
||||
else:
|
||||
# 当前块已满,保存并开始新块
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
if len(para) > chunk_size:
|
||||
# 单个段落超过 chunk_size,强制切分
|
||||
for i in range(0, len(para), chunk_size - overlap):
|
||||
piece = para[i : i + chunk_size]
|
||||
if piece.strip():
|
||||
chunks.append(piece)
|
||||
current_chunk = ""
|
||||
else:
|
||||
current_chunk = para
|
||||
|
||||
if current_chunk.strip():
|
||||
chunks.append(current_chunk)
|
||||
|
||||
return chunks
|
||||
479
app/services/llm_service.py
Normal file
479
app/services/llm_service.py
Normal file
@@ -0,0 +1,479 @@
|
||||
"""
|
||||
LLM 服务模块
|
||||
提供统一的 LLM 调用接口,用于查询扩展、标签提取等任务
|
||||
支持 MiniMax / DeepSeek / OpenAI(均为 OpenAI 兼容格式)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMService:
|
||||
"""
|
||||
LLM 服务类
|
||||
根据配置的 LLM_PROVIDER 自动选择对应的 API 端点。
|
||||
所有支持的提供商均兼容 OpenAI Chat Completions 格式。
|
||||
"""
|
||||
|
||||
_client = None
|
||||
|
||||
@classmethod
|
||||
def _get_client(cls):
|
||||
"""延迟初始化 OpenAI 兼容客户端"""
|
||||
if cls._client is not None:
|
||||
return cls._client
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
provider = settings.LLM_PROVIDER.lower()
|
||||
|
||||
if provider == "minimax":
|
||||
cls._client = AsyncOpenAI(
|
||||
api_key=settings.MINIMAX_API_KEY or "EMPTY",
|
||||
base_url=settings.MINIMAX_BASE_URL,
|
||||
)
|
||||
cls._model = settings.MINIMAX_CHAT_MODEL
|
||||
elif provider == "deepseek":
|
||||
cls._client = AsyncOpenAI(
|
||||
api_key=settings.DEEPSEEK_API_KEY or "EMPTY",
|
||||
base_url=settings.DEEPSEEK_BASE_URL,
|
||||
)
|
||||
cls._model = settings.DEEPSEEK_OCR_MODEL
|
||||
elif provider == "openai":
|
||||
cls._client = AsyncOpenAI(
|
||||
api_key=settings.OPENAI_API_KEY or "EMPTY",
|
||||
base_url=settings.OPENAI_BASE_URL,
|
||||
)
|
||||
cls._model = settings.EMBEDDING_MODEL
|
||||
else:
|
||||
raise ValueError(f"不支持的 LLM 提供商: {provider}")
|
||||
|
||||
logger.info("LLM 服务初始化完成: provider=%s, model=%s", provider, cls._model)
|
||||
return cls._client
|
||||
|
||||
@staticmethod
|
||||
def _clean_reasoning_content(content: str) -> str:
|
||||
"""清理推理模型的思考过程,只保留最终回答"""
|
||||
import re
|
||||
|
||||
# 方式1: 清理 🧠...💠 标签(MiniMax M2.7 reasoning 模式)
|
||||
content = re.sub(r'🧠[\s\S]*?💠', '', content)
|
||||
|
||||
# 方式2: 清理 💭(U+1F4AD) 包裹的思考块
|
||||
if '💭' in content:
|
||||
parts = content.split('💭')
|
||||
answer_parts = [parts[i] for i in range(len(parts)) if i % 2 == 1]
|
||||
if answer_parts:
|
||||
content = ''.join(answer_parts)
|
||||
|
||||
# 方式3: 清理 </think 标签(M2.7 有时使用)
|
||||
think_end = content.find('</think')
|
||||
if think_end >= 0:
|
||||
after = content[think_end + len('</think'):]
|
||||
after = re.sub(r'^[>\n\s]*', '', after)
|
||||
content = after
|
||||
|
||||
# 方式4: 清理 \x0f 分隔的思考块
|
||||
if '\x0f' in content:
|
||||
parts = content.split('\x0f')
|
||||
content = ''.join(parts[1::2]) if len(parts) > 1 else parts[-1] if parts else content
|
||||
|
||||
# 方式5: 清理 0x00 等特殊字符
|
||||
content = content.replace('\x00', '').strip()
|
||||
|
||||
# 方式6: 如果内容仍然很长(说明还有思考残留),取最后一个短段落作为答案
|
||||
if len(content) > 100 and '\n\n' in content:
|
||||
segments = [s.strip() for s in content.split('\n\n')]
|
||||
for seg in reversed(segments):
|
||||
if seg and len(seg) <= 100:
|
||||
content = seg
|
||||
break
|
||||
|
||||
return content.strip()
|
||||
|
||||
@classmethod
|
||||
async def chat(cls, messages: List[dict], temperature: float = 0.3, max_tokens: int = 1024) -> str:
|
||||
"""
|
||||
发送聊天请求。
|
||||
|
||||
Args:
|
||||
messages: 消息列表,格式 [{"role": "system/user/assistant", "content": "..."}]
|
||||
temperature: 采样温度
|
||||
max_tokens: 最大生成 token 数
|
||||
|
||||
Returns:
|
||||
模型回复文本(已清理思考过程)
|
||||
"""
|
||||
client = cls._get_client()
|
||||
|
||||
# MiniMax 模型默认开启思考模式,使用 reasoning_split 将思考内容
|
||||
# 分离到 reasoning_details 字段,content 只保留最终回答
|
||||
extra_body = {}
|
||||
if settings.LLM_PROVIDER.lower() == "minimax":
|
||||
extra_body["reasoning_split"] = True
|
||||
|
||||
response = await client.chat.completions.create(
|
||||
model=cls._model,
|
||||
messages=messages,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
**({"extra_body": extra_body} if extra_body else {}),
|
||||
)
|
||||
content = response.choices[0].message.content or ""
|
||||
|
||||
# 兜底清理:以防 reasoning_split 未生效或使用了其他模型
|
||||
content = cls._clean_reasoning_content(content)
|
||||
|
||||
return content
|
||||
|
||||
@classmethod
|
||||
async def expand_query(cls, query: str) -> List[str]:
|
||||
"""
|
||||
查询扩展:将原始查询改写为多个语义相近的表述。
|
||||
用于混合搜索时提高召回率。
|
||||
|
||||
Args:
|
||||
query: 原始查询文本
|
||||
|
||||
Returns:
|
||||
扩展后的查询列表(包含原始查询)
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"你是一个搜索查询优化助手。用户会给你一个搜索查询,"
|
||||
"你需要生成 2 个语义相近但表述不同的替代查询。"
|
||||
"只输出 JSON 数组格式,不要添加任何其他文字。"
|
||||
'例如:["原始查询", "替代查询1", "替代查询2"]'
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"请为以下查询生成 2 个替代表述:{query}",
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
result = await cls.chat(messages, temperature=0.3, max_tokens=256)
|
||||
# 尝试解析 JSON
|
||||
result = result.strip()
|
||||
if result.startswith("```"):
|
||||
result = result.split("\n", 1)[-1]
|
||||
if result.endswith("```"):
|
||||
result = result[:-3]
|
||||
result = result.strip()
|
||||
queries = json.loads(result)
|
||||
if isinstance(queries, list):
|
||||
# 去重并限制数量
|
||||
seen = set()
|
||||
unique = []
|
||||
for q in queries:
|
||||
q = str(q).strip()
|
||||
if q and q not in seen:
|
||||
seen.add(q)
|
||||
unique.append(q)
|
||||
return unique[:3]
|
||||
except Exception as exc:
|
||||
logger.warning("查询扩展失败: %s,使用原始查询", exc)
|
||||
|
||||
return [query]
|
||||
|
||||
@classmethod
|
||||
async def detect_entities(cls, text: str) -> List[dict]:
|
||||
"""
|
||||
实体检测:从文本中提取知识点、人物、概念等实体。
|
||||
|
||||
Args:
|
||||
text: 待分析的文本
|
||||
|
||||
Returns:
|
||||
实体列表,格式 [{"name": "...", "type": "knowledge_point|person|concept|technique", "description": "..."}]
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"你是一个教育内容分析助手。用户会给你一段直播课的文字稿,"
|
||||
"你需要从中提取关键实体(知识点、技巧、人物、概念等)。"
|
||||
"只输出 JSON 数组格式,不要添加任何其他文字。\n"
|
||||
'格式:[{"name": "实体名称", "type": "knowledge_point|technique|person|concept", "description": "简要描述"}]\n'
|
||||
"type 说明:\n"
|
||||
"- knowledge_point: 知识点(如公式、定理、定义)\n"
|
||||
"- technique: 技巧/方法(如解题技巧、记忆方法)\n"
|
||||
"- person: 人物\n"
|
||||
"- concept: 概念/术语"
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"请从以下文字中提取关键实体:\n\n{text[:3000]}",
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
result = await cls.chat(messages, temperature=0.1, max_tokens=2048)
|
||||
result = result.strip()
|
||||
if result.startswith("```"):
|
||||
result = result.split("\n", 1)[-1]
|
||||
if result.endswith("```"):
|
||||
result = result[:-3]
|
||||
result = result.strip()
|
||||
entities = json.loads(result)
|
||||
if isinstance(entities, list):
|
||||
return entities
|
||||
except Exception as exc:
|
||||
logger.warning("实体检测失败: %s", exc)
|
||||
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def extract_tags(cls, text: str) -> List[str]:
|
||||
"""
|
||||
从文本中提取语义标签。
|
||||
用于图片 OCR 内容的标签化和搜索查询的标签化。
|
||||
|
||||
Args:
|
||||
text: 待提取标签的文本
|
||||
|
||||
Returns:
|
||||
标签列表,如 ["教育", "高考数学", "二次方程", "解题技巧"]
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"你是一个内容标签提取助手。用户会给你一段文字内容,"
|
||||
"你需要从中提取 3-10 个能概括内容主题的标签。\n"
|
||||
"标签要求:\n"
|
||||
"1. 用简短的词语(2-6个字)\n"
|
||||
"2. 涵盖主题、场景、情感、关键实体等维度\n"
|
||||
"3. 考虑同义词和近义词(如'不想工作'也打上'躺平'标签)\n"
|
||||
"4. 只输出 JSON 数组,不要其他文字\n"
|
||||
'例如:["高考数学", "二次方程", "韦达定理", "解题技巧"]'
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"请从以下内容中提取标签:\n\n{text[:2000]}",
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
result = await cls.chat(messages, temperature=0.3, max_tokens=256)
|
||||
result = result.strip()
|
||||
if result.startswith("```"):
|
||||
result = result.split("\n", 1)[-1]
|
||||
if result.endswith("```"):
|
||||
result = result[:-3]
|
||||
result = result.strip()
|
||||
tags = json.loads(result)
|
||||
if isinstance(tags, list):
|
||||
# 去重、清洗
|
||||
seen = set()
|
||||
clean = []
|
||||
for t in tags:
|
||||
t = str(t).strip()
|
||||
if 1 <= len(t) <= 20 and t not in seen:
|
||||
seen.add(t)
|
||||
clean.append(t)
|
||||
return clean[:15]
|
||||
except Exception as exc:
|
||||
logger.warning("标签提取失败: %s", exc)
|
||||
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def extract_search_tags(cls, query: str) -> List[str]:
|
||||
"""
|
||||
从用户搜索查询中提取标签,用于匹配数据库中存储的标签。
|
||||
和 extract_tags 类似,但更侧重于提取搜索意图相关的标签。
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"你是一个搜索意图分析助手。用户会给你一个搜索查询,"
|
||||
"你需要提取 3-8 个可能匹配的标签词。\n"
|
||||
"标签要求:\n"
|
||||
"1. 用简短的词语(2-6个字)\n"
|
||||
"2. 包含同义词和近义词\n"
|
||||
"3. 包含上义词和下义词\n"
|
||||
"4. 只输出 JSON 数组\n"
|
||||
'例如:用户搜"孩子躺平",输出 ["躺平", "不上班", "年轻人", "就业", "摆烂", "啃老"]'
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"请从以下搜索查询中提取匹配标签:{query}",
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
result = await cls.chat(messages, temperature=0.3, max_tokens=256)
|
||||
result = result.strip()
|
||||
if result.startswith("```"):
|
||||
result = result.split("\n", 1)[-1]
|
||||
if result.endswith("```"):
|
||||
result = result[:-3]
|
||||
result = result.strip()
|
||||
tags = json.loads(result)
|
||||
if isinstance(tags, list):
|
||||
seen = set()
|
||||
clean = []
|
||||
for t in tags:
|
||||
t = str(t).strip()
|
||||
if 1 <= len(t) <= 20 and t not in seen:
|
||||
seen.add(t)
|
||||
clean.append(t)
|
||||
return clean[:10]
|
||||
except Exception as exc:
|
||||
logger.warning("搜索标签提取失败: %s", exc)
|
||||
|
||||
# 降级:直接用原始查询词作为标签
|
||||
return [query.strip()] if query.strip() else []
|
||||
|
||||
@classmethod
|
||||
async def summarize_story(cls, text: str) -> str:
|
||||
"""
|
||||
用 Qwen3-8B 提炼图片 OCR 文本的故事内容。
|
||||
非思考模式,直接输出。
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"你是一个内容提炼助手。用户会给你一段从图片中识别出的文字内容,"
|
||||
"你需要提炼出其中讲述的故事或事件。\n"
|
||||
"要求:\n"
|
||||
"1. 用简洁的自然语言描述图片内容讲述的故事或事件\n"
|
||||
"2. 保留关键人物、事件、情感、场景等核心信息\n"
|
||||
"3. 50-200字\n"
|
||||
"4. 直接输出提炼内容,不要加前缀"
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"/no_think\n请提炼以下内容的故事:\n\n{text[:3000]}",
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
from openai import AsyncOpenAI
|
||||
client = AsyncOpenAI(
|
||||
api_key=settings.OPENAI_API_KEY,
|
||||
base_url=settings.OPENAI_BASE_URL,
|
||||
)
|
||||
response = await client.chat.completions.create(
|
||||
model="Qwen/Qwen3-8B",
|
||||
messages=messages,
|
||||
temperature=0.3,
|
||||
max_tokens=512,
|
||||
extra_body={"chat_template_kwargs": {"enable_thinking": False}},
|
||||
)
|
||||
content = response.choices[0].message.content or ""
|
||||
# 清理可能的 /no_think 回显
|
||||
content = content.replace("/no_think", "").strip()
|
||||
return content
|
||||
except Exception as exc:
|
||||
logger.warning("故事提炼失败: %s", exc)
|
||||
return ""
|
||||
|
||||
@classmethod
|
||||
async def judge_batch(cls, query: str, articles: List[dict]) -> List[int]:
|
||||
"""
|
||||
用 LLM 批量判断哪些文章符合查询。
|
||||
|
||||
Args:
|
||||
query: 用户搜索查询
|
||||
articles: [{"id": int, "text": str}, ...],最多10个
|
||||
|
||||
Returns:
|
||||
匹配的文章 id 列表(空列表表示都不匹配)
|
||||
"""
|
||||
if not articles:
|
||||
return []
|
||||
|
||||
# 构建文章列表
|
||||
articles_text = ""
|
||||
for i, article in enumerate(articles, 1):
|
||||
text = (article.get("text") or "")[:800]
|
||||
articles_text += f"{i}. {text}\n"
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"你是一个严格的语义搜索匹配引擎。用户会给你一个搜索查询和多篇文章片段,"
|
||||
"你需要判断哪些文章与查询在**核心语义上高度相关**。\n\n"
|
||||
"判断原则:\n"
|
||||
"- 文章的**核心主题**必须与查询直接相关,而不是仅仅提及了查询中的某个词\n"
|
||||
"- 例如:查询\"爸爸出轨\",文章只提到\"爸爸带孩子玩\"是不相关的,"
|
||||
"只有文章讨论出轨、婚姻危机、夫妻矛盾等才算相关\n"
|
||||
"- 例如:查询\"孩子不想上学\",文章提到\"孩子打游戏不去学校\"是相关的,"
|
||||
"但只提到\"孩子\"或\"学校\"其他方面则不相关\n\n"
|
||||
"不相关的典型情况(必须排除):\n"
|
||||
"- 仅包含查询中的某个词,但讨论的是完全不同的话题\n"
|
||||
"- 主题相近但具体内容无关(如查询出轨,文章讲的是正常的夫妻相处)\n"
|
||||
"- 只有一个宽泛的关联词,没有实质性的语义联系\n\n"
|
||||
"只输出匹配的文章编号,用逗号分隔。\n"
|
||||
"如果没有文章与查询相关,只输出 0。\n"
|
||||
"不要输出任何解释。宁可漏判,不可误判。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"搜索查询: {query}\n\n{articles_text}\n请输出匹配的文章编号:",
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
from openai import AsyncOpenAI
|
||||
client = AsyncOpenAI(
|
||||
api_key=settings.MINIMAX_API_KEY,
|
||||
base_url=settings.MINIMAX_BASE_URL,
|
||||
http_client=httpx.AsyncClient(timeout=60.0),
|
||||
)
|
||||
response = await client.chat.completions.create(
|
||||
model=settings.MINIMAX_CHAT_MODEL,
|
||||
messages=messages,
|
||||
temperature=0.0,
|
||||
max_tokens=512,
|
||||
stream=False,
|
||||
extra_body={"reasoning_split": True},
|
||||
)
|
||||
content = response.choices[0].message.content or ""
|
||||
content = cls._clean_reasoning_content(content)
|
||||
|
||||
# 解析返回的编号
|
||||
numbers = re.findall(r'\d+', content)
|
||||
|
||||
if not numbers:
|
||||
return []
|
||||
|
||||
matched_ids = []
|
||||
for num_str in numbers:
|
||||
num = int(num_str)
|
||||
if num == 0:
|
||||
continue # 0 表示都不匹配
|
||||
if 1 <= num <= len(articles):
|
||||
matched_ids.append(articles[num - 1]["id"])
|
||||
|
||||
return matched_ids
|
||||
except Exception as exc:
|
||||
logger.warning("LLM 批量判断失败: %s", exc)
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def reset(cls) -> None:
|
||||
"""重置客户端(用于测试或切换配置)"""
|
||||
cls._client = None
|
||||
540
app/services/ocr_service.py
Normal file
540
app/services/ocr_service.py
Normal file
@@ -0,0 +1,540 @@
|
||||
"""
|
||||
OCR 识别服务模块
|
||||
支持多种 OCR 提供商:PaddleOCR(本地)/ 阿里云 / 腾讯云
|
||||
auto 模式下本地优先 + 云端 fallback
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from app.config import OCRProvider, settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ──────────────────────────── 数据结构 ────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class OCRBlock:
|
||||
"""OCR 识别的文本块"""
|
||||
text: str
|
||||
bbox: Optional[List[float]] = None # [x1, y1, x2, y2]
|
||||
confidence: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class OCRResult:
|
||||
"""OCR 识别结果"""
|
||||
text: str
|
||||
blocks: List[OCRBlock] = field(default_factory=list)
|
||||
confidence: float = 0.0
|
||||
provider: str = ""
|
||||
keywords: List[str] = field(default_factory=list) # OCR 时提取的关键词标签
|
||||
|
||||
|
||||
# ──────────────────────────── 抽象基类 ────────────────────────────
|
||||
|
||||
class OCRProviderBase(abc.ABC):
|
||||
"""OCR 提供商抽象基类"""
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def name(self) -> str:
|
||||
"""提供商名称"""
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
async def recognize(self, image_path: str) -> OCRResult:
|
||||
"""
|
||||
识别图片中的文字。
|
||||
|
||||
Args:
|
||||
image_path: 图片文件路径
|
||||
|
||||
Returns:
|
||||
OCRResult 包含识别文本、文本块和置信度
|
||||
|
||||
Raises:
|
||||
Exception: 识别失败时抛出异常
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
# ──────────────────────────── PaddleOCR 实现(本地) ────────────────────────────
|
||||
|
||||
class PaddleOCRProvider(OCRProviderBase):
|
||||
"""
|
||||
PaddleOCR 本地识别
|
||||
优点:免费、离线可用、中文识别效果好
|
||||
缺点:需要较多内存、首次加载模型较慢
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._ocr = None
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "paddleocr"
|
||||
|
||||
def _init_ocr(self):
|
||||
"""延迟初始化 PaddleOCR 引擎"""
|
||||
if self._ocr is not None:
|
||||
return
|
||||
|
||||
logger.info("正在初始化 PaddleOCR 引擎...")
|
||||
from paddleocr import PaddleOCR
|
||||
|
||||
self._ocr = PaddleOCR(
|
||||
use_angle_cls=True,
|
||||
lang="ch",
|
||||
use_gpu=False, # 默认 CPU,生产环境可通过环境变量控制
|
||||
show_log=False,
|
||||
enable_mkldnn=True, # 启用 MKLDNN 加速
|
||||
)
|
||||
logger.info("PaddleOCR 引擎初始化完成")
|
||||
|
||||
async def recognize(self, image_path: str) -> OCRResult:
|
||||
"""使用 PaddleOCR 识别图片"""
|
||||
self._init_ocr()
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _run_ocr():
|
||||
result = self._ocr.ocr(image_path, cls=True)
|
||||
return result
|
||||
|
||||
raw_result = await loop.run_in_executor(None, _run_ocr)
|
||||
|
||||
# 解析 PaddleOCR 返回结果
|
||||
blocks: List[OCRBlock] = []
|
||||
all_text_parts: List[str] = []
|
||||
total_confidence = 0.0
|
||||
|
||||
if raw_result and raw_result[0]:
|
||||
for line in raw_result[0]:
|
||||
bbox_points = line[0]
|
||||
text = line[1][0]
|
||||
confidence = float(line[1][1])
|
||||
|
||||
# 将四个角点转为 [x1, y1, x2, y2]
|
||||
xs = [p[0] for p in bbox_points]
|
||||
ys = [p[1] for p in bbox_points]
|
||||
bbox = [min(xs), min(ys), max(xs), max(ys)]
|
||||
|
||||
blocks.append(OCRBlock(text=text, bbox=bbox, confidence=confidence))
|
||||
all_text_parts.append(text)
|
||||
total_confidence += confidence
|
||||
|
||||
full_text = "\n".join(all_text_parts)
|
||||
avg_confidence = total_confidence / len(blocks) if blocks else 0.0
|
||||
|
||||
return OCRResult(
|
||||
text=full_text,
|
||||
blocks=blocks,
|
||||
confidence=avg_confidence,
|
||||
provider=self.name,
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────── 阿里云 OCR 实现 ────────────────────────────
|
||||
|
||||
class AliyunOCRProvider(OCRProviderBase):
|
||||
"""
|
||||
阿里云 OCR 识别
|
||||
使用阿里云文字识别 API(通用文字识别)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
if not settings.ALIYUN_OCR_ACCESS_KEY or not settings.ALIYUN_OCR_SECRET:
|
||||
raise ValueError("阿里云 OCR 需要配置 ALIYUN_OCR_ACCESS_KEY 和 ALIYUN_OCR_SECRET")
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "aliyun"
|
||||
|
||||
async def recognize(self, image_path: str) -> OCRResult:
|
||||
"""调用阿里云 OCR API 识别图片"""
|
||||
import base64
|
||||
import json
|
||||
|
||||
import asyncio
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _call_api():
|
||||
from alibabacloud_tea_openapi.models import Config
|
||||
from alibabacloud_ocr_api20210707.client import Client
|
||||
from alibabacloud_ocr_api20210707.models import RecognizeGeneralRequest
|
||||
from alibabacloud_tea_util.models import RuntimeOptions
|
||||
|
||||
config = Config(
|
||||
access_key_id=settings.ALIYUN_OCR_ACCESS_KEY,
|
||||
access_key_secret=settings.ALIYUN_OCR_SECRET,
|
||||
endpoint="ocr-api.cn-hangzhou.aliyuncs.com",
|
||||
)
|
||||
client = Client(config)
|
||||
|
||||
# 读取图片并 Base64 编码
|
||||
with open(image_path, "rb") as f:
|
||||
image_bytes = f.read()
|
||||
image_b64 = base64.b64encode(image_bytes).decode("utf-8")
|
||||
|
||||
request = RecognizeGeneralRequest(
|
||||
body=json.dumps({"url": f"data:image/png;base64,{image_b64}"}),
|
||||
)
|
||||
runtime = RuntimeOptions()
|
||||
|
||||
response = client.recognize_general_with_options(request, runtime)
|
||||
return response
|
||||
|
||||
response = await loop.run_in_executor(None, _call_api)
|
||||
|
||||
if response.body and response.body.data:
|
||||
data = json.loads(response.body.data)
|
||||
blocks: List[OCRBlock] = []
|
||||
all_text_parts: List[str] = []
|
||||
total_confidence = 0.0
|
||||
|
||||
# 阿里云返回格式可能包含多个文本块
|
||||
if isinstance(data, dict) and "content" in data:
|
||||
text = data["content"]
|
||||
all_text_parts.append(text)
|
||||
blocks.append(OCRBlock(text=text, confidence=1.0))
|
||||
total_confidence = 1.0
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
text = item.get("text", item.get("word", ""))
|
||||
score = float(item.get("score", item.get("confidence", 1.0)))
|
||||
blocks.append(OCRBlock(text=text, confidence=score))
|
||||
all_text_parts.append(text)
|
||||
total_confidence += score
|
||||
|
||||
full_text = "\n".join(all_text_parts)
|
||||
avg_confidence = total_confidence / len(blocks) if blocks else 0.0
|
||||
|
||||
return OCRResult(
|
||||
text=full_text,
|
||||
blocks=blocks,
|
||||
confidence=avg_confidence,
|
||||
provider=self.name,
|
||||
)
|
||||
|
||||
return OCRResult(text="", blocks=[], confidence=0.0, provider=self.name)
|
||||
|
||||
|
||||
# ──────────────────────────── 腾讯云 OCR 实现 ────────────────────────────
|
||||
|
||||
class TencentOCRProvider(OCRProviderBase):
|
||||
"""
|
||||
腾讯云 OCR 识别
|
||||
使用腾讯云文字识别 API(通用印刷体识别)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
if not settings.TENCENT_OCR_SECRET_ID or not settings.TENCENT_OCR_SECRET_KEY:
|
||||
raise ValueError("腾讯云 OCR 需要配置 TENCENT_OCR_SECRET_ID 和 TENCENT_OCR_SECRET_KEY")
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "tencent"
|
||||
|
||||
async def recognize(self, image_path: str) -> OCRResult:
|
||||
"""调用腾讯云 OCR API 识别图片"""
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _call_api():
|
||||
from tencentcloud.common import credential
|
||||
from tencentcloud.common.profile.client_profile import ClientProfile
|
||||
from tencentcloud.common.profile.http_profile import HttpProfile
|
||||
from tencentcloud.ocr.v20181119 import ocr_client, models
|
||||
|
||||
cred = credential.Credential(
|
||||
settings.TENCENT_OCR_SECRET_ID,
|
||||
settings.TENCENT_OCR_SECRET_KEY,
|
||||
)
|
||||
http_profile = HttpProfile()
|
||||
http_profile.endpoint = "ocr.tencentcloudapi.com"
|
||||
|
||||
client_profile = ClientProfile()
|
||||
client_profile.httpProfile = http_profile
|
||||
|
||||
client = ocr_client.OcrClient(cred, "", client_profile)
|
||||
|
||||
# 读取图片并 Base64 编码
|
||||
with open(image_path, "rb") as f:
|
||||
image_bytes = f.read()
|
||||
image_b64 = base64.b64encode(image_bytes).decode("utf-8")
|
||||
|
||||
req = models.GeneralBasicOCRRequest()
|
||||
req.ImageBase64 = image_b64
|
||||
|
||||
resp = client.GeneralBasicOCR(req)
|
||||
return resp
|
||||
|
||||
resp = await loop.run_in_executor(None, _call_api)
|
||||
|
||||
blocks: List[OCRBlock] = []
|
||||
all_text_parts: List[str] = []
|
||||
total_confidence = 0.0
|
||||
|
||||
if resp.TextDetections:
|
||||
for item in resp.TextDetections:
|
||||
text = item.DetectedText or ""
|
||||
confidence = item.Confidence / 100.0 if item.Confidence else 0.0
|
||||
|
||||
# 解析多边形顶点
|
||||
bbox = None
|
||||
if item.Polygon:
|
||||
xs = [p.X for p in item.Polygon]
|
||||
ys = [p.Y for p in item.Polygon]
|
||||
bbox = [min(xs), min(ys), max(xs), max(ys)]
|
||||
|
||||
blocks.append(OCRBlock(text=text, bbox=bbox, confidence=confidence))
|
||||
all_text_parts.append(text)
|
||||
total_confidence += confidence
|
||||
|
||||
full_text = "\n".join(all_text_parts)
|
||||
avg_confidence = total_confidence / len(blocks) if blocks else 0.0
|
||||
|
||||
return OCRResult(
|
||||
text=full_text,
|
||||
blocks=blocks,
|
||||
confidence=avg_confidence,
|
||||
provider=self.name,
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────── DeepSeek OCR 实现 ────────────────────────────
|
||||
|
||||
class DeepSeekOCRProvider(OCRProviderBase):
|
||||
"""
|
||||
DeepSeek Vision OCR
|
||||
通过 OpenAI 兼容接口发送图片 base64,让视觉模型识别文字。
|
||||
支持 DeepSeek 官方 API 和 Ollama 自部署(如 deepseek-ocr 模型)。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
if not settings.DEEPSEEK_API_KEY:
|
||||
raise ValueError("DeepSeek OCR 需要配置 DEEPSEEK_API_KEY")
|
||||
import httpx
|
||||
from openai import AsyncOpenAI
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=settings.DEEPSEEK_API_KEY,
|
||||
base_url=settings.DEEPSEEK_BASE_URL,
|
||||
http_client=httpx.AsyncClient(timeout=120.0), # OCR 较慢,超时 120 秒
|
||||
)
|
||||
self._model = settings.DEEPSEEK_OCR_MODEL
|
||||
# 判断是否为 Ollama 部署
|
||||
self._is_ollama = "ollama" in settings.DEEPSEEK_API_KEY.lower() or \
|
||||
"11434" in settings.DEEPSEEK_BASE_URL
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "deepseek"
|
||||
|
||||
async def recognize(self, image_path: str) -> OCRResult:
|
||||
"""使用 DeepSeek Vision 模型识别图片中的文字"""
|
||||
import base64
|
||||
|
||||
# 读取图片并编码为 base64
|
||||
with open(image_path, "rb") as f:
|
||||
image_bytes = f.read()
|
||||
base64_image = base64.b64encode(image_bytes).decode("utf-8")
|
||||
|
||||
# 根据图片格式确定 MIME 类型
|
||||
suffix = Path(image_path).suffix.lower()
|
||||
mime_map = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".bmp": "image/bmp",
|
||||
".webp": "image/webp",
|
||||
}
|
||||
mime_type = mime_map.get(suffix, "image/png")
|
||||
|
||||
system_prompt = (
|
||||
"你是一个 OCR 文字识别工具。"
|
||||
"请识别图片中的所有文字,逐行输出,不要添加任何解释。"
|
||||
)
|
||||
|
||||
response = await self._client.chat.completions.create(
|
||||
model=self._model,
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": system_prompt,
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:{mime_type};base64,{base64_image}"
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "请识别这张图片中的所有文字内容。",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
max_tokens=4096,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
text = response.choices[0].message.content or ""
|
||||
|
||||
# 清理特殊 token(deepseek-ocr 模型可能输出 <|begin of sentence|> 等)
|
||||
import re
|
||||
text = re.sub(r'<|[^>]*|>', '', text)
|
||||
text = re.sub(r'<\|[^>]*\|>', '', text)
|
||||
|
||||
# 清理可能的 markdown 包裹
|
||||
text = text.strip()
|
||||
if text.startswith("```"):
|
||||
text = text.split("\n", 1)[-1]
|
||||
if text.endswith("```"):
|
||||
text = text[:-3]
|
||||
text = text.strip()
|
||||
|
||||
lines = [line for line in text.split("\n") if line.strip()]
|
||||
blocks = [OCRBlock(text=line, confidence=1.0) for line in lines]
|
||||
|
||||
return OCRResult(
|
||||
text=text,
|
||||
blocks=blocks,
|
||||
confidence=1.0,
|
||||
provider=self.name,
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────── 工厂类 ────────────────────────────
|
||||
|
||||
class OCRService:
|
||||
"""
|
||||
OCR 服务工厂类
|
||||
根据配置自动选择 OCR 提供商。
|
||||
auto 模式下:本地 PaddleOCR 优先,失败后 fallback 到云端 OCR。
|
||||
"""
|
||||
|
||||
_instance: Optional[OCRProviderBase] = None
|
||||
|
||||
@classmethod
|
||||
def _create_provider(cls, provider: OCRProvider) -> OCRProviderBase:
|
||||
"""根据提供商枚举创建对应的 OCR 实例"""
|
||||
provider_map = {
|
||||
OCRProvider.PADDLEOCR: PaddleOCRProvider,
|
||||
OCRProvider.ALIYUN: AliyunOCRProvider,
|
||||
OCRProvider.TENCENT: TencentOCRProvider,
|
||||
OCRProvider.DEEPSEEK: DeepSeekOCRProvider,
|
||||
}
|
||||
provider_cls = provider_map.get(provider)
|
||||
if provider_cls is None:
|
||||
raise ValueError(f"不支持的 OCR 提供商: {provider}")
|
||||
return provider_cls()
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> OCRProviderBase:
|
||||
"""获取 OCR 服务单例实例"""
|
||||
if cls._instance is not None:
|
||||
return cls._instance
|
||||
|
||||
if settings.OCR_PROVIDER == OCRProvider.AUTO:
|
||||
# auto 模式返回 PaddleOCR,fallback 逻辑在 recognize_auto 中处理
|
||||
cls._instance = PaddleOCRProvider()
|
||||
else:
|
||||
cls._instance = cls._create_provider(settings.OCR_PROVIDER)
|
||||
|
||||
logger.info("OCR 服务初始化完成: provider=%s", cls._instance.name)
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
async def recognize(cls, image_path: str) -> OCRResult:
|
||||
"""
|
||||
识别图片文字。
|
||||
auto 模式下:PaddleOCR 优先,失败后依次尝试阿里云、腾讯云。
|
||||
"""
|
||||
if settings.OCR_PROVIDER == OCRProvider.AUTO:
|
||||
return await cls._recognize_auto(image_path)
|
||||
|
||||
instance = cls.get_instance()
|
||||
return await instance.recognize(image_path)
|
||||
|
||||
@classmethod
|
||||
async def _recognize_auto(cls, image_path: str) -> OCRResult:
|
||||
"""
|
||||
auto 模式识别逻辑:
|
||||
1. 先尝试本地 PaddleOCR
|
||||
2. 如果失败且配置了阿里云,尝试阿里云 OCR
|
||||
3. 如果仍失败且配置了腾讯云,尝试腾讯云 OCR
|
||||
"""
|
||||
# 第一步:本地 PaddleOCR
|
||||
try:
|
||||
provider = PaddleOCRProvider()
|
||||
result = await provider.recognize(image_path)
|
||||
if result.text.strip():
|
||||
logger.info("PaddleOCR 识别成功,文本长度: %d", len(result.text))
|
||||
return result
|
||||
logger.warning("PaddleOCR 返回空文本,尝试 fallback")
|
||||
except Exception as exc:
|
||||
logger.warning("PaddleOCR 识别失败: %s,尝试 fallback", exc)
|
||||
|
||||
# 第二步:DeepSeek OCR(如果配置了)
|
||||
if settings.DEEPSEEK_API_KEY:
|
||||
try:
|
||||
provider = DeepSeekOCRProvider()
|
||||
result = await provider.recognize(image_path)
|
||||
if result.text.strip():
|
||||
logger.info("DeepSeek OCR 识别成功,文本长度: %d", len(result.text))
|
||||
return result
|
||||
logger.warning("DeepSeek OCR 返回空文本,尝试 fallback")
|
||||
except Exception as exc:
|
||||
logger.warning("DeepSeek OCR 识别失败: %s,尝试 fallback", exc)
|
||||
|
||||
# 第三步:阿里云 OCR
|
||||
if settings.ALIYUN_OCR_ACCESS_KEY and settings.ALIYUN_OCR_SECRET:
|
||||
try:
|
||||
provider = AliyunOCRProvider()
|
||||
result = await provider.recognize(image_path)
|
||||
if result.text.strip():
|
||||
logger.info("阿里云 OCR 识别成功,文本长度: %d", len(result.text))
|
||||
return result
|
||||
logger.warning("阿里云 OCR 返回空文本,继续 fallback")
|
||||
except Exception as exc:
|
||||
logger.warning("阿里云 OCR 识别失败: %s,继续 fallback", exc)
|
||||
|
||||
# 第四步:腾讯云 OCR
|
||||
if settings.TENCENT_OCR_SECRET_ID and settings.TENCENT_OCR_SECRET_KEY:
|
||||
try:
|
||||
provider = TencentOCRProvider()
|
||||
result = await provider.recognize(image_path)
|
||||
if result.text.strip():
|
||||
logger.info("腾讯云 OCR 识别成功,文本长度: %d", len(result.text))
|
||||
return result
|
||||
logger.warning("腾讯云 OCR 返回空文本")
|
||||
except Exception as exc:
|
||||
logger.warning("腾讯云 OCR 识别失败: %s", exc)
|
||||
|
||||
# 所有 OCR 都失败
|
||||
return OCRResult(
|
||||
text="",
|
||||
blocks=[],
|
||||
confidence=0.0,
|
||||
provider="none",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def reset(cls) -> None:
|
||||
"""重置单例(用于测试或切换配置)"""
|
||||
cls._instance = None
|
||||
202
app/services/page_service.py
Normal file
202
app/services/page_service.py
Normal file
@@ -0,0 +1,202 @@
|
||||
"""
|
||||
知识页面服务
|
||||
提供知识页面的 CRUD 操作
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.base import KnowledgeChunk, KnowledgePage
|
||||
from app.schemas.page import PageCreate, PageResponse, PageUpdate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PageService:
|
||||
"""知识页面服务"""
|
||||
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
async def create_page(self, data: PageCreate) -> KnowledgePage:
|
||||
"""创建知识页面"""
|
||||
page = KnowledgePage(
|
||||
title=data.title,
|
||||
content=data.content,
|
||||
source_file=data.source_file,
|
||||
course_name=data.course_name,
|
||||
teacher_name=data.teacher_name,
|
||||
live_date=data.live_date,
|
||||
page_number=data.page_number,
|
||||
metadata_json=data.metadata_json,
|
||||
)
|
||||
self.db.add(page)
|
||||
await self.db.flush()
|
||||
await self.db.refresh(page)
|
||||
return page
|
||||
|
||||
async def get_page(self, page_id: int) -> Optional[KnowledgePage]:
|
||||
"""根据 ID 获取知识页面"""
|
||||
result = await self.db.execute(
|
||||
select(KnowledgePage).where(KnowledgePage.id == page_id)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def list_pages(
|
||||
self,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
course_name: Optional[str] = None,
|
||||
teacher_name: Optional[str] = None,
|
||||
keyword: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
分页查询知识页面列表。
|
||||
|
||||
Args:
|
||||
page: 页码(从 1 开始)
|
||||
page_size: 每页数量
|
||||
course_name: 按课程名称过滤
|
||||
teacher_name: 按讲师名称过滤
|
||||
keyword: 关键词搜索(标题或内容)
|
||||
|
||||
Returns:
|
||||
{"total": int, "page": int, "page_size": int, "items": list}
|
||||
"""
|
||||
query = select(KnowledgePage)
|
||||
count_query = select(func.count(KnowledgePage.id))
|
||||
|
||||
# 过滤条件
|
||||
if course_name:
|
||||
query = query.where(KnowledgePage.course_name == course_name)
|
||||
count_query = count_query.where(KnowledgePage.course_name == course_name)
|
||||
if teacher_name:
|
||||
query = query.where(KnowledgePage.teacher_name == teacher_name)
|
||||
count_query = count_query.where(KnowledgePage.teacher_name == teacher_name)
|
||||
if keyword:
|
||||
like_pattern = f"%{keyword}%"
|
||||
query = query.where(
|
||||
KnowledgePage.title.ilike(like_pattern)
|
||||
| KnowledgePage.content.ilike(like_pattern)
|
||||
)
|
||||
count_query = count_query.where(
|
||||
KnowledgePage.title.ilike(like_pattern)
|
||||
| KnowledgePage.content.ilike(like_pattern)
|
||||
)
|
||||
|
||||
# 总数
|
||||
total_result = await self.db.execute(count_query)
|
||||
total = total_result.scalar() or 0
|
||||
|
||||
# 分页
|
||||
offset = (page - 1) * page_size
|
||||
query = query.order_by(KnowledgePage.created_at.desc()).offset(offset).limit(page_size)
|
||||
result = await self.db.execute(query)
|
||||
items = result.scalars().all()
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"items": items,
|
||||
}
|
||||
|
||||
async def update_page(self, page_id: int, data: PageUpdate) -> Optional[KnowledgePage]:
|
||||
"""更新知识页面"""
|
||||
page = await self.get_page(page_id)
|
||||
if page is None:
|
||||
return None
|
||||
|
||||
update_data = data.model_dump(exclude_unset=True)
|
||||
for field, value in update_data.items():
|
||||
setattr(page, field, value)
|
||||
|
||||
await self.db.flush()
|
||||
await self.db.refresh(page)
|
||||
return page
|
||||
|
||||
async def delete_page(self, page_id: int) -> bool:
|
||||
"""删除知识页面及其关联分块"""
|
||||
page = await self.get_page(page_id)
|
||||
if page is None:
|
||||
return False
|
||||
|
||||
# 先删除关联的分块
|
||||
await self.db.execute(
|
||||
text("DELETE FROM knowledge_chunks WHERE page_id = :page_id"),
|
||||
{"page_id": page_id},
|
||||
)
|
||||
# 再删除页面
|
||||
await self.db.delete(page)
|
||||
await self.db.flush()
|
||||
return True
|
||||
|
||||
async def get_page_chunks(self, page_id: int) -> List[KnowledgeChunk]:
|
||||
"""获取知识页面的所有分块"""
|
||||
result = await self.db.execute(
|
||||
select(KnowledgeChunk)
|
||||
.where(KnowledgeChunk.page_id == page_id)
|
||||
.order_by(KnowledgeChunk.chunk_index)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def reindex_page(self, page_id: int) -> dict:
|
||||
"""
|
||||
重新索引知识页面:重新分块并生成嵌入向量。
|
||||
用于页面内容更新后刷新向量索引。
|
||||
"""
|
||||
page = await self.get_page(page_id)
|
||||
if page is None:
|
||||
raise ValueError(f"知识页面不存在: {page_id}")
|
||||
|
||||
# 删除旧分块
|
||||
await self.db.execute(
|
||||
text("DELETE FROM knowledge_chunks WHERE page_id = :page_id"),
|
||||
{"page_id": page_id},
|
||||
)
|
||||
|
||||
# 重新分块
|
||||
from app.services.import_service import ImportService
|
||||
chunks = ImportService._chunk_text(page.content)
|
||||
|
||||
if not chunks:
|
||||
await self.db.flush()
|
||||
return {"page_id": page_id, "chunks": 0}
|
||||
|
||||
# 生成嵌入并插入
|
||||
from app.services.embedding_service import EmbeddingService
|
||||
try:
|
||||
embeddings = await EmbeddingService.embed_batch(chunks)
|
||||
except Exception as exc:
|
||||
logger.error("重新索引嵌入失败: %s", exc)
|
||||
embeddings = [None] * len(chunks)
|
||||
|
||||
for idx, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):
|
||||
embedding_str = None
|
||||
if embedding:
|
||||
embedding_str = "[" + ",".join(str(x) for x in embedding) + "]"
|
||||
|
||||
await self.db.execute(text("""
|
||||
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
|
||||
VALUES (:page_id, :chunk_index, :content, :embedding::vector)
|
||||
"""), {
|
||||
"page_id": page_id,
|
||||
"chunk_index": idx,
|
||||
"content": chunk_text,
|
||||
"embedding": embedding_str,
|
||||
})
|
||||
|
||||
# 更新全文搜索向量
|
||||
await self.db.execute(text("""
|
||||
UPDATE knowledge_chunks
|
||||
SET search_vector = to_tsvector('chinese_zh', content)
|
||||
WHERE page_id = :page_id
|
||||
"""), {"page_id": page_id})
|
||||
|
||||
await self.db.flush()
|
||||
return {"page_id": page_id, "chunks": len(chunks)}
|
||||
185
app/services/search_engine.py
Normal file
185
app/services/search_engine.py
Normal file
@@ -0,0 +1,185 @@
|
||||
"""
|
||||
BM25 倒排索引搜索引擎
|
||||
用于图片 OCR 内容的全文搜索
|
||||
参考 Meilisearch / Whoosh 设计
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import math
|
||||
import logging
|
||||
from collections import Counter, defaultdict
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 尝试导入 jieba,失败则用简单的字符分割
|
||||
try:
|
||||
import jieba
|
||||
jieba.setLogLevel(logging.WARNING)
|
||||
def tokenize(text: str) -> List[str]:
|
||||
"""中文分词"""
|
||||
return [w for w in jieba.cut_for_search(text) if len(w.strip()) > 1]
|
||||
except ImportError:
|
||||
logger.warning("jieba 未安装,使用简单分词(pip install jieba 获得更好效果)")
|
||||
def tokenize(text: str) -> List[str]:
|
||||
"""简单分词:按空格和标点分割"""
|
||||
return [w for w in re.split(r'[\s,.\-;:!?"\'/\\|()\[\]{}<>,。;:!?""''、()【】]+', text) if len(w.strip()) > 1]
|
||||
|
||||
|
||||
class BM25Index:
|
||||
"""
|
||||
BM25 倒排索引
|
||||
- 支持增量添加文档
|
||||
- 支持删除文档
|
||||
- 支持中文分词(jieba)
|
||||
- BM25 评分排序
|
||||
"""
|
||||
|
||||
def __init__(self, k1: float = 1.5, b: float = 0.75):
|
||||
"""
|
||||
Args:
|
||||
k1: 词频饱和参数(默认 1.5)
|
||||
b: 文档长度归一化参数(默认 0.75)
|
||||
"""
|
||||
self.k1 = k1
|
||||
self.b = b
|
||||
|
||||
# 倒排索引:term -> {doc_id: tf}
|
||||
self.inverted_index: Dict[str, Dict[int, int]] = defaultdict(lambda: defaultdict(int))
|
||||
|
||||
# 文档信息
|
||||
self.doc_lengths: Dict[int, int] = {} # doc_id -> 文档长度(词数)
|
||||
self.doc_count: int = 0
|
||||
self.avg_doc_length: float = 0.0
|
||||
|
||||
# 文档存储(存完整数据)
|
||||
self.documents: Dict[int, dict] = {}
|
||||
|
||||
# IDF 缓存
|
||||
self._idf_cache: Dict[str, float] = {}
|
||||
|
||||
def add_document(self, doc_id: int, text: str, metadata: Optional[dict] = None):
|
||||
"""
|
||||
添加文档到索引
|
||||
|
||||
Args:
|
||||
doc_id: 文档 ID
|
||||
text: 待索引的文本内容
|
||||
metadata: 附加元数据(如 file_path, tags 等)
|
||||
"""
|
||||
# 如果已存在,先删除
|
||||
if doc_id in self.documents:
|
||||
self.remove_document(doc_id)
|
||||
|
||||
tokens = tokenize(text)
|
||||
self.doc_lengths[doc_id] = len(tokens)
|
||||
self.documents[doc_id] = {
|
||||
"text": text,
|
||||
"tokens": tokens,
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
|
||||
# 构建倒排索引
|
||||
tf = Counter(tokens)
|
||||
for term, count in tf.items():
|
||||
self.inverted_index[term][doc_id] = count
|
||||
|
||||
self.doc_count += 1
|
||||
self._update_avg_doc_length()
|
||||
self._idf_cache.clear() # 文档数变了,IDF 需要重算
|
||||
|
||||
def remove_document(self, doc_id: int):
|
||||
"""从索引中删除文档"""
|
||||
if doc_id not in self.documents:
|
||||
return
|
||||
|
||||
# 从倒排索引中移除
|
||||
tokens = self.documents[doc_id].get("tokens", [])
|
||||
for term in set(tokens):
|
||||
if term in self.inverted_index and doc_id in self.inverted_index[term]:
|
||||
del self.inverted_index[term][doc_id]
|
||||
if not self.inverted_index[term]:
|
||||
del self.inverted_index[term]
|
||||
|
||||
del self.documents[doc_id]
|
||||
del self.doc_lengths[doc_id]
|
||||
self.doc_count -= 1
|
||||
self._update_avg_doc_length()
|
||||
self._idf_cache.clear()
|
||||
|
||||
def _update_avg_doc_length(self):
|
||||
"""更新平均文档长度"""
|
||||
if self.doc_count > 0:
|
||||
self.avg_doc_length = sum(self.doc_lengths.values()) / self.doc_count
|
||||
else:
|
||||
self.avg_doc_length = 0.0
|
||||
|
||||
def _idf(self, term: str) -> float:
|
||||
"""
|
||||
计算逆文档频率 IDF
|
||||
IDF(t) = ln((N - df(t) + 0.5) / (df(t) + 0.5) + 1)
|
||||
其中 N 是文档总数,df(t) 是包含 term 的文档数
|
||||
"""
|
||||
if term in self._idf_cache:
|
||||
return self._idf_cache[term]
|
||||
|
||||
df = len(self.inverted_index.get(term, {}))
|
||||
idf = math.log((self.doc_count - df + 0.5) / (df + 0.5) + 1)
|
||||
|
||||
self._idf_cache[term] = idf
|
||||
return idf
|
||||
|
||||
def search(self, query: str, top_k: int = 10) -> List[Tuple[int, float]]:
|
||||
"""
|
||||
BM25 搜索
|
||||
|
||||
Args:
|
||||
query: 搜索查询(会自动分词)
|
||||
top_k: 返回前 K 个结果
|
||||
|
||||
Returns:
|
||||
[(doc_id, score), ...] 按 BM25 分数降序排列
|
||||
"""
|
||||
if self.doc_count == 0 or self.avg_doc_length == 0:
|
||||
return []
|
||||
|
||||
query_tokens = tokenize(query)
|
||||
if not query_tokens:
|
||||
return []
|
||||
|
||||
scores: Dict[int, float] = defaultdict(float)
|
||||
|
||||
for term in query_tokens:
|
||||
idf = self._idf(term)
|
||||
postings = self.inverted_index.get(term, {})
|
||||
|
||||
for doc_id, tf in postings.items():
|
||||
dl = self.doc_lengths[doc_id]
|
||||
# BM25 公式
|
||||
numerator = tf * (self.k1 + 1)
|
||||
denominator = tf + self.k1 * (1 - self.b + self.b * dl / self.avg_doc_length)
|
||||
scores[doc_id] += idf * numerator / denominator
|
||||
|
||||
# 按分数降序排序
|
||||
ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True)
|
||||
return ranked[:top_k]
|
||||
|
||||
def get_document(self, doc_id: int) -> Optional[dict]:
|
||||
"""获取文档完整数据"""
|
||||
return self.documents.get(doc_id)
|
||||
|
||||
def clear(self):
|
||||
"""清空索引"""
|
||||
self.inverted_index.clear()
|
||||
self.doc_lengths.clear()
|
||||
self.documents.clear()
|
||||
self._idf_cache.clear()
|
||||
self.doc_count = 0
|
||||
self.avg_doc_length = 0.0
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
"""索引中的文档数量"""
|
||||
return self.doc_count
|
||||
286
app/services/search_service.py
Normal file
286
app/services/search_service.py
Normal file
@@ -0,0 +1,286 @@
|
||||
"""
|
||||
语义搜索服务
|
||||
支持 PostgreSQL(向量+全文混合)和 SQLite(LIKE 关键词搜索)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.database import IS_SQLITE
|
||||
from app.schemas.search import SearchRequest, SearchResult, SearchResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SearchService:
|
||||
"""语义搜索服务"""
|
||||
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
async def search(self, request: SearchRequest) -> SearchResponse:
|
||||
"""
|
||||
执行搜索。
|
||||
- PostgreSQL: 向量搜索 + 中文全文搜索混合排序
|
||||
- SQLite: LIKE 关键词搜索(无向量能力)
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
if IS_SQLITE:
|
||||
return await self._search_sqlite(request, start_time)
|
||||
else:
|
||||
return await self._search_postgres(request, start_time)
|
||||
|
||||
# ──────────────────────────── SQLite 搜索 ────────────────────────────
|
||||
|
||||
async def _search_sqlite(self, request: SearchRequest, start_time: float) -> SearchResponse:
|
||||
"""SQLite 模式:使用 LIKE 关键词搜索"""
|
||||
# LLM 查询扩展(可选)
|
||||
expanded_queries = [request.query]
|
||||
try:
|
||||
from app.services.llm_service import LLMService
|
||||
expanded_queries = await LLMService.expand_query(request.query)
|
||||
except Exception as exc:
|
||||
logger.debug("查询扩展跳过: %s", exc)
|
||||
|
||||
results: list[SearchResult] = []
|
||||
seen_chunk_ids = set()
|
||||
|
||||
for q in expanded_queries:
|
||||
like_pattern = f"%{q}%"
|
||||
sql = text("""
|
||||
SELECT
|
||||
kc.id AS chunk_id,
|
||||
kp.id AS page_id,
|
||||
kp.title AS page_title,
|
||||
kc.content,
|
||||
kp.course_name,
|
||||
kp.teacher_name,
|
||||
kp.live_date
|
||||
FROM knowledge_chunks kc
|
||||
JOIN knowledge_pages kp ON kc.page_id = kp.id
|
||||
WHERE kc.content LIKE :pattern
|
||||
AND (:course IS NULL OR kp.course_name = :course)
|
||||
AND (:teacher IS NULL OR kp.teacher_name = :teacher)
|
||||
ORDER BY kc.id
|
||||
LIMIT :limit
|
||||
""")
|
||||
params = {
|
||||
"pattern": like_pattern,
|
||||
"course": request.course_name,
|
||||
"teacher": request.teacher_name,
|
||||
"limit": request.top_k,
|
||||
}
|
||||
result = await self.db.execute(sql, params)
|
||||
rows = result.fetchall()
|
||||
|
||||
for row in rows:
|
||||
if row.chunk_id not in seen_chunk_ids:
|
||||
seen_chunk_ids.add(row.chunk_id)
|
||||
# 简单的相关性评分:关键词出现次数
|
||||
score = min(1.0, row.content.lower().count(q.lower()) * 0.2 + 0.5)
|
||||
results.append(SearchResult(
|
||||
chunk_id=row.chunk_id,
|
||||
page_id=row.page_id,
|
||||
page_title=row.page_title,
|
||||
content=row.content,
|
||||
score=round(score, 4),
|
||||
course_name=row.course_name,
|
||||
teacher_name=row.teacher_name,
|
||||
live_date=row.live_date,
|
||||
highlight=None,
|
||||
))
|
||||
|
||||
results.sort(key=lambda x: x.score, reverse=True)
|
||||
results = results[:request.top_k]
|
||||
|
||||
elapsed_ms = (time.time() - start_time) * 1000
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
total=len(results),
|
||||
results=results,
|
||||
elapsed_ms=round(elapsed_ms, 2),
|
||||
)
|
||||
|
||||
# ──────────────────────────── PostgreSQL 搜索 ────────────────────────────
|
||||
|
||||
async def _search_postgres(self, request: SearchRequest, start_time: float) -> SearchResponse:
|
||||
"""PostgreSQL 模式:向量搜索 + 中文全文搜索混合"""
|
||||
# LLM 查询扩展
|
||||
expanded_queries = [request.query]
|
||||
if request.use_fulltext:
|
||||
try:
|
||||
from app.services.llm_service import LLMService
|
||||
expanded_queries = await LLMService.expand_query(request.query)
|
||||
except Exception as exc:
|
||||
logger.debug("查询扩展跳过: %s", exc)
|
||||
|
||||
# 生成查询向量
|
||||
from app.services.embedding_service import EmbeddingService
|
||||
all_query_embeddings = await EmbeddingService.embed_batch(expanded_queries)
|
||||
|
||||
# 构建过滤条件
|
||||
where_clauses = []
|
||||
params: dict = {}
|
||||
|
||||
if request.course_name:
|
||||
where_clauses.append("kp.course_name = :course_name")
|
||||
params["course_name"] = request.course_name
|
||||
if request.teacher_name:
|
||||
where_clauses.append("kp.teacher_name = :teacher_name")
|
||||
params["teacher_name"] = request.teacher_name
|
||||
if request.live_date_from:
|
||||
where_clauses.append("kp.live_date >= :live_date_from")
|
||||
params["live_date_from"] = request.live_date_from
|
||||
if request.live_date_to:
|
||||
where_clauses.append("kp.live_date <= :live_date_to")
|
||||
params["live_date_to"] = request.live_date_to
|
||||
|
||||
where_sql = ""
|
||||
if where_clauses:
|
||||
where_sql = "AND " + " AND ".join(where_clauses)
|
||||
|
||||
# 向量搜索
|
||||
vector_results = []
|
||||
params["limit"] = request.top_k * 2
|
||||
for q_embedding in all_query_embeddings:
|
||||
embedding_str = "[" + ",".join(str(x) for x in q_embedding) + "]"
|
||||
vector_sql = f"""
|
||||
SELECT
|
||||
kc.id AS chunk_id,
|
||||
kp.id AS page_id,
|
||||
kp.title AS page_title,
|
||||
kc.content,
|
||||
1 - (kc.embedding <=> :embedding::vector) AS vector_score,
|
||||
0.0 AS text_score,
|
||||
kp.course_name,
|
||||
kp.teacher_name,
|
||||
kp.live_date
|
||||
FROM knowledge_chunks kc
|
||||
JOIN knowledge_pages kp ON kc.page_id = kp.id
|
||||
WHERE kc.embedding IS NOT NULL
|
||||
{where_sql}
|
||||
ORDER BY kc.embedding <=> :embedding::vector
|
||||
LIMIT :limit
|
||||
"""
|
||||
v_params = {**params, "embedding": embedding_str}
|
||||
result = await self.db.execute(text(vector_sql), v_params)
|
||||
vector_results.extend(result.fetchall())
|
||||
|
||||
# 全文搜索
|
||||
text_results = []
|
||||
if request.use_fulltext:
|
||||
for q in expanded_queries:
|
||||
clean_query = q.replace("'", "''")
|
||||
fulltext_sql = f"""
|
||||
SELECT
|
||||
kc.id AS chunk_id,
|
||||
kp.id AS page_id,
|
||||
kp.title AS page_title,
|
||||
kc.content,
|
||||
0.0 AS vector_score,
|
||||
ts_rank_cd(kc.search_vector, to_tsquery('chinese_zh', :query)) AS text_score,
|
||||
kp.course_name,
|
||||
kp.teacher_name,
|
||||
kp.live_date,
|
||||
ts_headline('chinese_zh', kc.content, to_tsquery('chinese_zh', :query),
|
||||
'MaxWords=50, MinWords=20, ShortWord=2') AS highlight
|
||||
FROM knowledge_chunks kc
|
||||
JOIN knowledge_pages kp ON kc.page_id = kp.id
|
||||
WHERE kc.search_vector @@ to_tsquery('chinese_zh', :query)
|
||||
{where_sql}
|
||||
ORDER BY text_score DESC
|
||||
LIMIT :limit
|
||||
"""
|
||||
ft_params = {**params, "query": clean_query}
|
||||
ft_result = await self.db.execute(text(fulltext_sql), ft_params)
|
||||
text_results.extend(ft_result.fetchall())
|
||||
|
||||
# 合并结果
|
||||
merged: dict[int, dict] = {}
|
||||
|
||||
for row in vector_results:
|
||||
chunk_id = row.chunk_id
|
||||
if chunk_id not in merged:
|
||||
merged[chunk_id] = {
|
||||
"chunk_id": chunk_id,
|
||||
"page_id": row.page_id,
|
||||
"page_title": row.page_title,
|
||||
"content": row.content,
|
||||
"vector_score": row.vector_score,
|
||||
"text_score": 0.0,
|
||||
"course_name": row.course_name,
|
||||
"teacher_name": row.teacher_name,
|
||||
"live_date": row.live_date,
|
||||
"highlight": None,
|
||||
}
|
||||
else:
|
||||
merged[chunk_id]["vector_score"] = max(
|
||||
merged[chunk_id]["vector_score"], row.vector_score
|
||||
)
|
||||
|
||||
for row in text_results:
|
||||
chunk_id = row.chunk_id
|
||||
if chunk_id not in merged:
|
||||
merged[chunk_id] = {
|
||||
"chunk_id": chunk_id,
|
||||
"page_id": row.page_id,
|
||||
"page_title": row.page_title,
|
||||
"content": row.content,
|
||||
"vector_score": 0.0,
|
||||
"text_score": row.text_score or 0.0,
|
||||
"course_name": row.course_name,
|
||||
"teacher_name": row.teacher_name,
|
||||
"live_date": row.live_date,
|
||||
"highlight": row.highlight,
|
||||
}
|
||||
else:
|
||||
merged[chunk_id]["text_score"] = max(
|
||||
merged[chunk_id]["text_score"], row.text_score or 0.0
|
||||
)
|
||||
if row.highlight:
|
||||
merged[chunk_id]["highlight"] = row.highlight
|
||||
|
||||
# 计算综合得分
|
||||
max_text_score = max(
|
||||
(r["text_score"] for r in merged.values()), default=1.0
|
||||
) or 1.0
|
||||
|
||||
scored_results = []
|
||||
for item in merged.values():
|
||||
normalized_text = item["text_score"] / max_text_score if max_text_score > 0 else 0
|
||||
combined_score = 0.7 * item["vector_score"] + 0.3 * normalized_text
|
||||
|
||||
if combined_score >= request.threshold:
|
||||
scored_results.append(
|
||||
SearchResult(
|
||||
chunk_id=item["chunk_id"],
|
||||
page_id=item["page_id"],
|
||||
page_title=item["page_title"],
|
||||
content=item["content"],
|
||||
score=round(combined_score, 4),
|
||||
course_name=item["course_name"],
|
||||
teacher_name=item["teacher_name"],
|
||||
live_date=item["live_date"],
|
||||
highlight=item["highlight"],
|
||||
)
|
||||
)
|
||||
|
||||
scored_results.sort(key=lambda x: x.score, reverse=True)
|
||||
scored_results = scored_results[:request.top_k]
|
||||
|
||||
elapsed_ms = (time.time() - start_time) * 1000
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
total=len(scored_results),
|
||||
results=scored_results,
|
||||
elapsed_ms=round(elapsed_ms, 2),
|
||||
)
|
||||
Reference in New Issue
Block a user