refactor: 移除 PostgreSQL 支持,简化为纯 SQLite 部署
- config.py: DATABASE_URL 默认值改为 SQLite - database.py: 移除 PostgreSQL 分支,简化为纯 SQLite - models/base.py: 移除 pgvector 导入和条件分支 - search_service.py: 移除 _search_postgres 方法 - import_service.py: 移除 pgvector 相关代码 - requirements.txt: 移除 asyncpg/alembic/pgvector 依赖 - pyproject.toml: 同步移除相关依赖 - docker-compose.yml: 移除 db 服务,- 删除 alembic.ini/Dockerfile.db/sql 目录 - README.md: 更新文档,移除 PostgreSQL 相关内容 适合 NAS 等资源受限环境的轻量级部署
This commit is contained in:
@@ -46,8 +46,8 @@ class Settings(BaseSettings):
|
||||
|
||||
# ──────────────────────────── 数据库 ────────────────────────────
|
||||
DATABASE_URL: str = Field(
|
||||
default="postgresql+asyncpg://postgres:postgres@db:5432/edu_brain",
|
||||
description="PostgreSQL 异步连接字符串",
|
||||
default="sqlite+aiosqlite:///./edu_brain.db",
|
||||
description="SQLite 数据库连接字符串",
|
||||
)
|
||||
|
||||
# ──────────────────────────── 嵌入模型 ────────────────────────────
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
"""
|
||||
数据库连接模块
|
||||
支持 PostgreSQL(生产)和 SQLite(本地开发)两种后端
|
||||
数据库连接管理
|
||||
|
||||
SQLite 数据库,支持异步操作。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
from pathlib import Path
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
@@ -14,35 +15,25 @@ from sqlalchemy.ext.asyncio import (
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.config import settings
|
||||
|
||||
# ──────────────────────────── 判断数据库类型 ────────────────────────────
|
||||
# ──────────────────────────── 数据库引擎 ────────────────────────────
|
||||
|
||||
DB_URL = settings.DATABASE_URL
|
||||
IS_SQLITE = DB_URL.startswith("sqlite")
|
||||
|
||||
# ──────────────────────────── 异步引擎 ────────────────────────────
|
||||
# 解析 SQLite 文件路径
|
||||
sqlite_path = DB_URL.replace("sqlite+aiosqlite:///", "")
|
||||
if not sqlite_path.startswith("/"):
|
||||
sqlite_path = str(Path(__file__).parent.parent / sqlite_path)
|
||||
|
||||
if IS_SQLITE:
|
||||
# SQLite 配置
|
||||
sqlite_path = DB_URL.replace("sqlite+aiosqlite:///", "")
|
||||
if not sqlite_path.startswith("/"):
|
||||
sqlite_path = str(Path(__file__).parent.parent / sqlite_path)
|
||||
# 确保数据目录存在
|
||||
Path(sqlite_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
engine = create_async_engine(
|
||||
f"sqlite+aiosqlite:///{sqlite_path}",
|
||||
echo=False,
|
||||
)
|
||||
else:
|
||||
# PostgreSQL 配置
|
||||
engine = create_async_engine(
|
||||
DB_URL,
|
||||
echo=False,
|
||||
poolclass=NullPool,
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
engine = create_async_engine(
|
||||
f"sqlite+aiosqlite:///{sqlite_path}",
|
||||
echo=False,
|
||||
)
|
||||
|
||||
# ──────────────────────────── 会话工厂 ────────────────────────────
|
||||
|
||||
@@ -61,47 +52,21 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
FastAPI 依赖项:获取异步数据库会话。
|
||||
"""
|
||||
async with async_session_factory() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
yield session
|
||||
|
||||
|
||||
# ──────────────────────────── 生命周期辅助 ────────────────────────────
|
||||
async def close_db() -> None:
|
||||
"""关闭数据库连接池"""
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
"""初始化数据库:建表"""
|
||||
from app.models.base import Base
|
||||
|
||||
async with engine.begin() as conn:
|
||||
if IS_SQLITE:
|
||||
# SQLite 启用 WAL 模式和外键约束
|
||||
await conn.execute(text("PRAGMA journal_mode=WAL"))
|
||||
await conn.execute(text("PRAGMA foreign_keys=ON"))
|
||||
# SQLite 启用 WAL 模式和外键约束
|
||||
await conn.execute(text("PRAGMA journal_mode=WAL"))
|
||||
await conn.execute(text("PRAGMA foreign_keys=ON"))
|
||||
# 创建所有表
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
if not IS_SQLITE:
|
||||
# PostgreSQL 需要创建扩展
|
||||
async with engine.begin() as conn:
|
||||
try:
|
||||
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS zhparser"))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await conn.execute(text(
|
||||
"CREATE TEXT SEARCH CONFIGURATION IF NOT EXISTS chinese_zh (PARSER = zhparser)"
|
||||
))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def close_db() -> None:
|
||||
"""关闭数据库引擎"""
|
||||
await engine.dispose()
|
||||
|
||||
@@ -1,26 +1,18 @@
|
||||
"""
|
||||
SQLAlchemy 基础模型 + 通用 Mixin
|
||||
提供 DeclarativeBase、时间戳 Mixin 和公共序列化方法
|
||||
兼容 PostgreSQL(pgvector)和 SQLite(本地开发)
|
||||
ORM 模型定义
|
||||
|
||||
SQLite 数据库模型,使用 Text 存储 JSON 格式的向量。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict
|
||||
|
||||
from sqlalchemy import DateTime, Float, Integer, String, Text, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
from app.database import IS_SQLITE
|
||||
|
||||
# SQLite 下不使用 pgvector,用 Text 存储向量(JSON 格式)
|
||||
if IS_SQLITE:
|
||||
VectorType = Text
|
||||
else:
|
||||
from pgvector.sqlalchemy import Vector
|
||||
VectorType = Vector
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
"""所有 ORM 模型的基类"""
|
||||
@@ -28,96 +20,83 @@ class Base(DeclarativeBase):
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
"""
|
||||
时间戳 Mixin,为模型自动添加 created_at / updated_at 字段。
|
||||
"""
|
||||
"""时间戳 Mixin"""
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
comment="创建时间",
|
||||
DateTime, default=func.now(), comment="创建时间"
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
comment="更新时间",
|
||||
DateTime, default=func.now(), onupdate=func.now(), comment="更新时间"
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""将模型实例转换为字典"""
|
||||
result: Dict[str, Any] = {}
|
||||
for column in self.__table__.columns: # type: ignore[attr-defined]
|
||||
value = getattr(self, column.name, None)
|
||||
if isinstance(value, datetime):
|
||||
value = value.isoformat()
|
||||
elif hasattr(value, "tolist"):
|
||||
value = value.tolist()
|
||||
result[column.name] = value
|
||||
return result
|
||||
|
||||
|
||||
# ──────────────────────────── 业务模型 ────────────────────────────
|
||||
|
||||
class KnowledgePage(Base, TimestampMixin):
|
||||
"""知识页面模型"""
|
||||
"""知识页面"""
|
||||
__tablename__ = "knowledge_pages"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
title: Mapped[str] = mapped_column(String(500), nullable=False, comment="页面标题")
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False, comment="页面正文内容")
|
||||
source_file: Mapped[str | None] = mapped_column(String(500), nullable=True, comment="来源文件名")
|
||||
course_name: Mapped[str | None] = mapped_column(String(200), nullable=True, comment="课程名称")
|
||||
teacher_name: Mapped[str | None] = mapped_column(String(100), nullable=True, comment="讲师名称")
|
||||
live_date: Mapped[str | None] = mapped_column(String(20), nullable=True, comment="直播日期")
|
||||
page_number: Mapped[int | None] = mapped_column(Integer, nullable=True, comment="原始页码")
|
||||
metadata_json: Mapped[str | None] = mapped_column(Text, nullable=True, comment="额外元数据")
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False, comment="页面内容")
|
||||
source: Mapped[str | None] = mapped_column(String(200), nullable=True, comment="来源文件名")
|
||||
page_number: Mapped[int | None] = mapped_column(Integer, nullable=True, comment="原文件页码")
|
||||
# 向量嵌入(JSON 格式存储)
|
||||
embedding: Mapped[str | None] = mapped_column(Text, nullable=True, comment="向量嵌入(JSON)")
|
||||
|
||||
@property
|
||||
def embedding_vector(self) -> list[float] | None:
|
||||
if self.embedding is None:
|
||||
return None
|
||||
return json.loads(self.embedding)
|
||||
|
||||
class KnowledgeChunk(Base, TimestampMixin):
|
||||
"""知识分块模型"""
|
||||
__tablename__ = "knowledge_chunks"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
page_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True, comment="关联页面 ID")
|
||||
chunk_index: Mapped[int] = mapped_column(Integer, nullable=False, comment="分块序号")
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False, comment="分块文本内容")
|
||||
embedding: Mapped[list | None] = mapped_column(
|
||||
VectorType(768) if not IS_SQLITE else Text,
|
||||
nullable=True,
|
||||
comment="嵌入向量",
|
||||
)
|
||||
search_vector: Mapped[Any | None] = mapped_column(
|
||||
Text, nullable=True, comment="全文搜索向量(仅 PG)",
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
result.pop("embedding", None)
|
||||
result.pop("search_vector", None)
|
||||
return result
|
||||
@embedding_vector.setter
|
||||
def embedding_vector(self, value: list[float] | None):
|
||||
self.embedding = json.dumps(value) if value else None
|
||||
|
||||
|
||||
class OCRImage(Base, TimestampMixin):
|
||||
"""OCR 图片记录模型"""
|
||||
"""OCR 图片记录"""
|
||||
__tablename__ = "ocr_images"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
file_path: Mapped[str] = mapped_column(String(500), nullable=False, comment="图片文件路径")
|
||||
ocr_text: Mapped[str | None] = mapped_column(Text, nullable=True, comment="OCR 识别全文")
|
||||
confidence: Mapped[float | None] = mapped_column(Float, nullable=True, comment="平均置信度")
|
||||
original_filename: Mapped[str | None] = mapped_column(String(200), nullable=True, comment="原始文件名")
|
||||
ocr_text: Mapped[str | None] = mapped_column(Text, nullable=True, comment="OCR 识别文本")
|
||||
confidence: Mapped[float | None] = mapped_column(Float, nullable=True, comment="OCR 置信度")
|
||||
provider: Mapped[str | None] = mapped_column(String(50), nullable=True, comment="OCR 提供商")
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending", comment="处理状态")
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True, comment="错误信息")
|
||||
tags: Mapped[str | None] = mapped_column(Text, nullable=True, comment="LLM 提取的标签,JSON 数组格式")
|
||||
story_summary: Mapped[str | None] = mapped_column(Text, nullable=True, comment="Qwen3-8B 提炼的故事摘要")
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(20), default="pending", comment="状态: pending/processing/completed/failed"
|
||||
)
|
||||
tags: Mapped[str | None] = mapped_column(Text, nullable=True, comment="关键词标签(JSON 数组)")
|
||||
story_summary: Mapped[str | None] = mapped_column(Text, nullable=True, comment="故事摘要")
|
||||
blocks: Mapped[str | None] = mapped_column(Text, nullable=True, comment="OCR 文本块(JSON)")
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
# 将 tags 从 JSON 字符串解析为列表返回
|
||||
if result.get("tags") and isinstance(result["tags"], str):
|
||||
try:
|
||||
import json
|
||||
result["tags"] = json.loads(result["tags"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return result
|
||||
@property
|
||||
def tags_list(self) -> list[str]:
|
||||
if self.tags is None:
|
||||
return []
|
||||
return json.loads(self.tags)
|
||||
|
||||
@tags_list.setter
|
||||
def tags_list(self, value: list[str]):
|
||||
self.tags = json.dumps(value) if value else None
|
||||
|
||||
@property
|
||||
def blocks_list(self) -> list[dict]:
|
||||
if self.blocks is None:
|
||||
return []
|
||||
return json.loads(self.blocks)
|
||||
|
||||
@blocks_list.setter
|
||||
def blocks_list(self, value: list[dict]):
|
||||
self.blocks = json.dumps(value) if value else None
|
||||
|
||||
|
||||
class WebsiteSettings(Base):
|
||||
"""网站设置(单例)"""
|
||||
__tablename__ = "website_settings"
|
||||
|
||||
key: Mapped[str] = mapped_column(String(100), primary_key=True, comment="设置键")
|
||||
value: Mapped[str | None] = mapped_column(Text, nullable=True, comment="设置值(JSON)")
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, default=func.now(), onupdate=func.now(), comment="更新时间"
|
||||
)
|
||||
|
||||
@@ -14,7 +14,6 @@ from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.database import IS_SQLITE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -343,21 +342,13 @@ class ImportService:
|
||||
|
||||
# 批量插入
|
||||
for idx, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):
|
||||
if IS_SQLITE:
|
||||
# SQLite: 向量存为 JSON 文本
|
||||
import json as _json
|
||||
embedding_str = _json.dumps(embedding) if embedding else None
|
||||
sql = text("""
|
||||
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
|
||||
VALUES (:page_id, :chunk_index, :content, :embedding)
|
||||
""")
|
||||
else:
|
||||
# PostgreSQL: 使用 pgvector 类型
|
||||
embedding_str = "[" + ",".join(str(x) for x in embedding) + "]" if embedding else None
|
||||
sql = text("""
|
||||
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
|
||||
VALUES (:page_id, :chunk_index, :content, :embedding::vector)
|
||||
""")
|
||||
# SQLite: 向量存为 JSON 文本
|
||||
import json as _json
|
||||
embedding_str = _json.dumps(embedding) if embedding else None
|
||||
sql = text("""
|
||||
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
|
||||
VALUES (:page_id, :chunk_index, :content, :embedding)
|
||||
""")
|
||||
await self.db.execute(sql, {
|
||||
"page_id": page_id,
|
||||
"chunk_index": idx,
|
||||
@@ -365,14 +356,6 @@ class ImportService:
|
||||
"embedding": embedding_str,
|
||||
})
|
||||
|
||||
# 更新全文搜索向量(仅 PostgreSQL)
|
||||
if not IS_SQLITE:
|
||||
await self.db.execute(text("""
|
||||
UPDATE knowledge_chunks
|
||||
SET search_vector = to_tsvector('chinese_zh', content)
|
||||
WHERE page_id = :page_id
|
||||
"""), {"page_id": page_id})
|
||||
|
||||
await self.db.flush()
|
||||
return len(chunks)
|
||||
|
||||
|
||||
@@ -1,20 +1,18 @@
|
||||
"""
|
||||
语义搜索服务
|
||||
支持 PostgreSQL(向量+全文混合)和 SQLite(LIKE 关键词搜索)
|
||||
|
||||
SQLite 数据库,使用 LIKE 关键词搜索。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import List, Optional
|
||||
from typing import List
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.database import IS_SQLITE
|
||||
from app.schemas.search import SearchRequest, SearchResult, SearchResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -28,21 +26,10 @@ class SearchService:
|
||||
|
||||
async def search(self, request: SearchRequest) -> SearchResponse:
|
||||
"""
|
||||
执行搜索。
|
||||
- PostgreSQL: 向量搜索 + 中文全文搜索混合排序
|
||||
- SQLite: LIKE 关键词搜索(无向量能力)
|
||||
执行搜索:使用 LIKE 关键词搜索
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
if IS_SQLITE:
|
||||
return await self._search_sqlite(request, start_time)
|
||||
else:
|
||||
return await self._search_postgres(request, start_time)
|
||||
|
||||
# ──────────────────────────── SQLite 搜索 ────────────────────────────
|
||||
|
||||
async def _search_sqlite(self, request: SearchRequest, start_time: float) -> SearchResponse:
|
||||
"""SQLite 模式:使用 LIKE 关键词搜索"""
|
||||
# LLM 查询扩展(可选)
|
||||
expanded_queries = [request.query]
|
||||
try:
|
||||
@@ -109,178 +96,3 @@ class SearchService:
|
||||
results=results,
|
||||
elapsed_ms=round(elapsed_ms, 2),
|
||||
)
|
||||
|
||||
# ──────────────────────────── PostgreSQL 搜索 ────────────────────────────
|
||||
|
||||
async def _search_postgres(self, request: SearchRequest, start_time: float) -> SearchResponse:
|
||||
"""PostgreSQL 模式:向量搜索 + 中文全文搜索混合"""
|
||||
# LLM 查询扩展
|
||||
expanded_queries = [request.query]
|
||||
if request.use_fulltext:
|
||||
try:
|
||||
from app.services.llm_service import LLMService
|
||||
expanded_queries = await LLMService.expand_query(request.query)
|
||||
except Exception as exc:
|
||||
logger.debug("查询扩展跳过: %s", exc)
|
||||
|
||||
# 生成查询向量
|
||||
from app.services.embedding_service import EmbeddingService
|
||||
all_query_embeddings = await EmbeddingService.embed_batch(expanded_queries)
|
||||
|
||||
# 构建过滤条件
|
||||
where_clauses = []
|
||||
params: dict = {}
|
||||
|
||||
if request.course_name:
|
||||
where_clauses.append("kp.course_name = :course_name")
|
||||
params["course_name"] = request.course_name
|
||||
if request.teacher_name:
|
||||
where_clauses.append("kp.teacher_name = :teacher_name")
|
||||
params["teacher_name"] = request.teacher_name
|
||||
if request.live_date_from:
|
||||
where_clauses.append("kp.live_date >= :live_date_from")
|
||||
params["live_date_from"] = request.live_date_from
|
||||
if request.live_date_to:
|
||||
where_clauses.append("kp.live_date <= :live_date_to")
|
||||
params["live_date_to"] = request.live_date_to
|
||||
|
||||
where_sql = ""
|
||||
if where_clauses:
|
||||
where_sql = "AND " + " AND ".join(where_clauses)
|
||||
|
||||
# 向量搜索
|
||||
vector_results = []
|
||||
params["limit"] = request.top_k * 2
|
||||
for q_embedding in all_query_embeddings:
|
||||
embedding_str = "[" + ",".join(str(x) for x in q_embedding) + "]"
|
||||
vector_sql = f"""
|
||||
SELECT
|
||||
kc.id AS chunk_id,
|
||||
kp.id AS page_id,
|
||||
kp.title AS page_title,
|
||||
kc.content,
|
||||
1 - (kc.embedding <=> :embedding::vector) AS vector_score,
|
||||
0.0 AS text_score,
|
||||
kp.course_name,
|
||||
kp.teacher_name,
|
||||
kp.live_date
|
||||
FROM knowledge_chunks kc
|
||||
JOIN knowledge_pages kp ON kc.page_id = kp.id
|
||||
WHERE kc.embedding IS NOT NULL
|
||||
{where_sql}
|
||||
ORDER BY kc.embedding <=> :embedding::vector
|
||||
LIMIT :limit
|
||||
"""
|
||||
v_params = {**params, "embedding": embedding_str}
|
||||
result = await self.db.execute(text(vector_sql), v_params)
|
||||
vector_results.extend(result.fetchall())
|
||||
|
||||
# 全文搜索
|
||||
text_results = []
|
||||
if request.use_fulltext:
|
||||
for q in expanded_queries:
|
||||
clean_query = q.replace("'", "''")
|
||||
fulltext_sql = f"""
|
||||
SELECT
|
||||
kc.id AS chunk_id,
|
||||
kp.id AS page_id,
|
||||
kp.title AS page_title,
|
||||
kc.content,
|
||||
0.0 AS vector_score,
|
||||
ts_rank_cd(kc.search_vector, to_tsquery('chinese_zh', :query)) AS text_score,
|
||||
kp.course_name,
|
||||
kp.teacher_name,
|
||||
kp.live_date,
|
||||
ts_headline('chinese_zh', kc.content, to_tsquery('chinese_zh', :query),
|
||||
'MaxWords=50, MinWords=20, ShortWord=2') AS highlight
|
||||
FROM knowledge_chunks kc
|
||||
JOIN knowledge_pages kp ON kc.page_id = kp.id
|
||||
WHERE kc.search_vector @@ to_tsquery('chinese_zh', :query)
|
||||
{where_sql}
|
||||
ORDER BY text_score DESC
|
||||
LIMIT :limit
|
||||
"""
|
||||
ft_params = {**params, "query": clean_query}
|
||||
ft_result = await self.db.execute(text(fulltext_sql), ft_params)
|
||||
text_results.extend(ft_result.fetchall())
|
||||
|
||||
# 合并结果
|
||||
merged: dict[int, dict] = {}
|
||||
|
||||
for row in vector_results:
|
||||
chunk_id = row.chunk_id
|
||||
if chunk_id not in merged:
|
||||
merged[chunk_id] = {
|
||||
"chunk_id": chunk_id,
|
||||
"page_id": row.page_id,
|
||||
"page_title": row.page_title,
|
||||
"content": row.content,
|
||||
"vector_score": row.vector_score,
|
||||
"text_score": 0.0,
|
||||
"course_name": row.course_name,
|
||||
"teacher_name": row.teacher_name,
|
||||
"live_date": row.live_date,
|
||||
"highlight": None,
|
||||
}
|
||||
else:
|
||||
merged[chunk_id]["vector_score"] = max(
|
||||
merged[chunk_id]["vector_score"], row.vector_score
|
||||
)
|
||||
|
||||
for row in text_results:
|
||||
chunk_id = row.chunk_id
|
||||
if chunk_id not in merged:
|
||||
merged[chunk_id] = {
|
||||
"chunk_id": chunk_id,
|
||||
"page_id": row.page_id,
|
||||
"page_title": row.page_title,
|
||||
"content": row.content,
|
||||
"vector_score": 0.0,
|
||||
"text_score": row.text_score or 0.0,
|
||||
"course_name": row.course_name,
|
||||
"teacher_name": row.teacher_name,
|
||||
"live_date": row.live_date,
|
||||
"highlight": row.highlight,
|
||||
}
|
||||
else:
|
||||
merged[chunk_id]["text_score"] = max(
|
||||
merged[chunk_id]["text_score"], row.text_score or 0.0
|
||||
)
|
||||
if row.highlight:
|
||||
merged[chunk_id]["highlight"] = row.highlight
|
||||
|
||||
# 计算综合得分
|
||||
max_text_score = max(
|
||||
(r["text_score"] for r in merged.values()), default=1.0
|
||||
) or 1.0
|
||||
|
||||
scored_results = []
|
||||
for item in merged.values():
|
||||
normalized_text = item["text_score"] / max_text_score if max_text_score > 0 else 0
|
||||
combined_score = 0.7 * item["vector_score"] + 0.3 * normalized_text
|
||||
|
||||
if combined_score >= request.threshold:
|
||||
scored_results.append(
|
||||
SearchResult(
|
||||
chunk_id=item["chunk_id"],
|
||||
page_id=item["page_id"],
|
||||
page_title=item["page_title"],
|
||||
content=item["content"],
|
||||
score=round(combined_score, 4),
|
||||
course_name=item["course_name"],
|
||||
teacher_name=item["teacher_name"],
|
||||
live_date=item["live_date"],
|
||||
highlight=item["highlight"],
|
||||
)
|
||||
)
|
||||
|
||||
scored_results.sort(key=lambda x: x.score, reverse=True)
|
||||
scored_results = scored_results[:request.top_k]
|
||||
|
||||
elapsed_ms = (time.time() - start_time) * 1000
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
total=len(scored_results),
|
||||
results=scored_results,
|
||||
elapsed_ms=round(elapsed_ms, 2),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user