refactor: 移除 PostgreSQL 支持,简化为纯 SQLite 部署

- config.py: DATABASE_URL 默认值改为 SQLite
- database.py: 移除 PostgreSQL 分支,简化为纯 SQLite
- models/base.py: 移除 pgvector 导入和条件分支
- search_service.py: 移除 _search_postgres 方法
- import_service.py: 移除 pgvector 相关代码
- requirements.txt: 移除 asyncpg/alembic/pgvector 依赖
- pyproject.toml: 同步移除相关依赖
- docker-compose.yml: 移除 db 服务,- 删除 alembic.ini/Dockerfile.db/sql 目录
- README.md: 更新文档,移除 PostgreSQL 相关内容

适合 NAS 等资源受限环境的轻量级部署
This commit is contained in:
EduBrain Dev
2026-04-14 14:50:53 +08:00
parent f60b45358d
commit 496e11e26e
14 changed files with 112 additions and 697 deletions

View File

@@ -46,8 +46,8 @@ class Settings(BaseSettings):
# ──────────────────────────── 数据库 ────────────────────────────
DATABASE_URL: str = Field(
default="postgresql+asyncpg://postgres:postgres@db:5432/edu_brain",
description="PostgreSQL 异步连接字符串",
default="sqlite+aiosqlite:///./edu_brain.db",
description="SQLite 数据库连接字符串",
)
# ──────────────────────────── 嵌入模型 ────────────────────────────

View File

@@ -1,12 +1,13 @@
"""
数据库连接模块
支持 PostgreSQL生产和 SQLite本地开发两种后端
数据库连接管理
SQLite 数据库,支持异步操作。
"""
from __future__ import annotations
from collections.abc import AsyncGenerator
from pathlib import Path
from typing import AsyncGenerator
from sqlalchemy import text
from sqlalchemy.ext.asyncio import (
@@ -14,35 +15,25 @@ from sqlalchemy.ext.asyncio import (
async_sessionmaker,
create_async_engine,
)
from sqlalchemy.pool import NullPool
from app.config import settings
# ──────────────────────────── 判断数据库类型 ────────────────────────────
# ──────────────────────────── 数据库引擎 ────────────────────────────
DB_URL = settings.DATABASE_URL
IS_SQLITE = DB_URL.startswith("sqlite")
# ──────────────────────────── 异步引擎 ────────────────────────────
# 解析 SQLite 文件路径
sqlite_path = DB_URL.replace("sqlite+aiosqlite:///", "")
if not sqlite_path.startswith("/"):
sqlite_path = str(Path(__file__).parent.parent / sqlite_path)
if IS_SQLITE:
# SQLite 配置
sqlite_path = DB_URL.replace("sqlite+aiosqlite:///", "")
if not sqlite_path.startswith("/"):
sqlite_path = str(Path(__file__).parent.parent / sqlite_path)
# 确保数据目录存在
Path(sqlite_path).parent.mkdir(parents=True, exist_ok=True)
engine = create_async_engine(
f"sqlite+aiosqlite:///{sqlite_path}",
echo=False,
)
else:
# PostgreSQL 配置
engine = create_async_engine(
DB_URL,
echo=False,
poolclass=NullPool,
pool_pre_ping=True,
)
engine = create_async_engine(
f"sqlite+aiosqlite:///{sqlite_path}",
echo=False,
)
# ──────────────────────────── 会话工厂 ────────────────────────────
@@ -61,47 +52,21 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]:
FastAPI 依赖项:获取异步数据库会话。
"""
async with async_session_factory() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
yield session
# ──────────────────────────── 生命周期辅助 ────────────────────────────
async def close_db() -> None:
"""关闭数据库连接池"""
await engine.dispose()
async def init_db() -> None:
"""初始化数据库:建表"""
from app.models.base import Base
async with engine.begin() as conn:
if IS_SQLITE:
# SQLite 启用 WAL 模式和外键约束
await conn.execute(text("PRAGMA journal_mode=WAL"))
await conn.execute(text("PRAGMA foreign_keys=ON"))
# SQLite 启用 WAL 模式和外键约束
await conn.execute(text("PRAGMA journal_mode=WAL"))
await conn.execute(text("PRAGMA foreign_keys=ON"))
# 创建所有表
await conn.run_sync(Base.metadata.create_all)
if not IS_SQLITE:
# PostgreSQL 需要创建扩展
async with engine.begin() as conn:
try:
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
except Exception:
pass
try:
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS zhparser"))
except Exception:
pass
try:
await conn.execute(text(
"CREATE TEXT SEARCH CONFIGURATION IF NOT EXISTS chinese_zh (PARSER = zhparser)"
))
except Exception:
pass
async def close_db() -> None:
"""关闭数据库引擎"""
await engine.dispose()

View File

@@ -1,26 +1,18 @@
"""
SQLAlchemy 基础模型 + 通用 Mixin
提供 DeclarativeBase、时间戳 Mixin 和公共序列化方法
兼容 PostgreSQLpgvector和 SQLite本地开发
ORM 模型定义
SQLite 数据库模型,使用 Text 存储 JSON 格式的向量。
"""
from __future__ import annotations
import json
from datetime import datetime
from typing import Any, Dict
from sqlalchemy import DateTime, Float, Integer, String, Text, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from app.database import IS_SQLITE
# SQLite 下不使用 pgvector用 Text 存储向量JSON 格式)
if IS_SQLITE:
VectorType = Text
else:
from pgvector.sqlalchemy import Vector
VectorType = Vector
class Base(DeclarativeBase):
"""所有 ORM 模型的基类"""
@@ -28,96 +20,83 @@ class Base(DeclarativeBase):
class TimestampMixin:
"""
时间戳 Mixin为模型自动添加 created_at / updated_at 字段。
"""
"""时间戳 Mixin"""
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
comment="创建时间",
DateTime, default=func.now(), comment="创建时间"
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now(),
comment="更新时间",
DateTime, default=func.now(), onupdate=func.now(), comment="更新时间"
)
def to_dict(self) -> Dict[str, Any]:
"""将模型实例转换为字典"""
result: Dict[str, Any] = {}
for column in self.__table__.columns: # type: ignore[attr-defined]
value = getattr(self, column.name, None)
if isinstance(value, datetime):
value = value.isoformat()
elif hasattr(value, "tolist"):
value = value.tolist()
result[column.name] = value
return result
# ──────────────────────────── 业务模型 ────────────────────────────
class KnowledgePage(Base, TimestampMixin):
"""知识页面模型"""
"""知识页面"""
__tablename__ = "knowledge_pages"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
title: Mapped[str] = mapped_column(String(500), nullable=False, comment="页面标题")
content: Mapped[str] = mapped_column(Text, nullable=False, comment="页面正文内容")
source_file: Mapped[str | None] = mapped_column(String(500), nullable=True, comment="来源文件名")
course_name: Mapped[str | None] = mapped_column(String(200), nullable=True, comment="课程名称")
teacher_name: Mapped[str | None] = mapped_column(String(100), nullable=True, comment="讲师名称")
live_date: Mapped[str | None] = mapped_column(String(20), nullable=True, comment="直播日期")
page_number: Mapped[int | None] = mapped_column(Integer, nullable=True, comment="原始页码")
metadata_json: Mapped[str | None] = mapped_column(Text, nullable=True, comment="额外元数据")
content: Mapped[str] = mapped_column(Text, nullable=False, comment="页面内容")
source: Mapped[str | None] = mapped_column(String(200), nullable=True, comment="来源文件名")
page_number: Mapped[int | None] = mapped_column(Integer, nullable=True, comment="原文件页码")
# 向量嵌入JSON 格式存储)
embedding: Mapped[str | None] = mapped_column(Text, nullable=True, comment="向量嵌入JSON")
@property
def embedding_vector(self) -> list[float] | None:
if self.embedding is None:
return None
return json.loads(self.embedding)
class KnowledgeChunk(Base, TimestampMixin):
"""知识分块模型"""
__tablename__ = "knowledge_chunks"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
page_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True, comment="关联页面 ID")
chunk_index: Mapped[int] = mapped_column(Integer, nullable=False, comment="分块序号")
content: Mapped[str] = mapped_column(Text, nullable=False, comment="分块文本内容")
embedding: Mapped[list | None] = mapped_column(
VectorType(768) if not IS_SQLITE else Text,
nullable=True,
comment="嵌入向量",
)
search_vector: Mapped[Any | None] = mapped_column(
Text, nullable=True, comment="全文搜索向量(仅 PG",
)
def to_dict(self) -> Dict[str, Any]:
result = super().to_dict()
result.pop("embedding", None)
result.pop("search_vector", None)
return result
@embedding_vector.setter
def embedding_vector(self, value: list[float] | None):
self.embedding = json.dumps(value) if value else None
class OCRImage(Base, TimestampMixin):
"""OCR 图片记录模型"""
"""OCR 图片记录"""
__tablename__ = "ocr_images"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
file_path: Mapped[str] = mapped_column(String(500), nullable=False, comment="图片文件路径")
ocr_text: Mapped[str | None] = mapped_column(Text, nullable=True, comment="OCR 识别全文")
confidence: Mapped[float | None] = mapped_column(Float, nullable=True, comment="平均置信度")
original_filename: Mapped[str | None] = mapped_column(String(200), nullable=True, comment="原始文件名")
ocr_text: Mapped[str | None] = mapped_column(Text, nullable=True, comment="OCR 识别文本")
confidence: Mapped[float | None] = mapped_column(Float, nullable=True, comment="OCR 置信度")
provider: Mapped[str | None] = mapped_column(String(50), nullable=True, comment="OCR 提供商")
status: Mapped[str] = mapped_column(String(20), default="pending", comment="处理状态")
error_message: Mapped[str | None] = mapped_column(Text, nullable=True, comment="错误信息")
tags: Mapped[str | None] = mapped_column(Text, nullable=True, comment="LLM 提取的标签JSON 数组格式")
story_summary: Mapped[str | None] = mapped_column(Text, nullable=True, comment="Qwen3-8B 提炼的故事摘要")
status: Mapped[str] = mapped_column(
String(20), default="pending", comment="状态: pending/processing/completed/failed"
)
tags: Mapped[str | None] = mapped_column(Text, nullable=True, comment="关键词标签JSON 数组)")
story_summary: Mapped[str | None] = mapped_column(Text, nullable=True, comment="故事摘要")
blocks: Mapped[str | None] = mapped_column(Text, nullable=True, comment="OCR 文本块JSON")
def to_dict(self) -> Dict[str, Any]:
result = super().to_dict()
# 将 tags 从 JSON 字符串解析为列表返回
if result.get("tags") and isinstance(result["tags"], str):
try:
import json
result["tags"] = json.loads(result["tags"])
except (json.JSONDecodeError, TypeError):
pass
return result
@property
def tags_list(self) -> list[str]:
if self.tags is None:
return []
return json.loads(self.tags)
@tags_list.setter
def tags_list(self, value: list[str]):
self.tags = json.dumps(value) if value else None
@property
def blocks_list(self) -> list[dict]:
if self.blocks is None:
return []
return json.loads(self.blocks)
@blocks_list.setter
def blocks_list(self, value: list[dict]):
self.blocks = json.dumps(value) if value else None
class WebsiteSettings(Base):
"""网站设置(单例)"""
__tablename__ = "website_settings"
key: Mapped[str] = mapped_column(String(100), primary_key=True, comment="设置键")
value: Mapped[str | None] = mapped_column(Text, nullable=True, comment="设置值JSON")
updated_at: Mapped[datetime] = mapped_column(
DateTime, default=func.now(), onupdate=func.now(), comment="更新时间"
)

View File

@@ -14,7 +14,6 @@ from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.database import IS_SQLITE
logger = logging.getLogger(__name__)
@@ -343,21 +342,13 @@ class ImportService:
# 批量插入
for idx, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):
if IS_SQLITE:
# SQLite: 向量存为 JSON 文本
import json as _json
embedding_str = _json.dumps(embedding) if embedding else None
sql = text("""
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
VALUES (:page_id, :chunk_index, :content, :embedding)
""")
else:
# PostgreSQL: 使用 pgvector 类型
embedding_str = "[" + ",".join(str(x) for x in embedding) + "]" if embedding else None
sql = text("""
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
VALUES (:page_id, :chunk_index, :content, :embedding::vector)
""")
# SQLite: 向量存为 JSON 文本
import json as _json
embedding_str = _json.dumps(embedding) if embedding else None
sql = text("""
INSERT INTO knowledge_chunks (page_id, chunk_index, content, embedding)
VALUES (:page_id, :chunk_index, :content, :embedding)
""")
await self.db.execute(sql, {
"page_id": page_id,
"chunk_index": idx,
@@ -365,14 +356,6 @@ class ImportService:
"embedding": embedding_str,
})
# 更新全文搜索向量(仅 PostgreSQL
if not IS_SQLITE:
await self.db.execute(text("""
UPDATE knowledge_chunks
SET search_vector = to_tsvector('chinese_zh', content)
WHERE page_id = :page_id
"""), {"page_id": page_id})
await self.db.flush()
return len(chunks)

View File

@@ -1,20 +1,18 @@
"""
语义搜索服务
支持 PostgreSQL向量+全文混合)和 SQLiteLIKE 关键词搜索)
SQLite 数据库,使用 LIKE 关键词搜索。
"""
from __future__ import annotations
import json
import logging
import time
from typing import List, Optional
from typing import List
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.database import IS_SQLITE
from app.schemas.search import SearchRequest, SearchResult, SearchResponse
logger = logging.getLogger(__name__)
@@ -28,21 +26,10 @@ class SearchService:
async def search(self, request: SearchRequest) -> SearchResponse:
"""
执行搜索
- PostgreSQL: 向量搜索 + 中文全文搜索混合排序
- SQLite: LIKE 关键词搜索(无向量能力)
执行搜索:使用 LIKE 关键词搜索
"""
start_time = time.time()
if IS_SQLITE:
return await self._search_sqlite(request, start_time)
else:
return await self._search_postgres(request, start_time)
# ──────────────────────────── SQLite 搜索 ────────────────────────────
async def _search_sqlite(self, request: SearchRequest, start_time: float) -> SearchResponse:
"""SQLite 模式:使用 LIKE 关键词搜索"""
# LLM 查询扩展(可选)
expanded_queries = [request.query]
try:
@@ -109,178 +96,3 @@ class SearchService:
results=results,
elapsed_ms=round(elapsed_ms, 2),
)
# ──────────────────────────── PostgreSQL 搜索 ────────────────────────────
async def _search_postgres(self, request: SearchRequest, start_time: float) -> SearchResponse:
"""PostgreSQL 模式:向量搜索 + 中文全文搜索混合"""
# LLM 查询扩展
expanded_queries = [request.query]
if request.use_fulltext:
try:
from app.services.llm_service import LLMService
expanded_queries = await LLMService.expand_query(request.query)
except Exception as exc:
logger.debug("查询扩展跳过: %s", exc)
# 生成查询向量
from app.services.embedding_service import EmbeddingService
all_query_embeddings = await EmbeddingService.embed_batch(expanded_queries)
# 构建过滤条件
where_clauses = []
params: dict = {}
if request.course_name:
where_clauses.append("kp.course_name = :course_name")
params["course_name"] = request.course_name
if request.teacher_name:
where_clauses.append("kp.teacher_name = :teacher_name")
params["teacher_name"] = request.teacher_name
if request.live_date_from:
where_clauses.append("kp.live_date >= :live_date_from")
params["live_date_from"] = request.live_date_from
if request.live_date_to:
where_clauses.append("kp.live_date <= :live_date_to")
params["live_date_to"] = request.live_date_to
where_sql = ""
if where_clauses:
where_sql = "AND " + " AND ".join(where_clauses)
# 向量搜索
vector_results = []
params["limit"] = request.top_k * 2
for q_embedding in all_query_embeddings:
embedding_str = "[" + ",".join(str(x) for x in q_embedding) + "]"
vector_sql = f"""
SELECT
kc.id AS chunk_id,
kp.id AS page_id,
kp.title AS page_title,
kc.content,
1 - (kc.embedding <=> :embedding::vector) AS vector_score,
0.0 AS text_score,
kp.course_name,
kp.teacher_name,
kp.live_date
FROM knowledge_chunks kc
JOIN knowledge_pages kp ON kc.page_id = kp.id
WHERE kc.embedding IS NOT NULL
{where_sql}
ORDER BY kc.embedding <=> :embedding::vector
LIMIT :limit
"""
v_params = {**params, "embedding": embedding_str}
result = await self.db.execute(text(vector_sql), v_params)
vector_results.extend(result.fetchall())
# 全文搜索
text_results = []
if request.use_fulltext:
for q in expanded_queries:
clean_query = q.replace("'", "''")
fulltext_sql = f"""
SELECT
kc.id AS chunk_id,
kp.id AS page_id,
kp.title AS page_title,
kc.content,
0.0 AS vector_score,
ts_rank_cd(kc.search_vector, to_tsquery('chinese_zh', :query)) AS text_score,
kp.course_name,
kp.teacher_name,
kp.live_date,
ts_headline('chinese_zh', kc.content, to_tsquery('chinese_zh', :query),
'MaxWords=50, MinWords=20, ShortWord=2') AS highlight
FROM knowledge_chunks kc
JOIN knowledge_pages kp ON kc.page_id = kp.id
WHERE kc.search_vector @@ to_tsquery('chinese_zh', :query)
{where_sql}
ORDER BY text_score DESC
LIMIT :limit
"""
ft_params = {**params, "query": clean_query}
ft_result = await self.db.execute(text(fulltext_sql), ft_params)
text_results.extend(ft_result.fetchall())
# 合并结果
merged: dict[int, dict] = {}
for row in vector_results:
chunk_id = row.chunk_id
if chunk_id not in merged:
merged[chunk_id] = {
"chunk_id": chunk_id,
"page_id": row.page_id,
"page_title": row.page_title,
"content": row.content,
"vector_score": row.vector_score,
"text_score": 0.0,
"course_name": row.course_name,
"teacher_name": row.teacher_name,
"live_date": row.live_date,
"highlight": None,
}
else:
merged[chunk_id]["vector_score"] = max(
merged[chunk_id]["vector_score"], row.vector_score
)
for row in text_results:
chunk_id = row.chunk_id
if chunk_id not in merged:
merged[chunk_id] = {
"chunk_id": chunk_id,
"page_id": row.page_id,
"page_title": row.page_title,
"content": row.content,
"vector_score": 0.0,
"text_score": row.text_score or 0.0,
"course_name": row.course_name,
"teacher_name": row.teacher_name,
"live_date": row.live_date,
"highlight": row.highlight,
}
else:
merged[chunk_id]["text_score"] = max(
merged[chunk_id]["text_score"], row.text_score or 0.0
)
if row.highlight:
merged[chunk_id]["highlight"] = row.highlight
# 计算综合得分
max_text_score = max(
(r["text_score"] for r in merged.values()), default=1.0
) or 1.0
scored_results = []
for item in merged.values():
normalized_text = item["text_score"] / max_text_score if max_text_score > 0 else 0
combined_score = 0.7 * item["vector_score"] + 0.3 * normalized_text
if combined_score >= request.threshold:
scored_results.append(
SearchResult(
chunk_id=item["chunk_id"],
page_id=item["page_id"],
page_title=item["page_title"],
content=item["content"],
score=round(combined_score, 4),
course_name=item["course_name"],
teacher_name=item["teacher_name"],
live_date=item["live_date"],
highlight=item["highlight"],
)
)
scored_results.sort(key=lambda x: x.score, reverse=True)
scored_results = scored_results[:request.top_k]
elapsed_ms = (time.time() - start_time) * 1000
return SearchResponse(
query=request.query,
total=len(scored_results),
results=scored_results,
elapsed_ms=round(elapsed_ms, 2),
)