This commit is contained in:
2026-04-24 16:02:16 +08:00
commit 1d6f0cc370
66 changed files with 9990 additions and 0 deletions

0
backend/app/__init__.py Normal file
View File

View File

195
backend/app/api/export.py Normal file
View File

@@ -0,0 +1,195 @@
import os
import json
import zipfile
from typing import Annotated
from fastapi import APIRouter, HTTPException, Depends
from fastapi.responses import FileResponse
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, text
from app.database import get_db
from app.models.story import Story
from app.models.person import Person
from app.config import settings
from app.services.docx_generator import generate_person_doc
router = APIRouter(prefix="/export", tags=["导出"])
UNKNOWN_PERSON_ID = "unknown"
UNKNOWN_PERSON_NAME = "暂无"
@router.get("/list")
async def export_list(db: Annotated[AsyncSession, Depends(get_db)]):
"""获取导出列表(所有已匹配的讲述者,含'暂无'"""
# 获取所有已匹配的故事按 person_id 分组
result = await db.execute(
select(Story.person_id)
.where(Story.match_status == "matched")
.distinct()
)
person_ids = [row[0] for row in result.all()]
items = []
for pid in person_ids:
if pid == UNKNOWN_PERSON_ID:
story_result = await db.execute(
select(Story).where(
Story.person_id == UNKNOWN_PERSON_ID,
Story.match_status == "matched",
)
)
stories = story_result.scalars().all()
items.append({
"person_id": UNKNOWN_PERSON_ID,
"person_name": UNKNOWN_PERSON_NAME,
"story_count": len(stories),
"has_photo": False,
})
else:
person = await db.get(Person, pid)
if not person:
continue
story_result = await db.execute(
select(Story).where(
Story.person_id == person.id,
Story.match_status == "matched",
)
)
stories = story_result.scalars().all()
items.append({
"person_id": person.id,
"person_name": person.name,
"story_count": len(stories),
"has_photo": bool(person.photo_path),
})
return items
@router.get("/preview/{person_id}")
async def preview_export(person_id: str, db: Annotated[AsyncSession, Depends(get_db)]):
"""预览某讲述者的文档内容"""
if person_id == UNKNOWN_PERSON_ID:
person_name = UNKNOWN_PERSON_NAME
person_info = ""
has_photo = False
else:
person = await db.get(Person, person_id)
if not person:
raise HTTPException(404, "讲述者不存在")
person_name = person.name
person_info = person.info
has_photo = bool(person.photo_path)
story_result = await db.execute(
select(Story).where(
Story.person_id == person_id,
Story.match_status == "matched",
)
)
stories = story_result.scalars().all()
return {
"person_name": person_name,
"person_info": person_info,
"has_photo": has_photo,
"stories": [{"title": s.title, "summary": s.summary, "tags": json.loads(s.tags) if s.tags else [], "content": s.content, "raw_material": s.raw_material or ""} for s in stories],
}
@router.get("/download/{person_id}")
async def download_one(person_id: str, db: Annotated[AsyncSession, Depends(get_db)]):
"""下载单个讲述者的 DOCX"""
if person_id == UNKNOWN_PERSON_ID:
person_name = UNKNOWN_PERSON_NAME
person_info = ""
photo_path = ""
else:
person = await db.get(Person, person_id)
if not person:
raise HTTPException(404, "讲述者不存在")
person_name = person.name
person_info = person.info
photo_path = person.photo_path or ""
story_result = await db.execute(
select(Story).where(
Story.person_id == person_id,
Story.match_status == "matched",
)
)
stories = story_result.scalars().all()
if not stories:
raise HTTPException(400, "该讲述者没有已匹配的故事")
output_path = os.path.join(settings.EXPORT_DIR, f"{person_name}_故事集.docx")
generate_person_doc(
person_name=person_name,
person_info=person_info,
photo_path=photo_path,
stories=[{"title": s.title, "summary": s.summary, "tags": json.loads(s.tags) if s.tags else [], "content": s.content} for s in stories],
output_path=output_path,
)
return FileResponse(
output_path,
media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
filename=f"{person_name}_故事集.docx",
)
@router.get("/download-all")
async def download_all(db: Annotated[AsyncSession, Depends(get_db)]):
"""批量下载所有已匹配讲述者的文档ZIP"""
# 获取所有已匹配的 person_id
result = await db.execute(
select(Story.person_id)
.where(Story.match_status == "matched")
.distinct()
)
person_ids = [row[0] for row in result.all()]
if not person_ids:
raise HTTPException(400, "没有可导出的文档")
# 生成 ZIP
zip_path = os.path.join(settings.EXPORT_DIR, "故事集_全部.zip")
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for pid in person_ids:
if pid == UNKNOWN_PERSON_ID:
person_name = UNKNOWN_PERSON_NAME
person_info = ""
photo_path = ""
else:
person = await db.get(Person, pid)
if not person:
continue
person_name = person.name
person_info = person.info
photo_path = person.photo_path or ""
story_result = await db.execute(
select(Story).where(
Story.person_id == pid,
Story.match_status == "matched",
)
)
stories = story_result.scalars().all()
if not stories:
continue
output_path = os.path.join(settings.EXPORT_DIR, f"{person_name}_故事集.docx")
generate_person_doc(
person_name=person_name,
person_info=person_info,
photo_path=photo_path,
stories=[{"title": s.title, "summary": s.summary, "tags": json.loads(s.tags) if s.tags else [], "content": s.content} for s in stories],
output_path=output_path,
)
zf.write(output_path, f"{person_name}_故事集.docx")
return FileResponse(
zip_path,
media_type="application/zip",
filename="故事集_全部.zip",
)

View File

@@ -0,0 +1,112 @@
import asyncio
import json
from typing import Annotated, Optional
from fastapi import APIRouter, HTTPException, Depends, Query
from fastapi.responses import StreamingResponse
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.database import get_db
from app.models.transcript import Transcript
from app.state import task_manager
from app.services.extractor import run_extraction
router = APIRouter(prefix="/extraction", tags=["故事提取"])
@router.post("/start")
async def start_extraction(
task_id: Optional[str] = Query(None),
db: Annotated[AsyncSession, Depends(get_db)] = None,
):
"""启动提取任务,支持多用户并发"""
# 如果传了 task_id复用已有任务
if task_id:
state = task_manager.get_task(task_id)
if state and state.is_running:
return {"task_id": task_id, "message": "任务正在进行中"}
else:
state = None
# 检查是否有待处理的文字稿
result = await db.execute(select(Transcript).where(Transcript.status == "pending"))
pending = result.scalars().all()
if not pending:
# 没有 pending 的,把所有非 pending 的文件重置为 pending包括 completed/processing/error
result2 = await db.execute(
select(Transcript).where(Transcript.status != "pending")
)
others = result2.scalars().all()
if others:
for t in others:
t.status = "pending"
t.error_message = ""
await db.commit()
pending = others
else:
raise HTTPException(400, "没有可提取的文字稿,请先上传文件")
total = len(pending)
# 创建新任务
state = task_manager.create_task()
state.reset()
state.is_running = True
asyncio.create_task(run_extraction(state))
return {"task_id": state.task_id, "total_files": total}
@router.get("/status")
async def get_status(task_id: Optional[str] = Query(None)):
"""获取提取进度"""
if task_id:
state = task_manager.get_task(task_id)
if state:
return {**state.progress, "task_id": state.task_id}
return {"status": "idle", "percent": 0}
@router.get("/status/stream")
async def stream_status(task_id: Optional[str] = Query(None)):
"""SSE 实时进度推送(支持多用户)"""
if not task_id:
raise HTTPException(400, "缺少 task_id 参数")
state = task_manager.get_task(task_id)
if not state:
raise HTTPException(404, "任务不存在")
async def event_generator():
# 立即推送当前状态
yield f"event: progress\ndata: {json.dumps({**state.progress, 'task_id': state.task_id}, ensure_ascii=False)}\n\n"
if state.progress.get("status") in ("completed", "error", "stopped"):
return
while True:
data = await state.get_update()
yield f"event: progress\ndata: {json.dumps({**data, 'task_id': state.task_id}, ensure_ascii=False)}\n\n"
if data.get("status") in ("completed", "error", "stopped"):
break
return StreamingResponse(
event_generator(),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@router.post("/stop")
async def stop_extraction(task_id: Optional[str] = Query(None)):
"""停止提取任务"""
if task_id:
state = task_manager.get_task(task_id)
else:
state = None
if not state or not state.is_running:
raise HTTPException(400, "没有正在进行的提取任务")
state.cancel_event.set()
return {"message": "正在停止...", "task_id": state.task_id}

136
backend/app/api/files.py Normal file
View File

@@ -0,0 +1,136 @@
import os
import uuid
from typing import Annotated
from fastapi import APIRouter, UploadFile, File, HTTPException, Depends
from fastapi.responses import FileResponse
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, delete
from app.database import get_db
from app.models.transcript import Transcript
from app.models.person import Person
from app.schemas.schemas import TranscriptOut, PersonOut
from app.services.file_service import file_service
from app.services.docx_parser import parse_transcript, parse_person_info
router = APIRouter(prefix="/files", tags=["文件管理"])
# ===== 文字稿 =====
@router.post("/transcripts", response_model=list[TranscriptOut])
async def upload_transcripts(files: list[UploadFile], db: Annotated[AsyncSession, Depends(get_db)]):
"""批量上传文字稿"""
results = []
for file in files:
if not file.filename.endswith(".docx"):
raise HTTPException(400, f"仅支持 .docx 格式: {file.filename}")
saved = await file_service.save_transcript(file)
# 解析行数
paragraphs = parse_transcript(saved["file_path"])
line_count = len(paragraphs)
record = Transcript(
id=saved["id"],
filename=saved["filename"],
file_path=saved["file_path"],
line_count=line_count,
file_size=saved["file_size"],
status="pending",
)
db.add(record)
results.append(record)
await db.commit()
for r in results:
await db.refresh(r)
return results
@router.get("/transcripts", response_model=list[TranscriptOut])
async def list_transcripts(db: Annotated[AsyncSession, Depends(get_db)]):
"""获取文字稿列表"""
result = await db.execute(select(Transcript).order_by(Transcript.uploaded_at.desc()))
return result.scalars().all()
@router.delete("/transcripts/{transcript_id}")
async def delete_transcript(transcript_id: str, db: Annotated[AsyncSession, Depends(get_db)]):
"""删除文字稿"""
record = await db.get(Transcript, transcript_id)
if not record:
raise HTTPException(404, "文字稿不存在")
file_service.delete_file(record.file_path)
await db.delete(record)
await db.commit()
return {"message": "已删除"}
# ===== 讲述者信息 =====
@router.post("/persons", response_model=list[PersonOut])
async def upload_persons(files: list[UploadFile], db: Annotated[AsyncSession, Depends(get_db)]):
"""批量上传讲述者信息"""
results = []
for file in files:
if not file.filename.endswith(".docx"):
raise HTTPException(400, f"仅支持 .docx 格式: {file.filename}")
saved = await file_service.save_person(file)
# 解析个人信息
info = parse_person_info(saved["file_path"])
record = Person(
id=saved["id"],
name=info["name"],
filename=saved["filename"],
file_path=saved["file_path"],
photo_path=info["photos"][0] if info["photos"] else "",
info=info["info_text"],
)
db.add(record)
results.append(record)
await db.commit()
for r in results:
await db.refresh(r)
return results
@router.get("/persons", response_model=list[PersonOut])
async def list_persons(db: Annotated[AsyncSession, Depends(get_db)]):
"""获取讲述者列表"""
result = await db.execute(select(Person).order_by(Person.uploaded_at.desc()))
return result.scalars().all()
@router.delete("/persons/{person_id}")
async def delete_person(person_id: str, db: Annotated[AsyncSession, Depends(get_db)]):
"""删除讲述者信息"""
record = await db.get(Person, person_id)
if not record:
raise HTTPException(404, "讲述者不存在")
file_service.delete_file(record.file_path)
if record.photo_path:
file_service.delete_file(record.photo_path)
await db.delete(record)
await db.commit()
return {"message": "已删除"}
@router.get("/persons/{person_id}/preview")
async def preview_person(person_id: str, db: Annotated[AsyncSession, Depends(get_db)]):
"""预览讲述者信息"""
record = await db.get(Person, person_id)
if not record:
raise HTTPException(404, "讲述者不存在")
return {
"id": record.id,
"name": record.name,
"nickname": record.nickname,
"info": record.info,
"photo_path": record.photo_path,
}

View File

@@ -0,0 +1,76 @@
from typing import Annotated
from fastapi import APIRouter, HTTPException, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.database import get_db
from app.models.story import Story
from app.models.person import Person
from app.schemas.schemas import MatchCreate, MatchOut
router = APIRouter(prefix="/matches", tags=["匹配管理"])
# 特殊讲述者 ID暂无
UNKNOWN_PERSON_ID = "unknown"
UNKNOWN_PERSON_NAME = "暂无"
@router.post("")
async def create_match(data: MatchCreate, db: Annotated[AsyncSession, Depends(get_db)]):
"""创建匹配关系(支持 person_id='unknown' 表示暂无讲述者)"""
story = await db.get(Story, data.story_id)
if not story or story.match_status == "deleted":
raise HTTPException(404, "故事不存在")
if data.person_id == UNKNOWN_PERSON_ID:
person_name = UNKNOWN_PERSON_NAME
else:
person = await db.get(Person, data.person_id)
if not person:
raise HTTPException(404, "讲述者不存在")
person_name = person.name
story.person_id = data.person_id
story.match_status = "matched"
await db.commit()
await db.refresh(story)
return {"message": "匹配成功", "story_id": story.id, "person_id": data.person_id, "person_name": person_name}
@router.delete("/{story_id}")
async def remove_match(story_id: str, db: Annotated[AsyncSession, Depends(get_db)]):
"""取消匹配"""
story = await db.get(Story, story_id)
if not story:
raise HTTPException(404, "故事不存在")
story.person_id = None
story.match_status = "pending"
await db.commit()
return {"message": "已取消匹配"}
@router.get("", response_model=list[MatchOut])
async def list_matches(db: Annotated[AsyncSession, Depends(get_db)]):
"""获取所有匹配关系"""
result = await db.execute(
select(Story, Person)
.join(Person, Story.person_id == Person.id, isouter=True)
.where(Story.match_status == "matched")
)
matches = []
for story, person in result.all():
if story.person_id == UNKNOWN_PERSON_ID:
person_name = UNKNOWN_PERSON_NAME
elif person:
person_name = person.name
else:
person_name = UNKNOWN_PERSON_NAME
matches.append({
"story_id": story.id,
"person_id": story.person_id,
"person_name": person_name,
"story_title": story.title,
})
return matches

View File

@@ -0,0 +1,70 @@
from typing import Annotated
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.database import get_db
from app.models.config import Config
from app.schemas.schemas import SettingsOut, SettingsUpdate
from app.services.llm_client import llm_client
from app.config import settings
router = APIRouter(prefix="/settings", tags=["系统设置"])
async def _get_config_value(db: AsyncSession, key: str, default: str = "") -> str:
"""获取配置值"""
result = await db.execute(select(Config).where(Config.key == key))
config = result.scalar_one_or_none()
return config.value if config else default
@router.get("", response_model=SettingsOut)
async def get_settings(db: Annotated[AsyncSession, Depends(get_db)]):
"""获取系统配置"""
return SettingsOut(
llm_provider=await _get_config_value(db, "llm_provider", settings.LLM_PROVIDER),
llm_model=await _get_config_value(db, "llm_model", settings.LLM_MODEL),
llm_base_url=await _get_config_value(db, "llm_base_url", settings.LLM_BASE_URL),
llm_api_key=await _get_config_value(db, "llm_api_key", "****"),
segment_max_lines=int(await _get_config_value(db, "segment_max_lines", str(settings.SEGMENT_MAX_LINES))),
story_min_lines=int(await _get_config_value(db, "story_min_lines", str(settings.STORY_MIN_LINES))),
confidence_threshold=float(await _get_config_value(db, "confidence_threshold", str(settings.CONFIDENCE_THRESHOLD))),
temperature=float(await _get_config_value(db, "temperature", str(settings.TEMPERATURE))),
)
@router.put("")
async def update_settings(data: SettingsUpdate, db: Annotated[AsyncSession, Depends(get_db)]):
"""更新系统配置"""
updates = data.model_dump(exclude_none=True)
for key, value in updates.items():
# API Key 脱敏处理
if key == "llm_api_key" and value == "****":
continue
existing = await db.execute(select(Config).where(Config.key == key))
config = existing.scalar_one_or_none()
if config:
config.value = str(value)
else:
db.add(Config(key=key, value=str(value)))
await db.commit()
# 更新 LLM 客户端配置
if "llm_base_url" in updates:
llm_client.update_config(base_url=updates["llm_base_url"])
if "llm_api_key" in updates:
llm_client.update_config(api_key=updates["llm_api_key"])
if "llm_model" in updates:
llm_client.update_config(model=updates["llm_model"])
return {"message": "配置已保存"}
@router.post("/test-llm")
async def test_llm():
"""测试 LLM 连接"""
result = await llm_client.test_connection()
return result

View File

@@ -0,0 +1,64 @@
from typing import Annotated
from fastapi import APIRouter, HTTPException, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, update
from app.database import get_db
from app.models.story import Story
from app.schemas.schemas import StoryOut, StoryEdit
router = APIRouter(prefix="/stories", tags=["故事管理"])
@router.get("", response_model=list[StoryOut])
async def list_stories(db: Annotated[AsyncSession, Depends(get_db)], status: str = "all"):
"""获取故事列表支持筛选all/matched/pending"""
query = select(Story).where(Story.match_status != "deleted")
if status == "matched":
query = query.where(Story.match_status == "matched")
elif status == "pending":
query = query.where(Story.match_status == "pending")
query = query.order_by(Story.created_at.desc())
result = await db.execute(query)
return result.scalars().all()
@router.get("/{story_id}", response_model=StoryOut)
async def get_story(story_id: str, db: Annotated[AsyncSession, Depends(get_db)]):
"""获取故事详情"""
story = await db.get(Story, story_id)
if not story or story.match_status == "deleted":
raise HTTPException(404, "故事不存在")
return story
@router.put("/{story_id}", response_model=StoryOut)
async def edit_story(story_id: str, data: StoryEdit, db: Annotated[AsyncSession, Depends(get_db)]):
"""编辑故事"""
story = await db.get(Story, story_id)
if not story or story.match_status == "deleted":
raise HTTPException(404, "故事不存在")
if data.title is not None:
story.title = data.title
if data.summary is not None:
story.summary = data.summary
if data.content is not None:
story.content = data.content
await db.commit()
await db.refresh(story)
return story
@router.delete("/{story_id}")
async def delete_story(story_id: str, db: Annotated[AsyncSession, Depends(get_db)]):
"""删除故事(标记为 deleted"""
story = await db.get(Story, story_id)
if not story or story.match_status == "deleted":
raise HTTPException(404, "故事不存在")
story.match_status = "deleted"
story.person_id = None
await db.commit()
return {"message": "已删除"}

47
backend/app/config.py Normal file
View File

@@ -0,0 +1,47 @@
import os
import sys
from dotenv import load_dotenv
load_dotenv()
def _get_base_dir() -> str:
"""获取基础目录(支持 PyInstaller 打包后的路径)"""
if getattr(sys, 'frozen', False):
# PyInstaller 打包后exe 所在目录
return os.path.dirname(sys.executable)
else:
# 开发模式backend 目录
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BASE_DIR = _get_base_dir()
class Settings:
# App
APP_HOST: str = os.getenv("APP_HOST", "0.0.0.0")
APP_PORT: int = int(os.getenv("APP_PORT", "8000"))
UPLOAD_DIR: str = os.getenv("UPLOAD_DIR", os.path.join(BASE_DIR, "uploads"))
EXPORT_DIR: str = os.getenv("EXPORT_DIR", os.path.join(BASE_DIR, "exports"))
DB_DIR: str = os.getenv("DB_DIR", os.path.join(BASE_DIR, "data"))
# LLM
LLM_PROVIDER: str = os.getenv("LLM_PROVIDER", "xiaomi")
LLM_MODEL: str = os.getenv("LLM_MODEL", "MiMo-V2-Flash")
LLM_BASE_URL: str = os.getenv("LLM_BASE_URL", "https://api.xiaomimimo.com/v1")
LLM_API_KEY: str = os.getenv("LLM_API_KEY", "")
# Extraction
SEGMENT_MAX_LINES: int = int(os.getenv("SEGMENT_MAX_LINES", "500"))
STORY_MIN_LINES: int = int(os.getenv("STORY_MIN_LINES", "50"))
CONFIDENCE_THRESHOLD: float = float(os.getenv("CONFIDENCE_THRESHOLD", "0.5"))
TEMPERATURE: float = float(os.getenv("TEMPERATURE", "0.3"))
@property
def database_url(self) -> str:
os.makedirs(self.DB_DIR, exist_ok=True)
return f"sqlite+aiosqlite:///{self.DB_DIR}/story_extractor.db"
settings = Settings()

26
backend/app/database.py Normal file
View File

@@ -0,0 +1,26 @@
from typing import AsyncGenerator
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy import text
from app.config import settings
engine = create_async_engine(settings.database_url, echo=False, connect_args={"check_same_thread": False})
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
class Base(DeclarativeBase):
pass
async def init_db():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
# 启用 WAL 模式,提升并发读写性能
await conn.execute(text("PRAGMA journal_mode=WAL"))
await conn.execute(text("PRAGMA busy_timeout=5000"))
async def get_db() -> AsyncGenerator[AsyncSession, None]:
async with async_session() as session:
yield session

64
backend/app/main.py Normal file
View File

@@ -0,0 +1,64 @@
import os
import sys
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from fastapi.middleware.cors import CORSMiddleware
from app.config import settings, BASE_DIR
from app.database import init_db
from app.api import files, extraction, stories, matching, export, settings as settings_api
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def _get_frontend_dir() -> str:
"""获取前端静态文件目录(支持 PyInstaller 打包)"""
if getattr(sys, 'frozen', False):
# 打包后exe 同级目录下的 frontend/dist
return os.path.join(BASE_DIR, "frontend", "dist")
else:
# 开发模式:项目根目录下的 frontend/dist
return os.path.join(os.path.dirname(BASE_DIR), "frontend", "dist")
@asynccontextmanager
async def lifespan(app: FastAPI):
"""应用生命周期:启动时初始化数据库"""
logger.info("初始化数据库...")
await init_db()
logger.info("数据库初始化完成")
yield
logger.info("应用关闭")
app = FastAPI(
title="Story Extractor",
description="直播文字稿故事提取工具",
version="1.0.0",
lifespan=lifespan,
)
# CORS
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# API 路由
app.include_router(files.router, prefix="/api/v1")
app.include_router(extraction.router, prefix="/api/v1")
app.include_router(stories.router, prefix="/api/v1")
app.include_router(matching.router, prefix="/api/v1")
app.include_router(export.router, prefix="/api/v1")
app.include_router(settings_api.router, prefix="/api/v1")
# 托管前端静态文件(放在最后,作为 fallback
frontend_dist = _get_frontend_dir()
if os.path.exists(frontend_dist):
app.mount("/", StaticFiles(directory=frontend_dist, html=True), name="static")

View File

View File

@@ -0,0 +1,11 @@
from sqlalchemy import Column, String, Text
from sqlalchemy.sql import func
from app.database import Base
class Config(Base):
__tablename__ = "config"
key = Column(String, primary_key=True)
value = Column(Text, nullable=False)

View File

@@ -0,0 +1,17 @@
from sqlalchemy import Column, String, Text, DateTime
from sqlalchemy.sql import func
from app.database import Base
class Person(Base):
__tablename__ = "persons"
id = Column(String, primary_key=True)
name = Column(String, nullable=False)
nickname = Column(String, default="") # 文字稿中的昵称
filename = Column(String, nullable=False)
file_path = Column(String, nullable=False)
photo_path = Column(String, default="")
info = Column(Text, default="") # JSON
uploaded_at = Column(DateTime, server_default=func.now())

View File

@@ -0,0 +1,26 @@
import uuid
from sqlalchemy import Column, String, Integer, Float, Text, DateTime, ForeignKey
from sqlalchemy.sql import func
from app.database import Base
class Story(Base):
__tablename__ = "stories"
id = Column(String, primary_key=True, default=lambda: uuid.uuid4().hex)
title = Column(String, nullable=False)
summary = Column(Text, default="")
tags = Column(Text, default="") # JSON: ["标签1", "标签2", ...]
content = Column(Text, default="")
raw_material = Column(Text, default="") # JSON: 第四步重组后的素材
speaker_nickname = Column(String, default="")
source_transcript_id = Column(String, ForeignKey("transcripts.id"), nullable=False)
source_lines = Column(Text, default="") # JSON: {start, end}
duration_minutes = Column(Float, default=0)
confidence = Column(Float, default=0)
confidence_level = Column(String, default="medium") # high/medium/low
person_id = Column(String, ForeignKey("persons.id"), nullable=True)
match_status = Column(String, default="pending") # pending/matched/deleted
created_at = Column(DateTime, server_default=func.now())

View File

@@ -0,0 +1,17 @@
from sqlalchemy import Column, String, Integer, Float, Text, DateTime
from sqlalchemy.sql import func
from app.database import Base
class Transcript(Base):
__tablename__ = "transcripts"
id = Column(String, primary_key=True)
filename = Column(String, nullable=False)
file_path = Column(String, nullable=False)
line_count = Column(Integer, default=0)
file_size = Column(Integer, default=0)
status = Column(String, default="pending") # pending/processing/done/error
error_message = Column(Text, default="")
uploaded_at = Column(DateTime, server_default=func.now())

View File

View File

@@ -0,0 +1,100 @@
from pydantic import BaseModel
from datetime import datetime
class TranscriptOut(BaseModel):
id: str
filename: str
file_path: str
line_count: int
file_size: int
status: str
error_message: str
uploaded_at: datetime
class Config:
from_attributes = True
class PersonOut(BaseModel):
id: str
name: str
nickname: str
filename: str
file_path: str
photo_path: str
info: str
uploaded_at: datetime
class Config:
from_attributes = True
class StoryOut(BaseModel):
id: str
title: str
summary: str
content: str
raw_material: str
speaker_nickname: str
source_transcript_id: str
source_lines: str
duration_minutes: float
confidence: float
confidence_level: str
person_id: str | None
match_status: str
created_at: datetime
class Config:
from_attributes = True
class StoryEdit(BaseModel):
title: str | None = None
summary: str | None = None
content: str | None = None
class MatchCreate(BaseModel):
story_id: str
person_id: str
class MatchOut(BaseModel):
story_id: str
person_id: str
person_name: str
story_title: str
class Config:
from_attributes = True
class ExportItemOut(BaseModel):
person_id: str
person_name: str
story_count: int
has_photo: bool
class SettingsOut(BaseModel):
llm_provider: str
llm_model: str
llm_base_url: str
llm_api_key: str
segment_max_lines: int
story_min_lines: int
confidence_threshold: float
temperature: float
class SettingsUpdate(BaseModel):
llm_provider: str | None = None
llm_model: str | None = None
llm_base_url: str | None = None
llm_api_key: str | None = None
segment_max_lines: int | None = None
story_min_lines: int | None = None
confidence_threshold: float | None = None
temperature: float | None = None

View File

View File

@@ -0,0 +1,166 @@
import os
import re
from docx import Document
from docx.shared import Pt, Inches, Cm, RGBColor
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.oxml.ns import qn
def generate_person_doc(person_name: str, person_info: str, photo_path: str,
stories: list[dict], output_path: str):
"""
为一位讲述者生成最终 DOCX 文档
person_info: 个人信息文本
photo_path: 照片路径(可为空)
stories: [{title, content}]
"""
doc = Document()
# 设置默认字体
style = doc.styles['Normal']
font = style.font
font.name = '宋体'
font.size = Pt(12)
style.element.rPr.rFonts.set(qn('w:eastAsia'), '宋体')
# 照片
if photo_path and os.path.exists(photo_path):
try:
doc.add_picture(photo_path, width=Inches(1.5))
last_paragraph = doc.paragraphs[-1]
last_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
except Exception:
pass
# 姓名
name_para = doc.add_paragraph()
name_para.alignment = WD_ALIGN_PARAGRAPH.CENTER
name_run = name_para.add_run(person_name)
name_run.font.size = Pt(18)
name_run.font.bold = True
name_run.font.name = '黑体'
name_run.element.rPr.rFonts.set(qn('w:eastAsia'), '黑体')
# 个人信息
if person_info:
info_para = doc.add_paragraph()
info_para.alignment = WD_ALIGN_PARAGRAPH.CENTER
info_run = info_para.add_run(person_info.strip())
info_run.font.size = Pt(11)
info_run.font.color.rgb = RGBColor(0x66, 0x66, 0x66)
# 分隔线
doc.add_paragraph("" * 30).alignment = WD_ALIGN_PARAGRAPH.CENTER
# 故事
for i, story in enumerate(stories):
# 故事标题
title_para = doc.add_paragraph()
title_run = title_para.add_run(story.get("title", f"故事 {i + 1}"))
title_run.font.size = Pt(14)
title_run.font.bold = True
title_run.font.name = '黑体'
title_run.element.rPr.rFonts.set(qn('w:eastAsia'), '黑体')
# 故事梗概
summary = story.get("summary", "")
if summary:
p = doc.add_paragraph()
run = p.add_run("【故事梗概】")
run.font.size = Pt(11)
run.font.bold = True
run.font.color.rgb = RGBColor(0x33, 0x33, 0x99)
p = doc.add_paragraph(summary.strip())
p.paragraph_format.first_line_indent = Cm(0.74)
p.paragraph_format.line_spacing = 1.5
for run in p.runs:
run.font.size = Pt(11)
# 关键词标签
tags = story.get("tags", [])
if tags:
p = doc.add_paragraph()
run = p.add_run("【关键词标签】")
run.font.size = Pt(11)
run.font.bold = True
run.font.color.rgb = RGBColor(0x33, 0x33, 0x99)
tag_text = "".join(tags)
p = doc.add_paragraph(tag_text)
p.paragraph_format.line_spacing = 1.5
for run in p.runs:
run.font.size = Pt(10)
run.font.color.rgb = RGBColor(0x66, 0x66, 0x66)
# 故事内容(支持 Markdown 格式)
content = story.get("content", "")
if content:
_add_markdown_content(doc, content)
# 多个故事之间加分隔
if i < len(stories) - 1:
doc.add_paragraph("" * 30).alignment = WD_ALIGN_PARAGRAPH.CENTER
# 保存
os.makedirs(os.path.dirname(output_path), exist_ok=True)
doc.save(output_path)
return output_path
def _add_markdown_content(doc, content: str):
"""将 Markdown 格式的内容添加到 DOCX支持标题、引用、列表等"""
lines = content.split("\n")
i = 0
while i < len(lines):
line = lines[i].strip()
if not line:
i += 1
continue
# ## 二级标题
if line.startswith("## "):
title = line[3:].strip()
p = doc.add_paragraph()
run = p.add_run(title)
run.font.size = Pt(13)
run.font.bold = True
run.font.name = '黑体'
run.element.rPr.rFonts.set(qn('w:eastAsia'), '黑体')
p.paragraph_format.space_before = Pt(12)
# ### 三级标题
elif line.startswith("### "):
title = line[4:].strip()
p = doc.add_paragraph()
run = p.add_run(title)
run.font.size = Pt(12)
run.font.bold = True
run.font.name = '黑体'
run.element.rPr.rFonts.set(qn('w:eastAsia'), '黑体')
p.paragraph_format.space_before = Pt(6)
# > 引用
elif line.startswith("> "):
quote_text = line[2:].strip()
p = doc.add_paragraph()
p.paragraph_format.left_indent = Cm(1)
run = p.add_run(quote_text)
run.font.italic = True
run.font.color.rgb = RGBColor(0x55, 0x55, 0x55)
run.font.size = Pt(11)
# - 列表项
elif line.startswith("- "):
item_text = line[2:].strip()
p = doc.add_paragraph(style='List Bullet')
run = p.add_run(item_text)
run.font.size = Pt(12)
# 普通段落
else:
p = doc.add_paragraph(line)
p.paragraph_format.first_line_indent = Cm(0.74)
p.paragraph_format.line_spacing = 1.5
i += 1

View File

@@ -0,0 +1,93 @@
import re
from docx import Document
def parse_transcript(file_path: str) -> list[dict]:
"""
解析直播回放文字稿 DOCX
格式:发言者(时间戳): 内容
返回:[{index, speaker, timestamp, content, raw_text}]
"""
doc = Document(file_path)
paragraphs = []
for i, para in enumerate(doc.paragraphs):
text = para.text.strip()
if not text:
continue
# 解析格式:发言者(时间戳): 内容
match = re.match(r'^(.+?)\((\d{2}:\d{2}:\d{2})\)\s*[:]\s*(.*)', text)
if match:
speaker = match.group(1).strip()
timestamp = match.group(2)
content = match.group(3).strip()
else:
# 尝试格式:发言者: 内容
match2 = re.match(r'^(.+?)\s*[:]\s*(.*)', text)
if match2:
speaker = match2.group(1).strip()
timestamp = ""
content = match2.group(2).strip()
else:
speaker = ""
timestamp = ""
content = text
paragraphs.append({
"index": i,
"speaker": speaker,
"timestamp": timestamp,
"content": content,
"raw_text": text,
})
return paragraphs
def is_teacher(speaker: str) -> bool:
"""判断是否为老师/助教发言"""
teacher_keywords = ["卢慧老师", "静静", "督导", "老师"]
return any(kw in speaker for kw in teacher_keywords)
def parse_person_info(file_path: str) -> dict:
"""
解析讲述者个人信息 DOCX
返回:{name, info_text, photos: [path]}
"""
doc = Document(file_path)
texts = []
photos = []
# 提取文本
for para in doc.paragraphs:
text = para.text.strip()
if text:
texts.append(text)
# 提取图片
import os
photo_dir = os.path.join(os.path.dirname(file_path), "photos")
os.makedirs(photo_dir, exist_ok=True)
for i, rel in enumerate(doc.part.rels.values()):
if "image" in rel.reltype:
image = rel.target_part
photo_name = f"{os.path.splitext(os.path.basename(file_path))[0]}_photo_{i}.{image.content_type.split('/')[-1]}"
photo_path = os.path.join(photo_dir, photo_name)
with open(photo_path, "wb") as f:
f.write(image.blob)
photos.append(photo_path)
# 从文件名提取姓名
filename = os.path.basename(file_path)
name = re.sub(r'[-_].*$', '', filename).strip()
# 去掉扩展名
name = os.path.splitext(name)[0].strip()
return {
"name": name,
"info_text": "\n".join(texts),
"photos": photos,
}

View File

@@ -0,0 +1,957 @@
"""
故事提取器 - v4 五步流水线策略
第一步:代码预处理(识别老师→过滤杂音→时间范围截取→过滤短段落/非主讲述人)
第二步:判断故事 + 提取事实(短段落直接提取,长段落先判断再切片提取)
第三步:切片提取事实(按自然段切分,并行提取)
第四步:重组(纯代码,事件/情感/原话/人物去重)
第五步生成最终故事基于素材生成5部分结构化故事
"""
import json
import asyncio
import logging
from sqlalchemy import select
from app.database import async_session
from app.models.story import Story
from app.models.transcript import Transcript
from app.services.docx_parser import parse_transcript
from app.services.preprocessor import Preprocessor
from app.services.llm_client import llm_client
from app.state import TaskState
logger = logging.getLogger(__name__)
# 并发控制
MAX_CONCURRENT_FILES = 3
MAX_CONCURRENT_SEGMENTS = 5
# 短段落阈值(字数):<=4000 字直接提取事实,>4000 字先判断是否有故事
SHORT_THRESHOLD = 4000
# 切片参数
CHUNK_MAX_CHARS = 3000
CHUNK_OVERLAP_PARAGRAPHS = 5
# ============================================================
# 主入口
# ============================================================
async def run_extraction(state: TaskState):
"""主提取流程:文件间串行(避免 API 限流),文件内各步骤并行"""
state.is_running = True
try:
async with async_session() as db:
result = await db.execute(
select(Transcript).where(Transcript.status == "pending")
)
transcripts = result.scalars().all()
if not transcripts:
state.update(status="completed", percent=100)
return
file_total = len(transcripts)
state.update(
status="running", percent=0,
files_total=file_total, files_completed=0,
stories_found=0,
current_file="", action=f"共发现 {file_total} 份文字稿,开始处理...",
phase="preprocess", phase_percent=0, phase_detail="准备中",
)
total_stories = [0]
# 文件间串行处理(避免 API 限流)
for file_index, transcript in enumerate(transcripts):
await _process_file(state, file_index, file_total, transcript, total_stories)
state.update(
status="completed", percent=100,
files_completed=file_total,
stories_found=total_stories[0],
current_file="", action="全部处理完成!",
phase="preprocess", phase_percent=100, phase_detail="全部完成",
)
state.is_running = False
except asyncio.CancelledError:
state.update(status="stopped", action="提取已停止")
state.is_running = False
except Exception as e:
logger.exception("Extraction failed")
state.update(status="error", action=str(e))
state.is_running = False
# ============================================================
# 单文件处理(五步流水线)
# ============================================================
async def _process_file(state: TaskState, file_index: int, file_total: int,
transcript: Transcript, total_stories: list):
"""处理单个文件 - 五步流水线"""
try:
async with async_session() as db:
t = await db.get(Transcript, transcript.id)
if not t:
_increment_files_completed(state)
return
t.status = "processing"
await db.commit()
# 计算该文件在总进度中的权重区间
# 预处理 0-5%, 判断+提取 5-30%, 切片提取 30-55%, 重组 55-60%, 生成 60-95%, 完成 100%
# 每个文件占 95/file_total 的权重
file_weight = 95.0 / max(file_total, 1)
file_start_percent = file_index * file_weight
def _file_percent(phase_start_ratio, phase_end_ratio, inner_ratio=1.0):
"""计算当前文件在总进度中的百分比"""
phase_range = phase_end_ratio - phase_start_ratio
return min(99, int(file_start_percent + file_weight * (phase_start_ratio + phase_range * inner_ratio)))
state.update(
current_file=t.filename, action="正在读取文件...",
phase="preprocess", phase_percent=0, phase_detail="读取文件",
)
# 解析 DOCX
lines = await asyncio.to_thread(parse_transcript, t.file_path)
if not lines:
t.status = "completed"
t.error_message = "文件为空"
await db.commit()
_increment_files_completed(state)
return
# ========== 第一步:代码预处理(纯代码,不用 LLM==========
state.update(
current_file=t.filename, action="正在分析对话结构...",
phase="preprocess", phase_percent=10, phase_detail="统计说话人",
)
preprocessor = Preprocessor()
filtered_lines = [p for p in lines if not preprocessor.is_management_line(p)]
if not filtered_lines:
t.status = "completed"
t.error_message = "过滤后无有效对话内容"
await db.commit()
_increment_files_completed(state)
return
segments, teacher = _preprocess_segments(filtered_lines)
logger.info(f"预处理完成: 识别老师={teacher}, 有效段落={len(segments)}")
if not segments:
t.status = "completed"
t.error_message = "未发现学员对话段落"
await db.commit()
_increment_files_completed(state)
return
state.update(
current_file=t.filename,
action=f"发现 {len(segments)} 个学员段落,开始识别故事...",
phase="preprocess", phase_percent=100, phase_detail="预处理完成",
percent=_file_percent(0, 0.05),
)
# ========== 第二步:判断故事 + 提取事实(并行)==========
state.update(
current_file=t.filename,
action=f"正在判断 {len(segments)} 个段落是否包含故事...",
phase="identify", phase_percent=0,
phase_detail=f"准备处理 {len(segments)} 个段落",
)
seg_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
identify_results = [None] * len(segments)
identify_errors = []
async def process_segment_for_identify(seg_idx, speaker, seg):
"""对单个段落进行判断+提取"""
async with seg_semaphore:
text = _segment_to_text(seg)
text_len = len(text)
# 更新进度
inner_pct = int((seg_idx + 1) / len(segments) * 100)
state.update(
phase="identify", phase_percent=inner_pct,
phase_detail=f"正在处理段落 {seg_idx + 1}/{len(segments)}{speaker}, {text_len}字)",
percent=_file_percent(0.05, 0.30, (seg_idx + 1) / len(segments)),
)
if text_len <= SHORT_THRESHOLD:
# 短段落:跳过判断,直接提取事实
logger.info(f" 段落 {seg_idx}: {speaker} ({text_len}字) - 短段落,直接提取事实")
facts = await _extract_facts_from_chunk(text, "")
if facts:
hints = facts.get("story_hints", "") or f"{speaker}的故事"
return (speaker, seg, hints, [facts])
else:
logger.info(f" 段落 {seg_idx}: 提取失败,跳过")
return None
else:
# 长段落:先判断是否有故事
is_story, hints = await _identify_story(text)
logger.info(f" 段落 {seg_idx}: {speaker} ({text_len}字) - is_story={is_story}, hints={hints}")
if is_story:
return (speaker, seg, hints, None) # None 表示需要第三步切片提取
else:
logger.info(f" 段落 {seg_idx}: 无故事,跳过")
return None
tasks = [
process_segment_for_identify(i, speaker, seg)
for i, (speaker, seg) in enumerate(segments)
]
results = await asyncio.gather(*tasks, return_exceptions=True)
# 收集结果:分为两类
# - 直接提取到事实的(短段落)
# - 需要切片提取的(长段落,有故事)
all_facts = [] # 最终的 (speaker, seg, hints, facts_list) 列表
needs_chunking = [] # 需要进入第三步的 (speaker, seg, hints)
for i, r in enumerate(results):
if isinstance(r, Exception):
error_str = str(r)
logger.warning(f"段落 {i} 处理异常: {error_str}")
identify_errors.append(error_str)
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
raise r
continue
if r is None:
continue
speaker, seg, hints, facts_list = r
if facts_list is not None:
# 短段落已直接提取到事实
all_facts.append((speaker, seg, hints, facts_list))
else:
# 长段落有故事,需要切片提取
needs_chunking.append((speaker, seg, hints))
if not all_facts and not needs_chunking:
t.status = "completed"
t.error_message = "未发现包含个人故事的段落"
await db.commit()
_increment_files_completed(state)
state.update(
percent=_file_percent(0.05, 0.30, 1.0),
action="未发现故事",
)
return
logger.info(f"第二步完成: 直接提取={len(all_facts)}, 需切片={len(needs_chunking)}")
# 第二步完成时已知道发现了多少个故事,立即更新
discovered = len(all_facts) + len(needs_chunking)
total_stories[0] += discovered
state.update(stories_found=total_stories[0])
# ========== 第三步:切片提取事实(并行)==========
if needs_chunking:
state.update(
current_file=t.filename,
action=f"正在切片提取 {len(needs_chunking)} 个故事...",
phase="chunk", phase_percent=0,
phase_detail=f"准备切片 {len(needs_chunking)} 个故事段落",
)
chunk_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
async def process_chunked_story(story_idx, speaker, seg, hints):
"""对单个长段落进行切片提取"""
async with chunk_semaphore:
text = _segment_to_text(seg)
chunks = _chunk_text_by_paragraphs(text, max_chars=CHUNK_MAX_CHARS, overlap_paragraphs=CHUNK_OVERLAP_PARAGRAPHS)
logger.info(f" 故事 {story_idx}: {speaker} - 切成 {len(chunks)}")
facts_list = []
chunk_facts = await asyncio.gather(
*[_extract_facts_from_chunk(chunk, hints) for chunk in chunks],
return_exceptions=True,
)
for j, facts in enumerate(chunk_facts):
inner_pct = int((story_idx * 10 + j + 1) / (len(needs_chunking) * 10) * 100)
state.update(
phase="chunk", phase_percent=inner_pct,
phase_detail=f"正在切片提取 故事{story_idx + 1}/{len(needs_chunking)}{j + 1}/{len(chunks)}",
percent=_file_percent(0.30, 0.55, (story_idx + j / max(len(chunks), 1)) / len(needs_chunking)),
)
if isinstance(facts, Exception):
logger.warning(f"{j}提取异常: {facts}")
continue
if facts:
facts_list.append(facts)
return (speaker, seg, hints, facts_list)
chunk_tasks = [
process_chunked_story(i, speaker, seg, hints)
for i, (speaker, seg, hints) in enumerate(needs_chunking)
]
chunk_results = await asyncio.gather(*chunk_tasks, return_exceptions=True)
for r in chunk_results:
if isinstance(r, Exception):
error_str = str(r)
logger.warning(f"切片提取异常: {error_str}")
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
raise r
continue
if r is not None:
all_facts.append(r)
logger.info(f"第三步完成: 总共 {len(all_facts)} 个故事段落完成事实提取")
state.update(
current_file=t.filename,
action="切片提取完成,正在重组素材...",
phase="chunk", phase_percent=100,
phase_detail="切片提取完成",
percent=_file_percent(0.30, 0.55, 1.0),
)
# ========== 第四步:重组(纯代码,无需 LLM==========
state.update(
current_file=t.filename,
action="正在重组素材...",
phase="merge", phase_percent=50,
phase_detail="去重合并素材",
percent=_file_percent(0.55, 0.60, 0.5),
)
merged = _merge_facts(all_facts)
logger.info(f"第四步完成: 重组出 {len(merged)} 个故事素材")
if not merged:
t.status = "completed"
t.error_message = "素材重组后为空"
await db.commit()
_increment_files_completed(state)
return
state.update(
current_file=t.filename,
action=f"重组完成,正在生成 {len(merged)} 个故事...",
phase="merge", phase_percent=100,
phase_detail="重组完成",
percent=_file_percent(0.55, 0.60, 1.0),
)
# ========== 第五步:生成最终故事(并行)==========
state.update(
current_file=t.filename,
action=f"正在生成 {len(merged)} 个故事...",
phase="generate", phase_percent=0,
phase_detail=f"准备生成 {len(merged)} 个故事",
percent=_file_percent(0.60, 0.95, 0),
)
gen_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
async def generate_one_story(story_idx, speaker, seg, material):
"""生成单个故事并保存到数据库"""
async with gen_semaphore:
inner_pct = int((story_idx + 1) / len(merged) * 100)
state.update(
phase="generate", phase_percent=inner_pct,
phase_detail=f"正在生成故事 {story_idx + 1}/{len(merged)}{speaker}",
percent=_file_percent(0.60, 0.95, (story_idx + 1) / len(merged)),
)
story_data = await _generate_story(material)
if not story_data:
logger.warning(f" 故事 {story_idx}: 生成失败")
return None
# 保存到数据库
raw_content = story_data.get("content", {})
if isinstance(raw_content, dict):
content_str = _format_story_content(raw_content)
else:
content_str = str(raw_content)
story = Story(
title=story_data.get("title", "未命名故事"),
summary=story_data.get("summary", ""),
tags=json.dumps(story_data.get("tags", []), ensure_ascii=False),
content=content_str,
raw_material=json.dumps(material, ensure_ascii=False),
speaker_nickname=story_data.get("speaker_nickname", ""),
source_transcript_id=t.id,
source_lines=json.dumps([l.get("index", 0) for l in seg if isinstance(l, dict)]),
duration_minutes=_estimate_duration(seg),
confidence=story_data.get("confidence", 0.7),
confidence_level=_confidence_level(story_data.get("confidence", 0.7)),
)
db.add(story)
await db.commit()
state.update(
action=f"生成第 {total_stories[0]} 个故事:{story.title}",
)
logger.info(f" 故事 {story_idx}: 保存成功 - {story.title}")
return story
gen_tasks = [
generate_one_story(i, speaker, seg, material)
for i, (speaker, seg, material) in enumerate(merged)
]
gen_results = await asyncio.gather(*gen_tasks, return_exceptions=True)
for r in gen_results:
if isinstance(r, Exception):
error_str = str(r)
logger.warning(f"故事生成异常: {error_str}")
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
raise r
t.status = "completed"
await db.commit()
_increment_files_completed(state)
state.update(
current_file=t.filename,
action=f"文件处理完成,共生成 {total_stories[0]} 个故事",
phase="generate", phase_percent=100,
phase_detail="文件处理完成",
percent=_file_percent(0.60, 0.95, 1.0),
)
except Exception as e:
error_str = str(e)
logger.exception(f"Error processing file {transcript.filename}")
try:
async with async_session() as db:
t = await db.get(Transcript, transcript.id)
if t:
t.status = "error"
t.error_message = error_str
await db.commit()
except Exception:
pass
_increment_files_completed(state)
if "529" in error_str or "overloaded" in error_str.lower():
state.update(status="error", error="AI 服务暂时过载529请稍后再试")
elif "timed out" in error_str.lower():
state.update(status="error", error="AI 服务请求超时,请稍后再试")
else:
state.update(status="error", error=f"处理文件时出错:{error_str[:200]}")
# ============================================================
# 第一步:代码预处理
# ============================================================
def _preprocess_segments(lines: list[dict]) -> tuple[list[tuple[str, list[dict]]], str]:
"""
代码预处理v3时间范围截取允许重叠
识别老师 → 过滤杂音(<20句→ 按时间范围截取 → 过滤短段落(<15句/<80字→ 过滤非主讲述人(占比<10%
返回: (segments, teacher)
segments: [(speaker, segment_lines), ...]
teacher: 老师名称
"""
# 1. 统计每个说话人
speaker_stats = {}
for i, line in enumerate(lines):
speaker = line.get("speaker", "")
if not speaker:
continue
if speaker not in speaker_stats:
speaker_stats[speaker] = {"lines": 0, "chars": 0, "first": i, "last": i}
speaker_stats[speaker]["lines"] += 1
speaker_stats[speaker]["chars"] += len(line.get("content", ""))
speaker_stats[speaker]["last"] = i
# 2. 识别老师(发言最多的人)
teacher = max(speaker_stats, key=lambda s: speaker_stats[s]["lines"])
logger.info(f"识别老师: {teacher} ({speaker_stats[teacher]['lines']}行, {speaker_stats[teacher]['chars']}字)")
# 3. 过滤杂音(< 20句的说话人+ 排除老师
students = []
noise_speakers = []
for s, stats in speaker_stats.items():
if s == teacher:
continue
if stats["lines"] < 20:
noise_speakers.append(f"{s} ({stats['lines']}句)")
continue
students.append(s)
if noise_speakers:
logger.info(f"杂音过滤: {', '.join(noise_speakers)}")
logger.info(f"有效学员: {students}")
# 4. 按时间范围截取:每个学员从第一句到最后一句,中间所有行全部截取
segments = []
for speaker in students:
stats = speaker_stats[speaker]
first_idx = stats["first"]
last_idx = stats["last"]
seg = lines[first_idx:last_idx + 1]
segments.append((speaker, seg))
# 5. 过滤:短段落 + 当事人占比过低(非主讲述人)
final_segments = []
for speaker, seg in segments:
student_lines = [l for l in seg if l.get("speaker", "") == speaker]
if len(student_lines) < 15:
logger.info(f"过滤: {speaker} ({len(student_lines)}句<15)")
continue
student_chars = sum(len(l.get("content", "")) for l in student_lines)
if student_chars < 80:
logger.info(f"过滤: {speaker} ({student_chars}字<80)")
continue
# 当事人说话占比 < 10% → 非主讲述人,丢弃
ratio = len(student_lines) / len(seg) if seg else 0
if ratio < 0.1:
logger.info(f"过滤: {speaker} (占比{ratio:.1%}<10%)")
continue
final_segments.append((speaker, seg))
logger.info(f"最终: {len(final_segments)} 个学员段落")
for i, (speaker, seg) in enumerate(final_segments):
student_lines = [l for l in seg if l.get("speaker", "") == speaker]
student_chars = sum(len(l.get("content", "")) for l in student_lines)
ratio = len(student_lines) / len(seg) if seg else 0
logger.info(f" 段落{i}: {speaker}, 截取{len(seg)}行, 学员{len(student_lines)}句/{student_chars}字, 占比{ratio:.1%}")
return final_segments, teacher
# ============================================================
# 第二步:判断故事
# ============================================================
async def _identify_story(text: str) -> tuple[bool, str]:
"""判断对话段落是否包含个人故事"""
prompt = """判断这段对话是否包含学员的个人故事。直接输出JSON不要分析。
个人故事特征满足至少2条学员讲述具体经历、包含时间地点人物细节、表达深层情感、老师深入引导疗愈、揭示心理或家庭问题。
非故事内容:纯课堂讲解、简单问答寒暄、课堂管理、纯疗愈引导词。
直接输出JSON
{"is_story": true或false, "confidence": 0.0到1.0, "story_hints": "一句话描述故事主题和讲述者"}
对话内容:
```
""" + text[:4000] + """
```"""
try:
result = await llm_client.chat_json([{"role": "user", "content": prompt}], temperature=0.2)
if result:
is_story = result.get("is_story", False)
confidence = result.get("confidence", 0)
hints = result.get("story_hints", "")
logger.info(f"Story identification: is_story={is_story}, confidence={confidence}, hints={hints[:80]}")
return bool(is_story) and confidence >= 0.5, hints
except Exception as e:
logger.warning(f"Story identification error: {e}")
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
raise
return False, ""
# ============================================================
# 第三步:切片提取事实
# ============================================================
def _chunk_text_by_paragraphs(text: str, max_chars: int = 3000, overlap_paragraphs: int = 5) -> list[str]:
"""按自然段切分,在 max_chars 附近找段落结尾切,重叠 overlap_paragraphs 个自然段"""
paragraphs = [p for p in text.split("\n") if p.strip()]
if not paragraphs:
return [text]
if len(text) <= max_chars:
return [text]
chunks = []
i = 0
while i < len(paragraphs):
# 从当前位置开始,累加段落直到超过 max_chars
current = []
current_len = 0
j = i
while j < len(paragraphs):
para = paragraphs[j]
# 超过 max_chars 且已有内容 → 在这个段落前切断(段落结尾切)
if current_len + len(para) > max_chars and current:
break
current.append(para)
current_len += len(para)
j += 1
chunks.append("\n".join(current))
# 到末尾了就结束
if j >= len(paragraphs):
break
# 下一次起点:当前切到的位置 - 重叠段落数
next_i = j - overlap_paragraphs
# 防止死循环确保至少前进1个段落
if next_i <= i:
next_i = i + 1
i = next_i
return chunks
async def _extract_facts_from_chunk(text: str, hints: str) -> dict | None:
"""从对话切片中提取关键信息(事件、情感、原话、人物)"""
prompt = f"""从这段对话中提取关键信息。只输出JSON。
故事主题:{hints}
提取以下内容:
- events: 发生的事件(尽量详细,保留具体细节)
- emotions: 表达的情感
- quotes: 讲述者的原话(保留原话,不要改写,尽量多提取;完全相同的重复语句只保留一条)
- key_people: 提到的人物
{{"events":["事件1","事件2"],"emotions":["情感1","情感2"],"quotes":["原话1","原话2","原话3"],"key_people":["人物1"]}}
对话内容:
```
{text}
```"""
try:
result = await llm_client.chat_json([{"role": "user", "content": prompt}], temperature=0.1)
if not result:
return None
# chat_json 可能返回 list解析错误需要转为 dict
if isinstance(result, list):
for item in result:
if isinstance(item, dict) and ("events" in item or "quotes" in item):
result = item
break
else:
logger.warning(f"chat_json returned list, cannot convert to dict: {result[:3]}")
return None
return result
except Exception as e:
logger.warning(f"Extract facts error: {e}")
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
raise
return None
# ============================================================
# 第四步:重组(纯代码,无需 LLM
# ============================================================
def _fuzzy_dedup_quotes(quotes: list[str]) -> list[str]:
"""
模糊去重:对高度相似的语句只保留第一条。
策略:提取每条语句的核心部分(去掉常见前缀后缀),相似度超过阈值的视为重复。
"""
import re
def normalize(s: str) -> str:
"""提取语句核心,去掉语气词和细微差异"""
s = s.strip()
# 去掉「」包裹
if s.startswith("") and s.endswith(""):
s = s[1:-1]
# 去掉常见前缀
for prefix in ["", "", "", "就是", "然后", "那个", "所以"]:
if s.startswith(prefix):
s = s[len(prefix):].strip()
# 去掉常见尾缀
for suffix in ["", "", "", "", "", ""]:
if s.endswith(suffix) and len(s) > 4:
s = s[:-1]
# 去掉多余空白和重复字(如"我我我"、"的的的"
s = re.sub(r'(.)\1{2,}', r'\1', s)
return s
def similarity(a: str, b: str) -> float:
"""简单的字符级相似度Jaccard on character bigrams"""
if not a or not b:
return 0.0
a_bigrams = set(a[i:i+2] for i in range(len(a)-1))
b_bigrams = set(b[i:i+2] for i in range(len(b)-1))
if not a_bigrams or not b_bigrams:
return 0.0
intersection = a_bigrams & b_bigrams
union = a_bigrams | b_bigrams
return len(intersection) / len(union)
kept = []
kept_normalized = []
for q in quotes:
nq = normalize(q)
is_dup = False
for kn in kept_normalized:
# 短语句用高阈值,长语句用低阈值
threshold = 0.8 if len(nq) < 15 else 0.7
if similarity(nq, kn) >= threshold:
is_dup = True
break
if not is_dup:
kept.append(q)
kept_normalized.append(nq)
removed = len(quotes) - len(kept)
if removed > 0:
logger.info(f" 模糊去重: {len(quotes)} -> {len(kept)} (去除 {removed} 条相似重复)")
return kept
def _merge_facts(all_facts: list) -> list[tuple[str, list, dict]]:
"""
重组素材Reduce
对每个故事段落的所有切片事实进行合并去重:
- 事件按原文顺序保留
- 情感去重
- 原话去重(不限制数量)
- 人物去重
输入: [(speaker, seg, hints, facts_list), ...]
输出: [(speaker, seg, material), ...]
material: {"events": [...], "emotions": [...], "quotes": [...], "key_people": [...], "story_hints": "..."}
"""
merged = []
for i, (speaker, seg, hints, facts_list) in enumerate(all_facts):
all_events, all_emotions, all_quotes, all_people = [], [], [], []
for facts in facts_list:
if isinstance(facts, dict):
all_events.extend(facts.get("events", []))
all_emotions.extend(facts.get("emotions", []))
all_quotes.extend(facts.get("quotes", []))
all_people.extend(facts.get("key_people", []))
# 情感去重(保持顺序)
all_emotions = list(dict.fromkeys(all_emotions))
# 人物去重(保持顺序)
all_people = list(dict.fromkeys(all_people))
# 原话去重(保持顺序,不限制数量)
# 第一轮:精确去重
seen_quotes = set()
unique_quotes = [q for q in all_quotes if q not in seen_quotes and not seen_quotes.add(q)]
# 第二轮:模糊去重(去除重复模式的语句,如释放语句只保留一条)
unique_quotes = _fuzzy_dedup_quotes(unique_quotes)
material = {
"events": all_events,
"emotions": all_emotions,
"quotes": unique_quotes,
"key_people": all_people,
"story_hints": hints,
}
logger.info(f" 重组故事 {i}: {speaker} - 事件:{len(all_events)}, 情感:{len(all_emotions)}, "
f"原话:{len(unique_quotes)}, 人物:{len(all_people)}")
merged.append((speaker, seg, material))
return merged
# ============================================================
# 第五步:生成最终故事
# ============================================================
async def _generate_story(material: dict) -> dict | None:
"""基于素材生成第三人称故事5部分结构化输出"""
prompt = f"""基于素材生成第三人称故事。只输出JSON不要其他内容。
素材:
主题:{material['story_hints']}
事件:{json.dumps(material['events'], ensure_ascii=False)}
情感:{json.dumps(material['emotions'], ensure_ascii=False)}
原话:{json.dumps(material['quotes'], ensure_ascii=False)}
人物:{json.dumps(material['key_people'], ensure_ascii=False)}
写作手法(非常重要):
【双时空结构】
故事有两个时空层:
- "当下时空":学员在课堂上与卢慧老师的对话——这是故事的框架,用对话体呈现,保留师生互动的完整脉络
- "过往时空":学员讲述的过去经历——这是故事的核心,用"场景重建"手法呈现,让读者穿越到那个时空
【场景重建手法(用于"过往的故事"部分)】
当学员讲述过去的经历时,不要写成"她说她小时候…"这种转述体,而是直接把读者拉进那个时空:
- 直接切入场景,用过去时态叙述。比如不要写"文晔回忆说她的儿子15岁去了美国",而要写"十五岁那年,文晔的儿子独自登上了飞往美国的航班"
- 用五感描写构建画面:她看到了什么、听到了什么、身体感受到了什么(比如"手心出汗""脚发凉"这些身体反应要融入场景中)
- 用环境细节暗示情绪:不要直接说"她很痛苦",而是通过场景让读者感受到痛苦
- 关键情感爆发处用直接引语(「」包裹原话),其他地方用间接引语和叙述自然融合
- 场景之间要有过渡,比如"然而,命运的转折发生在前年冬天——"
【对话体手法(用于"当下的困境""转变"部分)】
- 保留师生对话的现场感,写出谁说了什么、对方如何回应
- 在对话中穿插学员的内心活动、身体反应、情绪变化
- 老师的引导语要完整保留,这是疗愈过程的关键
【通用要求】
- 每个事件都要展开写,写出具体过程和细节,不要一句话概括
- 事件之间要有过渡和因果逻辑,不能跳跃
- 情感变化要写出过程:从什么情感转变到什么情感,中间经历了什么
- 不编造素材中没有的内容,但可以根据素材合理补充场景细节(如环境、氛围)
- 每个部分充分展开,不要限制长度,越详细越好
- 原文金句数组中的每条金句必须用英文双引号包裹,如 ["金句1", "金句2"],不要用「」替代双引号
禁止事项:
- 禁止用一句话概括多个事件(如"她倾诉了挣扎""她经历了疗愈过程"
- 禁止省略素材中的具体细节
- 禁止省略老师的引导和回应
- 禁止事件之间跳跃,要保持时间顺序和因果逻辑
- 禁止过度总结或抽象化描述
- 禁止在"过往的故事"部分全程使用转述体("她说…她回忆…"),必须用场景重建手法让读者身临其境
- 禁止为了简洁而牺牲内容的丰富性
输出格式:{{"title":"标题","summary":"80字摘要","tags":["标签"],"speaker_nickname":"昵称","content":{{"讲述者":"...","当下的困境":"...","过往的故事":"...","转变":"...","原文金句":["原话1","原话2","原话3"]}}}}"""
try:
result = await llm_client.chat_json(
[{"role": "user", "content": prompt}],
temperature=0.7,
)
if not result:
return None
# chat_json 可能返回 list解析错误需要转为 dict
if isinstance(result, list):
for item in result:
if isinstance(item, dict) and "title" in item:
result = item
break
else:
logger.warning(f"chat_json returned list, cannot convert to dict: {result[:3]}")
return None
# 解析 content 纯文本为结构化字段
raw_content = result.get("content", "")
if isinstance(raw_content, str) and "|||" in raw_content:
parts = raw_content.split("|||")
content_dict = {}
keys = ["讲述者", "当下的困境", "过往的故事", "转变", "原文金句"]
for idx, key in enumerate(keys):
content_dict[key] = parts[idx].strip() if idx < len(parts) else ""
result["content"] = content_dict
elif isinstance(raw_content, dict):
for key in ["讲述者", "当下的困境", "过往的故事", "转变", "原文金句"]:
if key not in raw_content:
raw_content[key] = ""
result["content"] = raw_content
else:
result["content"] = {"完整故事": str(raw_content)}
if not result.get("tags"):
result["tags"] = []
return result
except Exception as e:
logger.warning(f"Generate story error: {e}")
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
raise
return None
# ============================================================
# 工具函数
# ============================================================
def _segment_to_text(segment_lines: list) -> str:
"""将段落行列表转为对话文本"""
parts = []
for line in segment_lines:
speaker = line.get("speaker", "")
content = line.get("content", "")
if speaker:
parts.append(f"{speaker}: {content}")
else:
parts.append(content)
return "\n".join(parts)
def _increment_files_completed(state: TaskState):
"""增加已完成文件计数"""
p = state.progress
files_completed = p.get("files_completed", 0) + 1
files_total = p.get("files_total", 1)
state.update(files_completed=files_completed)
def _estimate_duration(segment_lines: list) -> float:
"""根据时间戳估算故事时长(分钟)"""
timestamps = []
for line in segment_lines:
if isinstance(line, dict) and line.get("timestamp"):
try:
parts = line["timestamp"].split(":")
if len(parts) >= 2:
minutes = int(parts[-2]) + int(parts[-1]) / 60
if len(parts) == 3:
minutes += int(parts[0]) * 60
timestamps.append(minutes)
except (ValueError, IndexError):
continue
if len(timestamps) >= 2:
return round(max(timestamps) - min(timestamps), 1)
return 0
def _confidence_level(confidence: float) -> str:
if confidence >= 0.8:
return "high"
elif confidence >= 0.6:
return "medium"
return "low"
def _format_story_content(content_dict: dict) -> str:
"""将 LLM 返回的结构化故事内容转为可读的 Markdown 格式"""
parts = []
# 按顺序输出各个章节
sections = ["讲述者", "当下的困境", "过往的故事", "转变"]
for section_name in sections:
if content_dict.get(section_name):
parts.append(f"## {section_name}\n\n{content_dict[section_name]}")
# 原文金句
quotes = content_dict.get("原文金句", [])
if not quotes:
quotes = content_dict.get("金句摘录", [])
if quotes and isinstance(quotes, list):
parts.append("## 原文金句")
for q in quotes:
parts.append(f"\n> {q}")
# 兼容旧格式:故事梗概
if content_dict.get("故事梗概") and "讲述者" not in content_dict:
parts.insert(0, f"## 故事梗概\n\n{content_dict['故事梗概']}")
# 兼容旧格式:关键要素
elements = content_dict.get("关键要素", [])
if elements and isinstance(elements, list) and "讲述者" not in content_dict:
parts.append("## 关键要素")
for elem in elements:
if isinstance(elem, dict):
title = elem.get("标题", "")
desc = elem.get("描述", "")
parts.append(f"\n### {title}\n\n{desc}")
# 如果上面都没匹配到,按通用 key-value 格式输出
if not parts:
for key, value in content_dict.items():
if isinstance(value, str):
parts.append(f"## {key}\n\n{value}")
elif isinstance(value, list):
parts.append(f"## {key}")
for item in value:
if isinstance(item, dict):
for k, v in item.items():
parts.append(f"\n**{k}**{v}")
else:
parts.append(f"\n- {item}")
return "\n\n".join(parts) if parts else json.dumps(content_dict, ensure_ascii=False)

View File

@@ -0,0 +1,63 @@
import os
import uuid
import shutil
from fastapi import UploadFile
from app.config import settings
class FileService:
"""文件上传、存储、删除"""
def __init__(self):
os.makedirs(settings.UPLOAD_DIR, exist_ok=True)
os.makedirs(os.path.join(settings.UPLOAD_DIR, "transcripts"), exist_ok=True)
os.makedirs(os.path.join(settings.UPLOAD_DIR, "persons"), exist_ok=True)
os.makedirs(settings.EXPORT_DIR, exist_ok=True)
async def save_transcript(self, file: UploadFile) -> dict:
"""保存上传的文字稿"""
file_id = str(uuid.uuid4())
ext = os.path.splitext(file.filename)[1]
save_name = f"{file_id}{ext}"
save_dir = os.path.join(settings.UPLOAD_DIR, "transcripts")
save_path = os.path.join(save_dir, save_name)
content = await file.read()
with open(save_path, "wb") as f:
f.write(content)
return {
"id": file_id,
"filename": file.filename,
"file_path": save_path,
"file_size": len(content),
}
async def save_person(self, file: UploadFile) -> dict:
"""保存上传的讲述者信息"""
file_id = str(uuid.uuid4())
ext = os.path.splitext(file.filename)[1]
save_name = f"{file_id}{ext}"
save_dir = os.path.join(settings.UPLOAD_DIR, "persons")
save_path = os.path.join(save_dir, save_name)
content = await file.read()
with open(save_path, "wb") as f:
f.write(content)
return {
"id": file_id,
"filename": file.filename,
"file_path": save_path,
"file_size": len(content),
}
def delete_file(self, file_path: str) -> bool:
"""删除文件"""
if os.path.exists(file_path):
os.remove(file_path)
return True
return False
file_service = FileService()

View File

@@ -0,0 +1,287 @@
import json
import logging
import re
import time
from openai import AsyncOpenAI
import anthropic
from app.config import settings
logger = logging.getLogger(__name__)
class LLMClient:
"""LLM 客户端,支持 OpenAI 和 Anthropic SDK"""
def __init__(self):
self._openai_client = None
self._anthropic_client = None
self._api_key = settings.LLM_API_KEY
self._base_url = settings.LLM_BASE_URL
self._model = settings.LLM_MODEL
self._provider = settings.LLM_PROVIDER
def _get_openai_client(self) -> AsyncOpenAI:
if self._openai_client is None:
self._openai_client = AsyncOpenAI(
api_key=self._api_key,
base_url=self._base_url,
max_retries=0,
timeout=300.0,
)
return self._openai_client
def _get_anthropic_client(self) -> anthropic.AsyncAnthropic:
if self._anthropic_client is None:
# MiniMax Anthropic 端点
base_url = self._base_url.replace("/v1", "/anthropic").replace("api.minimax.chat", "api.minimaxi.com")
if not base_url.endswith("/anthropic"):
# 如果 base_url 不是标准格式,手动拼接
base_url = "https://api.minimaxi.com/anthropic"
self._anthropic_client = anthropic.AsyncAnthropic(
api_key=self._api_key,
base_url=base_url,
timeout=300.0,
)
return self._anthropic_client
def _use_anthropic(self) -> bool:
"""判断是否使用 Anthropic SDK"""
return False # MiniMax thinking 占用太多 token统一用 OpenAI SDK
def update_config(self, base_url: str = None, api_key: str = None, model: str = None):
"""更新配置(设置页面保存后调用)"""
if base_url:
self._base_url = base_url
if api_key:
self._api_key = api_key
if model:
self._model = model
# 重置客户端
self._openai_client = None
self._anthropic_client = None
# 更新 provider 判断
if base_url:
self._provider = "minimax" if "minimax" in base_url.lower() else "openai"
async def chat(self, messages: list[dict], temperature: float = None, max_tokens: int = 16384, timeout: float = 300.0) -> str:
"""发送对话请求,返回文本内容(不含思考过程)"""
if temperature is None:
temperature = settings.TEMPERATURE
if self._use_anthropic():
return await self._chat_anthropic(messages, temperature, max_tokens, timeout)
else:
result = await self._chat_openai(messages, temperature, max_tokens, timeout)
return result["content"]
async def chat_detail(self, messages: list[dict], temperature: float = None, max_tokens: int = 16384, timeout: float = 300.0) -> dict:
"""发送对话请求返回详细信息content + reasoning + finish_reason + usage + elapsed"""
if temperature is None:
temperature = settings.TEMPERATURE
if self._use_anthropic():
# Anthropic 暂不支持 detail
content = await self._chat_anthropic(messages, temperature, max_tokens, timeout)
return {"content": content, "reasoning_content": "", "finish_reason": "stop", "usage": {}, "elapsed": 0}
else:
return await self._chat_openai(messages, temperature, max_tokens, timeout)
async def _chat_openai(self, messages: list[dict], temperature: float, max_tokens: int, timeout: float = 300.0) -> dict:
"""OpenAI SDK 调用,返回详细信息 dict"""
client = self._get_openai_client()
t0 = time.time()
response = await client.chat.completions.create(
model=self._model,
messages=messages,
temperature=temperature,
max_tokens=max_tokens,
timeout=timeout,
)
elapsed = time.time() - t0
content = response.choices[0].message.content or ""
# 提取 reasoning_contentthinking
reasoning = ""
msg = response.choices[0].message
if hasattr(msg, 'reasoning_content') and msg.reasoning_content:
reasoning = msg.reasoning_content
# 有些 SDK 把 reasoning 放在 model_extra 里
if not reasoning and hasattr(msg, 'model_extra') and isinstance(msg.model_extra, dict):
reasoning = msg.model_extra.get("reasoning_content", "")
# 检查是否有多个 choice
if len(response.choices) > 1:
print(f"[LLM] 警告: 返回了 {len(response.choices)} 个 choices")
# 检查 finish_reason
reason = response.choices[0].finish_reason if response.choices else "unknown"
if reason != "stop":
print(f"[LLM] 警告: finish_reason={reason}, 可能被截断")
# usage 信息
usage = {}
if hasattr(response, 'usage') and response.usage:
usage = {
"prompt_tokens": response.usage.prompt_tokens,
"completion_tokens": response.usage.completion_tokens,
"total_tokens": response.usage.total_tokens,
}
if hasattr(response.usage, 'completion_tokens_details') and response.usage.completion_tokens_details:
details = response.usage.completion_tokens_details
usage["reasoning_tokens"] = getattr(details, 'reasoning_tokens', 0) or 0
# 保存完整响应到文件,供调试
try:
resp_dict = response.model_dump() if hasattr(response, 'model_dump') else str(response)
debug_path = f"/tmp/llm_response_{id(response) % 100000}.json"
with open(debug_path, "w", encoding="utf-8") as _f:
json.dump(resp_dict, _f, ensure_ascii=False, indent=2, default=str)
print(f"[LLM] 完整响应已保存: {debug_path}")
except Exception:
pass
return {
"content": content,
"reasoning_content": reasoning,
"finish_reason": reason,
"usage": usage,
"elapsed": round(elapsed, 2),
}
async def _chat_anthropic(self, messages: list[dict], temperature: float, max_tokens: int, timeout: float = 300.0) -> str:
"""Anthropic SDK 调用 - thinking 和 text 自动分离"""
client = self._get_anthropic_client()
# 转换消息格式OpenAI -> Anthropic
system_msg = ""
anthropic_messages = []
for msg in messages:
role = msg.get("role", "user")
content = msg.get("content", "")
if role == "system":
system_msg = content
elif role in ("user", "assistant"):
anthropic_messages.append({"role": role, "content": content})
kwargs = {
"model": self._model,
"max_tokens": max_tokens,
"messages": anthropic_messages,
"thinking": {"type": "disabled"}, # 禁用 thinking避免占用 token
}
if system_msg:
kwargs["system"] = system_msg
if temperature is not None:
kwargs["temperature"] = temperature
response = await client.messages.create(**kwargs)
# 从 response 中提取文本内容thinking 自动分离在 block.thinking 中
text_parts = []
for block in response.content:
if hasattr(block, 'text') and block.type == 'text':
text_parts.append(block.text)
# thinking 内容自动忽略
result = "\n".join(text_parts)
return result
async def chat_json(self, messages: list[dict], temperature: float = None, max_tokens: int = 16384) -> dict:
"""发送对话请求,期望返回 JSON"""
raw = await self.chat(messages, temperature, max_tokens=max_tokens)
return _parse_json(raw)
async def test_connection(self) -> dict:
"""测试 LLM 连接"""
try:
response = await self.chat([{"role": "user", "content": "请回复:连接成功"}])
return {"success": True, "message": f"连接成功,模型回复:{response[:50]}"}
except Exception as e:
return {"success": False, "message": f"连接失败:{str(e)}"}
def _parse_json(raw: str) -> dict:
"""健壮的 JSON 解析,处理 LLM 返回的各种格式问题"""
print(f"[_parse_json] raw长度={len(raw)}, 前100字={raw[:100]}")
print(f"[_parse_json] raw后100字={raw[-100:]}")
raw = raw.strip()
# 去掉 markdown 代码块
if raw.startswith("```"):
raw = re.sub(r'^```\w*\n?', '', raw)
raw = re.sub(r'\n?```$', '', raw)
raw = raw.strip()
# 找 JSON 结构的位置
# 优先找第一个 { }JSON 对象),用括号匹配确保完整
first_brace = raw.find('{')
last_bracket = raw.rfind('[')
start = -1
if first_brace >= 0:
start = first_brace
# 从第一个 { 开始,找匹配的 }
depth = 0
end = -1
for i in range(start, len(raw)):
if raw[i] == '{':
depth += 1
elif raw[i] == '}':
depth -= 1
if depth == 0:
end = i + 1
break
if end <= start:
start = -1
elif last_bracket >= 0:
start = last_bracket
end = raw.rfind(']') + 1
if end <= start:
start = -1
if start < 0:
start = raw.find('{')
end = raw.rfind('}') + 1
if start < 0 or end <= start:
raise ValueError(f"无法找到 JSON: {raw[:200]}")
json_str = raw[start:end]
# 修复常见问题中文引号替换为英文引号在JSON值内部的需要转义
# 先把 JSON 结构中的英文引号保护起来,再替换中文引号
json_str = json_str.replace('\u201c', '\u201c').replace('\u201d', '\u201d') # 先保持不变
json_str = json_str.replace('\u2018', "'").replace('\u2019', "'")
json_str = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', '', json_str)
json_str = re.sub(r',\s*([}\]])', r'\1', json_str)
try:
return json.loads(json_str)
except json.JSONDecodeError as e:
# 如果失败,尝试将中文引号替换为转义的英文引号
json_str2 = json_str.replace('\u201c', '\\"').replace('\u201d', '\\"')
try:
return json.loads(json_str2)
except json.JSONDecodeError:
pass
# 尝试把换行替换为 \n
json_str2 = json_str.replace('\n', '\\n').replace('\r', '\\r')
json_str2 = json_str2.replace('\\\\n', '\\n')
try:
return json.loads(json_str2)
except json.JSONDecodeError:
pass
# 尝试自动补全缺失的闭合括号(模型输出被截断时常见)
for extra in ['}', '}}', '}}}', '}]', '}]}']:
try:
return json.loads(json_str + extra)
except json.JSONDecodeError:
continue
raise ValueError(f"无法解析 JSON: {json_str[:300]}")
llm_client = LLMClient()

View File

@@ -0,0 +1,220 @@
import re
class Preprocessor:
"""
预处理器:从原始对话中提取包含故事的段落
核心思路:
1. 过滤纯课堂管理内容(签到、纪律等)
2. 识别"个案对话段"——老师与某位学员的深度互动
3. 保留完整对话上下文(师生互动),不过度过滤
4. 按话题边界切分为合理的段落
"""
# 课堂管理关键词(出现这些词的整行直接过滤)
MANAGEMENT_KEYWORDS = ["签到", "打卡", "禁言", "扣三个一", "进组", "会议室链接"]
# 纯课堂管理发言模式(整条消息都是管理内容)
MANAGEMENT_PATTERNS = [
re.compile(r"大家.*签到"),
re.compile(r"请.*进组"),
re.compile(r"会议室"),
re.compile(r"纪律"),
re.compile(r"昵称.*改为"),
re.compile(r"欢迎.*进入"),
]
# 老师关键词
TEACHER_KEYWORDS = ["卢慧老师", "静静", "督导"]
def is_teacher(self, speaker: str) -> bool:
"""判断是否为老师/助教"""
return any(kw in speaker for kw in self.TEACHER_KEYWORDS)
def is_management_line(self, line: dict) -> bool:
"""判断是否为课堂管理行"""
content = line.get("content", "")
# 短内容 + 管理关键词 = 管理行
if len(content) < 30:
return any(kw in content for kw in self.MANAGEMENT_KEYWORDS)
return any(p.search(content) for p in self.MANAGEMENT_PATTERNS)
def preprocess(self, paragraphs: list[dict]) -> list[dict]:
"""
完整预处理流程:
1. 过滤课堂管理
2. 识别个案对话段(保留师生互动)
3. 按主要讲述者拆分(每个段落只包含一个主要讲述者)
"""
# 第一步:过滤课堂管理行
filtered = [p for p in paragraphs if not self.is_management_line(p)]
# 第二步:识别个案对话段
segments = self._extract_story_segments(filtered)
# 第三步:按主要讲述者拆分
segments = self._split_by_speaker(segments)
return segments
def _split_by_speaker(self, segments: list[list[dict]]) -> list[list[dict]]:
"""
将每个段落按讲述者拆分成子段落。
同一个讲述者的所有对话(即使中间隔了其他人的插话)收集在一起,
老师发言分配给前一个活跃的学员作为上下文。
"""
result = []
for seg in segments:
# 按讲述者收集对话,保留老师发言作为上下文
speaker_blocks = {} # speaker -> [lines]
last_student = None
for line in seg:
speaker = line.get("speaker", "")
if self.is_teacher(speaker):
# 老师发言归入前一个活跃的学员
if last_student and last_student in speaker_blocks:
speaker_blocks[last_student].append(line)
else:
if speaker not in speaker_blocks:
speaker_blocks[speaker] = []
speaker_blocks[speaker].append(line)
last_student = speaker
# 每个讲述者的对话作为一个子段落
for speaker, lines in speaker_blocks.items():
student_content = sum(
len(l.get("content", ""))
for l in lines
if not self.is_teacher(l.get("speaker", ""))
)
# 过滤掉内容太少的(可能是插话而非讲述者)
if student_content > 80 and len(lines) >= 3:
# 计算该讲述者在自己段落中的字数占比
# 如果老师发言占比超过50%,说明这主要是老师在讲课,不是学员故事
teacher_chars = sum(
len(l.get("content", ""))
for l in lines
if self.is_teacher(l.get("speaker", ""))
)
total_chars = student_content + teacher_chars
if total_chars > 0 and teacher_chars / total_chars > 0.5:
continue # 老师讲太多,跳过
result.append(lines)
return result
def _extract_story_segments(self, paragraphs: list[dict]) -> list[list[dict]]:
"""
从过滤后的对话中提取可能包含故事的段落。
策略:
- 找到学员的"长发言">50字作为故事线索
- 向前向后扩展,包含老师的引导和互动
- 直到遇到另一位学员的长发言或长时间沉默
- 最小段落 10 行,最大段落 200 行
"""
segments = []
i = 0
used = set() # 已分配的行索引
while i < len(paragraphs):
p = paragraphs[i]
# 跳过老师单独发言(非互动)
if self.is_teacher(p.get("speaker", "")):
i += 1
continue
# 跳过短发言(< 30字不太可能开启故事
if len(p.get("content", "")) < 30:
i += 1
continue
# 跳过已分配的行
if p.get("index", i) in used:
i += 1
continue
# 找到一个学员发言,开始扩展
segment_start = i
segment_end = i
# 向前扩展(找老师之前的引导)
expand_start = max(0, i - 5)
for j in range(i - 1, expand_start - 1, -1):
if j in used:
break
prev = paragraphs[j]
# 如果前一个是老师发言,包含
if self.is_teacher(prev.get("speaker", "")):
segment_start = j
# 如果前一个是同一学员的发言,包含
elif prev.get("speaker") == p.get("speaker"):
segment_start = j
else:
# 其他学员发言,停止
break
# 向后扩展(找老师的回应和后续互动)
current_speaker = p.get("speaker", "")
for j in range(i + 1, min(len(paragraphs), i + 200)):
if j in used:
segment_end = j - 1
break
curr = paragraphs[j]
curr_speaker = curr.get("speaker", "")
curr_content = curr.get("content", "")
# 老师发言:包含(互动引导)
if self.is_teacher(curr_speaker):
segment_end = j
continue
# 同一学员继续发言:包含
if curr_speaker == current_speaker:
segment_end = j
continue
# 其他学员短回应(< 20字可能是插话跳过但不终止
if len(curr_content) < 20:
continue
# 其他学员长发言:新的话题开始,终止
break
# 提取段落
segment = paragraphs[segment_start:segment_end + 1]
# 过滤掉太短的段落(< 8行
if len(segment) >= 8:
# 检查是否有足够的学员实质性内容(非纯疗愈练习)
student_content_len = sum(
len(s.get("content", ""))
for s in segment
if not self.is_teacher(s.get("speaker", ""))
and len(s.get("content", "")) > 15
)
# 学员实质内容 > 100字才认为是故事
if student_content_len > 100:
segments.append(segment)
for s in segment:
used.add(s.get("index", 0))
i = segment_end + 1
continue
i += 1
return segments
def is_teacher(speaker: str) -> bool:
"""判断是否为老师/助教发言"""
teacher_keywords = ["卢慧老师", "静静", "督导", "老师"]
return any(kw in speaker for kw in teacher_keywords)
preprocessor = Preprocessor()

95
backend/app/state.py Normal file
View File

@@ -0,0 +1,95 @@
import asyncio
import json
import uuid
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class TaskState:
"""单个提取任务的状态"""
task_id: str = ""
is_running: bool = False
cancel_event: asyncio.Event = field(default_factory=asyncio.Event)
progress: dict = field(default_factory=lambda: {
"percent": 0,
"status": "idle",
"current_file": "",
"action": "",
"files_total": 0,
"files_completed": 0,
"stories_found": 0,
"phase": "",
"phase_percent": 0,
"phase_detail": "",
})
_queue: asyncio.Queue = field(default_factory=asyncio.Queue)
def update(self, **kwargs):
"""更新进度,同时推送到 SSE 队列"""
self.progress.update(kwargs)
try:
self._queue.put_nowait(self.progress.copy())
except asyncio.QueueFull:
pass
async def get_update(self) -> dict:
"""SSE 消费端:等待新的进度更新"""
return await self._queue.get()
def reset(self):
"""重置状态"""
self.is_running = False
self.cancel_event = asyncio.Event()
self.progress = {
"percent": 0,
"status": "idle",
"current_file": "",
"action": "",
"files_total": 0,
"files_completed": 0,
"stories_found": 0,
"phase": "",
"phase_percent": 0,
"phase_detail": "",
}
self._queue = asyncio.Queue()
class TaskManager:
"""多任务管理器:支持多个用户同时提取"""
def __init__(self):
self._tasks: dict[str, TaskState] = {}
def create_task(self) -> TaskState:
"""创建新任务"""
task_id = uuid.uuid4().hex[:12]
state = TaskState(task_id=task_id)
self._tasks[task_id] = state
return state
def get_task(self, task_id: str) -> Optional[TaskState]:
"""获取任务状态"""
return self._tasks.get(task_id)
def remove_task(self, task_id: str):
"""清理已完成的任务"""
self._tasks.pop(task_id, None)
def cleanup_old_tasks(self):
"""清理所有非运行中的任务"""
to_remove = [
tid for tid, state in self._tasks.items()
if not state.is_running
]
for tid in to_remove:
self._tasks.pop(tid, None)
# 全局任务管理器
task_manager = TaskManager()
# 向后兼容:默认任务(单用户场景)
extraction_state = TaskState()