GuangYu/Story-extractor
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Nelson
2026-04-24 16:02:16 +08:00
commit 1d6f0cc370
66 changed files with 9990 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
backend/app/services/__init__.py Normal file
View File

166
backend/app/services/docx_generator.py Normal file
View File

@@ -0,0 +1,166 @@
import os
import re
from docx import Document
from docx.shared import Pt, Inches, Cm, RGBColor
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.oxml.ns import qn
def generate_person_doc(person_name: str, person_info: str, photo_path: str,
stories: list[dict], output_path: str):
"""
为一位讲述者生成最终 DOCX 文档
person_info: 个人信息文本
photo_path: 照片路径（可为空）
stories: [{title, content}]
"""
doc = Document()
# 设置默认字体
style = doc.styles['Normal']
font = style.font
font.name = '宋体'
font.size = Pt(12)
style.element.rPr.rFonts.set(qn('w:eastAsia'), '宋体')
# 照片
if photo_path and os.path.exists(photo_path):
try:
doc.add_picture(photo_path, width=Inches(1.5))
last_paragraph = doc.paragraphs[-1]
last_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
except Exception:
pass
# 姓名
name_para = doc.add_paragraph()
name_para.alignment = WD_ALIGN_PARAGRAPH.CENTER
name_run = name_para.add_run(person_name)
name_run.font.size = Pt(18)
name_run.font.bold = True
name_run.font.name = '黑体'
name_run.element.rPr.rFonts.set(qn('w:eastAsia'), '黑体')
# 个人信息
if person_info:
info_para = doc.add_paragraph()
info_para.alignment = WD_ALIGN_PARAGRAPH.CENTER
info_run = info_para.add_run(person_info.strip())
info_run.font.size = Pt(11)
info_run.font.color.rgb = RGBColor(0x66, 0x66, 0x66)
# 分隔线
doc.add_paragraph("─" * 30).alignment = WD_ALIGN_PARAGRAPH.CENTER
# 故事
for i, story in enumerate(stories):
# 故事标题
title_para = doc.add_paragraph()
title_run = title_para.add_run(story.get("title", f"故事 {i + 1}"))
title_run.font.size = Pt(14)
title_run.font.bold = True
title_run.font.name = '黑体'
title_run.element.rPr.rFonts.set(qn('w:eastAsia'), '黑体')
# 故事梗概
summary = story.get("summary", "")
if summary:
p = doc.add_paragraph()
run = p.add_run("【故事梗概】")
run.font.size = Pt(11)
run.font.bold = True
run.font.color.rgb = RGBColor(0x33, 0x33, 0x99)
p = doc.add_paragraph(summary.strip())
p.paragraph_format.first_line_indent = Cm(0.74)
p.paragraph_format.line_spacing = 1.5
for run in p.runs:
run.font.size = Pt(11)
# 关键词标签
tags = story.get("tags", [])
if tags:
p = doc.add_paragraph()
run = p.add_run("【关键词标签】")
run.font.size = Pt(11)
run.font.bold = True
run.font.color.rgb = RGBColor(0x33, 0x33, 0x99)
tag_text = "、".join(tags)
p = doc.add_paragraph(tag_text)
p.paragraph_format.line_spacing = 1.5
for run in p.runs:
run.font.size = Pt(10)
run.font.color.rgb = RGBColor(0x66, 0x66, 0x66)
# 故事内容（支持 Markdown 格式）
content = story.get("content", "")
if content:
_add_markdown_content(doc, content)
# 多个故事之间加分隔
if i < len(stories) - 1:
doc.add_paragraph("─" * 30).alignment = WD_ALIGN_PARAGRAPH.CENTER
# 保存
os.makedirs(os.path.dirname(output_path), exist_ok=True)
doc.save(output_path)
return output_path
def _add_markdown_content(doc, content: str):
"""将 Markdown 格式的内容添加到 DOCX，支持标题、引用、列表等"""
lines = content.split("\n")
i = 0
while i < len(lines):
line = lines[i].strip()
if not line:
i += 1
continue
# ## 二级标题
if line.startswith("## "):
title = line[3:].strip()
p = doc.add_paragraph()
run = p.add_run(title)
run.font.size = Pt(13)
run.font.bold = True
run.font.name = '黑体'
run.element.rPr.rFonts.set(qn('w:eastAsia'), '黑体')
p.paragraph_format.space_before = Pt(12)
# ### 三级标题
elif line.startswith("### "):
title = line[4:].strip()
p = doc.add_paragraph()
run = p.add_run(title)
run.font.size = Pt(12)
run.font.bold = True
run.font.name = '黑体'
run.element.rPr.rFonts.set(qn('w:eastAsia'), '黑体')
p.paragraph_format.space_before = Pt(6)
# > 引用
elif line.startswith("> "):
quote_text = line[2:].strip()
p = doc.add_paragraph()
p.paragraph_format.left_indent = Cm(1)
run = p.add_run(quote_text)
run.font.italic = True
run.font.color.rgb = RGBColor(0x55, 0x55, 0x55)
run.font.size = Pt(11)
# - 列表项
elif line.startswith("- "):
item_text = line[2:].strip()
p = doc.add_paragraph(style='List Bullet')
run = p.add_run(item_text)
run.font.size = Pt(12)
# 普通段落
else:
p = doc.add_paragraph(line)
p.paragraph_format.first_line_indent = Cm(0.74)
p.paragraph_format.line_spacing = 1.5
i += 1

93
backend/app/services/docx_parser.py Normal file
View File

@@ -0,0 +1,93 @@
import re
from docx import Document
def parse_transcript(file_path: str) -> list[dict]:
"""
解析直播回放文字稿 DOCX
格式：发言者(时间戳): 内容
返回：[{index, speaker, timestamp, content, raw_text}]
"""
doc = Document(file_path)
paragraphs = []
for i, para in enumerate(doc.paragraphs):
text = para.text.strip()
if not text:
continue
# 解析格式：发言者(时间戳): 内容
match = re.match(r'^(.+?)\((\d{2}:\d{2}:\d{2})\)\s*[:：]\s*(.*)', text)
if match:
speaker = match.group(1).strip()
timestamp = match.group(2)
content = match.group(3).strip()
else:
# 尝试格式：发言者: 内容
match2 = re.match(r'^(.+?)\s*[:：]\s*(.*)', text)
if match2:
speaker = match2.group(1).strip()
timestamp = ""
content = match2.group(2).strip()
else:
speaker = ""
timestamp = ""
content = text
paragraphs.append({
"index": i,
"speaker": speaker,
"timestamp": timestamp,
"content": content,
"raw_text": text,
})
return paragraphs
def is_teacher(speaker: str) -> bool:
"""判断是否为老师/助教发言"""
teacher_keywords = ["卢慧老师", "静静", "督导", "老师"]
return any(kw in speaker for kw in teacher_keywords)
def parse_person_info(file_path: str) -> dict:
"""
解析讲述者个人信息 DOCX
返回：{name, info_text, photos: [path]}
"""
doc = Document(file_path)
texts = []
photos = []
# 提取文本
for para in doc.paragraphs:
text = para.text.strip()
if text:
texts.append(text)
# 提取图片
import os
photo_dir = os.path.join(os.path.dirname(file_path), "photos")
os.makedirs(photo_dir, exist_ok=True)
for i, rel in enumerate(doc.part.rels.values()):
if "image" in rel.reltype:
image = rel.target_part
photo_name = f"{os.path.splitext(os.path.basename(file_path))[0]}_photo_{i}.{image.content_type.split('/')[-1]}"
photo_path = os.path.join(photo_dir, photo_name)
with open(photo_path, "wb") as f:
f.write(image.blob)
photos.append(photo_path)
# 从文件名提取姓名
filename = os.path.basename(file_path)
name = re.sub(r'[-_].*$', '', filename).strip()
# 去掉扩展名
name = os.path.splitext(name)[0].strip()
return {
"name": name,
"info_text": "\n".join(texts),
"photos": photos,
}

957
backend/app/services/extractor.py Normal file
View File

@@ -0,0 +1,957 @@
"""
故事提取器 - v4 五步流水线策略
第一步：代码预处理（识别老师→过滤杂音→时间范围截取→过滤短段落/非主讲述人）
第二步：判断故事 + 提取事实（短段落直接提取，长段落先判断再切片提取）
第三步：切片提取事实（按自然段切分，并行提取）
第四步：重组（纯代码，事件/情感/原话/人物去重）
第五步：生成最终故事（基于素材生成5部分结构化故事）
"""
import json
import asyncio
import logging
from sqlalchemy import select
from app.database import async_session
from app.models.story import Story
from app.models.transcript import Transcript
from app.services.docx_parser import parse_transcript
from app.services.preprocessor import Preprocessor
from app.services.llm_client import llm_client
from app.state import TaskState
logger = logging.getLogger(__name__)
# 并发控制
MAX_CONCURRENT_FILES = 3
MAX_CONCURRENT_SEGMENTS = 5
# 短段落阈值（字数）：<=4000 字直接提取事实，>4000 字先判断是否有故事
SHORT_THRESHOLD = 4000
# 切片参数
CHUNK_MAX_CHARS = 3000
CHUNK_OVERLAP_PARAGRAPHS = 5
# ============================================================
# 主入口
# ============================================================
async def run_extraction(state: TaskState):
"""主提取流程：文件间串行（避免 API 限流），文件内各步骤并行"""
state.is_running = True
try:
async with async_session() as db:
result = await db.execute(
select(Transcript).where(Transcript.status == "pending")
)
transcripts = result.scalars().all()
if not transcripts:
state.update(status="completed", percent=100)
return
file_total = len(transcripts)
state.update(
status="running", percent=0,
files_total=file_total, files_completed=0,
stories_found=0,
current_file="", action=f"共发现 {file_total} 份文字稿，开始处理...",
phase="preprocess", phase_percent=0, phase_detail="准备中",
)
total_stories = [0]
# 文件间串行处理（避免 API 限流）
for file_index, transcript in enumerate(transcripts):
await _process_file(state, file_index, file_total, transcript, total_stories)
state.update(
status="completed", percent=100,
files_completed=file_total,
stories_found=total_stories[0],
current_file="", action="全部处理完成！",
phase="preprocess", phase_percent=100, phase_detail="全部完成",
)
state.is_running = False
except asyncio.CancelledError:
state.update(status="stopped", action="提取已停止")
state.is_running = False
except Exception as e:
logger.exception("Extraction failed")
state.update(status="error", action=str(e))
state.is_running = False
# ============================================================
# 单文件处理（五步流水线）
# ============================================================
async def _process_file(state: TaskState, file_index: int, file_total: int,
transcript: Transcript, total_stories: list):
"""处理单个文件 - 五步流水线"""
try:
async with async_session() as db:
t = await db.get(Transcript, transcript.id)
if not t:
_increment_files_completed(state)
return
t.status = "processing"
await db.commit()
# 计算该文件在总进度中的权重区间
# 预处理 0-5%, 判断+提取 5-30%, 切片提取 30-55%, 重组 55-60%, 生成 60-95%, 完成 100%
# 每个文件占 95/file_total 的权重
file_weight = 95.0 / max(file_total, 1)
file_start_percent = file_index * file_weight
def _file_percent(phase_start_ratio, phase_end_ratio, inner_ratio=1.0):
"""计算当前文件在总进度中的百分比"""
phase_range = phase_end_ratio - phase_start_ratio
return min(99, int(file_start_percent + file_weight * (phase_start_ratio + phase_range * inner_ratio)))
state.update(
current_file=t.filename, action="正在读取文件...",
phase="preprocess", phase_percent=0, phase_detail="读取文件",
)
# 解析 DOCX
lines = await asyncio.to_thread(parse_transcript, t.file_path)
if not lines:
t.status = "completed"
t.error_message = "文件为空"
await db.commit()
_increment_files_completed(state)
return
# ========== 第一步：代码预处理（纯代码，不用 LLM）==========
state.update(
current_file=t.filename, action="正在分析对话结构...",
phase="preprocess", phase_percent=10, phase_detail="统计说话人",
)
preprocessor = Preprocessor()
filtered_lines = [p for p in lines if not preprocessor.is_management_line(p)]
if not filtered_lines:
t.status = "completed"
t.error_message = "过滤后无有效对话内容"
await db.commit()
_increment_files_completed(state)
return
segments, teacher = _preprocess_segments(filtered_lines)
logger.info(f"预处理完成: 识别老师={teacher}, 有效段落={len(segments)}")
if not segments:
t.status = "completed"
t.error_message = "未发现学员对话段落"
await db.commit()
_increment_files_completed(state)
return
state.update(
current_file=t.filename,
action=f"发现 {len(segments)} 个学员段落，开始识别故事...",
phase="preprocess", phase_percent=100, phase_detail="预处理完成",
percent=_file_percent(0, 0.05),
)
# ========== 第二步：判断故事 + 提取事实（并行）==========
state.update(
current_file=t.filename,
action=f"正在判断 {len(segments)} 个段落是否包含故事...",
phase="identify", phase_percent=0,
phase_detail=f"准备处理 {len(segments)} 个段落",
)
seg_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
identify_results = [None] * len(segments)
identify_errors = []
async def process_segment_for_identify(seg_idx, speaker, seg):
"""对单个段落进行判断+提取"""
async with seg_semaphore:
text = _segment_to_text(seg)
text_len = len(text)
# 更新进度
inner_pct = int((seg_idx + 1) / len(segments) * 100)
state.update(
phase="identify", phase_percent=inner_pct,
phase_detail=f"正在处理段落 {seg_idx + 1}/{len(segments)}（{speaker}, {text_len}字）",
percent=_file_percent(0.05, 0.30, (seg_idx + 1) / len(segments)),
)
if text_len <= SHORT_THRESHOLD:
# 短段落：跳过判断，直接提取事实
logger.info(f" 段落 {seg_idx}: {speaker} ({text_len}字) - 短段落，直接提取事实")
facts = await _extract_facts_from_chunk(text, "")
if facts:
hints = facts.get("story_hints", "") or f"{speaker}的故事"
return (speaker, seg, hints, [facts])
else:
logger.info(f" 段落 {seg_idx}: 提取失败，跳过")
return None
else:
# 长段落：先判断是否有故事
is_story, hints = await _identify_story(text)
logger.info(f" 段落 {seg_idx}: {speaker} ({text_len}字) - is_story={is_story}, hints={hints}")
if is_story:
return (speaker, seg, hints, None) # None 表示需要第三步切片提取
else:
logger.info(f" 段落 {seg_idx}: 无故事，跳过")
return None
tasks = [
process_segment_for_identify(i, speaker, seg)
for i, (speaker, seg) in enumerate(segments)
]
results = await asyncio.gather(*tasks, return_exceptions=True)
# 收集结果：分为两类
# - 直接提取到事实的（短段落）
# - 需要切片提取的（长段落，有故事）
all_facts = [] # 最终的 (speaker, seg, hints, facts_list) 列表
needs_chunking = [] # 需要进入第三步的 (speaker, seg, hints)
for i, r in enumerate(results):
if isinstance(r, Exception):
error_str = str(r)
logger.warning(f"段落 {i} 处理异常: {error_str}")
identify_errors.append(error_str)
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
raise r
continue
if r is None:
continue
speaker, seg, hints, facts_list = r
if facts_list is not None:
# 短段落已直接提取到事实
all_facts.append((speaker, seg, hints, facts_list))
else:
# 长段落有故事，需要切片提取
needs_chunking.append((speaker, seg, hints))
if not all_facts and not needs_chunking:
t.status = "completed"
t.error_message = "未发现包含个人故事的段落"
await db.commit()
_increment_files_completed(state)
state.update(
percent=_file_percent(0.05, 0.30, 1.0),
action="未发现故事",
)
return
logger.info(f"第二步完成: 直接提取={len(all_facts)}, 需切片={len(needs_chunking)}")
# 第二步完成时已知道发现了多少个故事，立即更新
discovered = len(all_facts) + len(needs_chunking)
total_stories[0] += discovered
state.update(stories_found=total_stories[0])
# ========== 第三步：切片提取事实（并行）==========
if needs_chunking:
state.update(
current_file=t.filename,
action=f"正在切片提取 {len(needs_chunking)} 个故事...",
phase="chunk", phase_percent=0,
phase_detail=f"准备切片 {len(needs_chunking)} 个故事段落",
)
chunk_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
async def process_chunked_story(story_idx, speaker, seg, hints):
"""对单个长段落进行切片提取"""
async with chunk_semaphore:
text = _segment_to_text(seg)
chunks = _chunk_text_by_paragraphs(text, max_chars=CHUNK_MAX_CHARS, overlap_paragraphs=CHUNK_OVERLAP_PARAGRAPHS)
logger.info(f" 故事 {story_idx}: {speaker} - 切成 {len(chunks)} 块")
facts_list = []
chunk_facts = await asyncio.gather(
*[_extract_facts_from_chunk(chunk, hints) for chunk in chunks],
return_exceptions=True,
)
for j, facts in enumerate(chunk_facts):
inner_pct = int((story_idx * 10 + j + 1) / (len(needs_chunking) * 10) * 100)
state.update(
phase="chunk", phase_percent=inner_pct,
phase_detail=f"正在切片提取 故事{story_idx + 1}/{len(needs_chunking)} 块{j + 1}/{len(chunks)}",
percent=_file_percent(0.30, 0.55, (story_idx + j / max(len(chunks), 1)) / len(needs_chunking)),
)
if isinstance(facts, Exception):
logger.warning(f" 块{j}提取异常: {facts}")
continue
if facts:
facts_list.append(facts)
return (speaker, seg, hints, facts_list)
chunk_tasks = [
process_chunked_story(i, speaker, seg, hints)
for i, (speaker, seg, hints) in enumerate(needs_chunking)
]
chunk_results = await asyncio.gather(*chunk_tasks, return_exceptions=True)
for r in chunk_results:
if isinstance(r, Exception):
error_str = str(r)
logger.warning(f"切片提取异常: {error_str}")
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
raise r
continue
if r is not None:
all_facts.append(r)
logger.info(f"第三步完成: 总共 {len(all_facts)} 个故事段落完成事实提取")
state.update(
current_file=t.filename,
action="切片提取完成，正在重组素材...",
phase="chunk", phase_percent=100,
phase_detail="切片提取完成",
percent=_file_percent(0.30, 0.55, 1.0),
)
# ========== 第四步：重组（纯代码，无需 LLM）==========
state.update(
current_file=t.filename,
action="正在重组素材...",
phase="merge", phase_percent=50,
phase_detail="去重合并素材",
percent=_file_percent(0.55, 0.60, 0.5),
)
merged = _merge_facts(all_facts)
logger.info(f"第四步完成: 重组出 {len(merged)} 个故事素材")
if not merged:
t.status = "completed"
t.error_message = "素材重组后为空"
await db.commit()
_increment_files_completed(state)
return
state.update(
current_file=t.filename,
action=f"重组完成，正在生成 {len(merged)} 个故事...",
phase="merge", phase_percent=100,
phase_detail="重组完成",
percent=_file_percent(0.55, 0.60, 1.0),
)
# ========== 第五步：生成最终故事（并行）==========
state.update(
current_file=t.filename,
action=f"正在生成 {len(merged)} 个故事...",
phase="generate", phase_percent=0,
phase_detail=f"准备生成 {len(merged)} 个故事",
percent=_file_percent(0.60, 0.95, 0),
)
gen_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
async def generate_one_story(story_idx, speaker, seg, material):
"""生成单个故事并保存到数据库"""
async with gen_semaphore:
inner_pct = int((story_idx + 1) / len(merged) * 100)
state.update(
phase="generate", phase_percent=inner_pct,
phase_detail=f"正在生成故事 {story_idx + 1}/{len(merged)}（{speaker}）",
percent=_file_percent(0.60, 0.95, (story_idx + 1) / len(merged)),
)
story_data = await _generate_story(material)
if not story_data:
logger.warning(f" 故事 {story_idx}: 生成失败")
return None
# 保存到数据库
raw_content = story_data.get("content", {})
if isinstance(raw_content, dict):
content_str = _format_story_content(raw_content)
else:
content_str = str(raw_content)
story = Story(
title=story_data.get("title", "未命名故事"),
summary=story_data.get("summary", ""),
tags=json.dumps(story_data.get("tags", []), ensure_ascii=False),
content=content_str,
raw_material=json.dumps(material, ensure_ascii=False),
speaker_nickname=story_data.get("speaker_nickname", ""),
source_transcript_id=t.id,
source_lines=json.dumps([l.get("index", 0) for l in seg if isinstance(l, dict)]),
duration_minutes=_estimate_duration(seg),
confidence=story_data.get("confidence", 0.7),
confidence_level=_confidence_level(story_data.get("confidence", 0.7)),
)
db.add(story)
await db.commit()
state.update(
action=f"生成第 {total_stories[0]} 个故事：{story.title}",
)
logger.info(f" 故事 {story_idx}: 保存成功 - {story.title}")
return story
gen_tasks = [
generate_one_story(i, speaker, seg, material)
for i, (speaker, seg, material) in enumerate(merged)
]
gen_results = await asyncio.gather(*gen_tasks, return_exceptions=True)
for r in gen_results:
if isinstance(r, Exception):
error_str = str(r)
logger.warning(f"故事生成异常: {error_str}")
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
raise r
t.status = "completed"
await db.commit()
_increment_files_completed(state)
state.update(
current_file=t.filename,
action=f"文件处理完成，共生成 {total_stories[0]} 个故事",
phase="generate", phase_percent=100,
phase_detail="文件处理完成",
percent=_file_percent(0.60, 0.95, 1.0),
)
except Exception as e:
error_str = str(e)
logger.exception(f"Error processing file {transcript.filename}")
try:
async with async_session() as db:
t = await db.get(Transcript, transcript.id)
if t:
t.status = "error"
t.error_message = error_str
await db.commit()
except Exception:
pass
_increment_files_completed(state)
if "529" in error_str or "overloaded" in error_str.lower():
state.update(status="error", error="AI 服务暂时过载（529），请稍后再试")
elif "timed out" in error_str.lower():
state.update(status="error", error="AI 服务请求超时，请稍后再试")
else:
state.update(status="error", error=f"处理文件时出错：{error_str[:200]}")
# ============================================================
# 第一步：代码预处理
# ============================================================
def _preprocess_segments(lines: list[dict]) -> tuple[list[tuple[str, list[dict]]], str]:
"""
代码预处理（v3：时间范围截取，允许重叠）。
识别老师 → 过滤杂音（<20句）→ 按时间范围截取 → 过滤短段落（<15句/<80字）→ 过滤非主讲述人（占比<10%）
返回: (segments, teacher)
segments: [(speaker, segment_lines), ...]
teacher: 老师名称
"""
# 1. 统计每个说话人
speaker_stats = {}
for i, line in enumerate(lines):
speaker = line.get("speaker", "")
if not speaker:
continue
if speaker not in speaker_stats:
speaker_stats[speaker] = {"lines": 0, "chars": 0, "first": i, "last": i}
speaker_stats[speaker]["lines"] += 1
speaker_stats[speaker]["chars"] += len(line.get("content", ""))
speaker_stats[speaker]["last"] = i
# 2. 识别老师（发言最多的人）
teacher = max(speaker_stats, key=lambda s: speaker_stats[s]["lines"])
logger.info(f"识别老师: {teacher} ({speaker_stats[teacher]['lines']}行, {speaker_stats[teacher]['chars']}字)")
# 3. 过滤杂音（< 20句的说话人）+ 排除老师
students = []
noise_speakers = []
for s, stats in speaker_stats.items():
if s == teacher:
continue
if stats["lines"] < 20:
noise_speakers.append(f"{s} ({stats['lines']}句)")
continue
students.append(s)
if noise_speakers:
logger.info(f"杂音过滤: {', '.join(noise_speakers)}")
logger.info(f"有效学员: {students}")
# 4. 按时间范围截取：每个学员从第一句到最后一句，中间所有行全部截取
segments = []
for speaker in students:
stats = speaker_stats[speaker]
first_idx = stats["first"]
last_idx = stats["last"]
seg = lines[first_idx:last_idx + 1]
segments.append((speaker, seg))
# 5. 过滤：短段落 + 当事人占比过低（非主讲述人）
final_segments = []
for speaker, seg in segments:
student_lines = [l for l in seg if l.get("speaker", "") == speaker]
if len(student_lines) < 15:
logger.info(f"过滤: {speaker} ({len(student_lines)}句<15)")
continue
student_chars = sum(len(l.get("content", "")) for l in student_lines)
if student_chars < 80:
logger.info(f"过滤: {speaker} ({student_chars}字<80)")
continue
# 当事人说话占比 < 10% → 非主讲述人，丢弃
ratio = len(student_lines) / len(seg) if seg else 0
if ratio < 0.1:
logger.info(f"过滤: {speaker} (占比{ratio:.1%}<10%)")
continue
final_segments.append((speaker, seg))
logger.info(f"最终: {len(final_segments)} 个学员段落")
for i, (speaker, seg) in enumerate(final_segments):
student_lines = [l for l in seg if l.get("speaker", "") == speaker]
student_chars = sum(len(l.get("content", "")) for l in student_lines)
ratio = len(student_lines) / len(seg) if seg else 0
logger.info(f" 段落{i}: {speaker}, 截取{len(seg)}行, 学员{len(student_lines)}句/{student_chars}字, 占比{ratio:.1%}")
return final_segments, teacher
# ============================================================
# 第二步：判断故事
# ============================================================
async def _identify_story(text: str) -> tuple[bool, str]:
"""判断对话段落是否包含个人故事"""
prompt = """判断这段对话是否包含学员的个人故事。直接输出JSON，不要分析。
个人故事特征（满足至少2条）：学员讲述具体经历、包含时间地点人物细节、表达深层情感、老师深入引导疗愈、揭示心理或家庭问题。
非故事内容：纯课堂讲解、简单问答寒暄、课堂管理、纯疗愈引导词。
直接输出JSON：
{"is_story": true或false, "confidence": 0.0到1.0, "story_hints": "一句话描述故事主题和讲述者"}
对话内容：
```
""" + text[:4000] + """
```"""
try:
result = await llm_client.chat_json([{"role": "user", "content": prompt}], temperature=0.2)
if result:
is_story = result.get("is_story", False)
confidence = result.get("confidence", 0)
hints = result.get("story_hints", "")
logger.info(f"Story identification: is_story={is_story}, confidence={confidence}, hints={hints[:80]}")
return bool(is_story) and confidence >= 0.5, hints
except Exception as e:
logger.warning(f"Story identification error: {e}")
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
raise
return False, ""
# ============================================================
# 第三步：切片提取事实
# ============================================================
def _chunk_text_by_paragraphs(text: str, max_chars: int = 3000, overlap_paragraphs: int = 5) -> list[str]:
"""按自然段切分，在 max_chars 附近找段落结尾切，重叠 overlap_paragraphs 个自然段"""
paragraphs = [p for p in text.split("\n") if p.strip()]
if not paragraphs:
return [text]
if len(text) <= max_chars:
return [text]
chunks = []
i = 0
while i < len(paragraphs):
# 从当前位置开始，累加段落直到超过 max_chars
current = []
current_len = 0
j = i
while j < len(paragraphs):
para = paragraphs[j]
# 超过 max_chars 且已有内容 → 在这个段落前切断（段落结尾切）
if current_len + len(para) > max_chars and current:
break
current.append(para)
current_len += len(para)
j += 1
chunks.append("\n".join(current))
# 到末尾了就结束
if j >= len(paragraphs):
break
# 下一次起点：当前切到的位置 - 重叠段落数
next_i = j - overlap_paragraphs
# 防止死循环：确保至少前进1个段落
if next_i <= i:
next_i = i + 1
i = next_i
return chunks
async def _extract_facts_from_chunk(text: str, hints: str) -> dict | None:
"""从对话切片中提取关键信息（事件、情感、原话、人物）"""
prompt = f"""从这段对话中提取关键信息。只输出JSON。
故事主题：{hints}
提取以下内容：
- events: 发生的事件（尽量详细，保留具体细节）
- emotions: 表达的情感
- quotes: 讲述者的原话（保留原话，不要改写，尽量多提取；完全相同的重复语句只保留一条）
- key_people: 提到的人物
{{"events":["事件1","事件2"],"emotions":["情感1","情感2"],"quotes":["原话1","原话2","原话3"],"key_people":["人物1"]}}
对话内容：
```
{text}
```"""
try:
result = await llm_client.chat_json([{"role": "user", "content": prompt}], temperature=0.1)
if not result:
return None
# chat_json 可能返回 list（解析错误），需要转为 dict
if isinstance(result, list):
for item in result:
if isinstance(item, dict) and ("events" in item or "quotes" in item):
result = item
break
else:
logger.warning(f"chat_json returned list, cannot convert to dict: {result[:3]}")
return None
return result
except Exception as e:
logger.warning(f"Extract facts error: {e}")
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
raise
return None
# ============================================================
# 第四步：重组（纯代码，无需 LLM）
# ============================================================
def _fuzzy_dedup_quotes(quotes: list[str]) -> list[str]:
"""
模糊去重：对高度相似的语句只保留第一条。
策略：提取每条语句的核心部分（去掉常见前缀后缀），相似度超过阈值的视为重复。
"""
import re
def normalize(s: str) -> str:
"""提取语句核心，去掉语气词和细微差异"""
s = s.strip()
# 去掉「」包裹
if s.startswith("「") and s.endswith("」"):
s = s[1:-1]
# 去掉常见前缀
for prefix in ["嗯", "啊", "呃", "就是", "然后", "那个", "所以"]:
if s.startswith(prefix):
s = s[len(prefix):].strip()
# 去掉常见尾缀
for suffix in ["啊", "吧", "呢", "嘛", "哦", "呀"]:
if s.endswith(suffix) and len(s) > 4:
s = s[:-1]
# 去掉多余空白和重复字（如"我我我"、"的的的"）
s = re.sub(r'(.)\1{2,}', r'\1', s)
return s
def similarity(a: str, b: str) -> float:
"""简单的字符级相似度（Jaccard on character bigrams）"""
if not a or not b:
return 0.0
a_bigrams = set(a[i:i+2] for i in range(len(a)-1))
b_bigrams = set(b[i:i+2] for i in range(len(b)-1))
if not a_bigrams or not b_bigrams:
return 0.0
intersection = a_bigrams & b_bigrams
union = a_bigrams | b_bigrams
return len(intersection) / len(union)
kept = []
kept_normalized = []
for q in quotes:
nq = normalize(q)
is_dup = False
for kn in kept_normalized:
# 短语句用高阈值，长语句用低阈值
threshold = 0.8 if len(nq) < 15 else 0.7
if similarity(nq, kn) >= threshold:
is_dup = True
break
if not is_dup:
kept.append(q)
kept_normalized.append(nq)
removed = len(quotes) - len(kept)
if removed > 0:
logger.info(f" 模糊去重: {len(quotes)} -> {len(kept)} (去除 {removed} 条相似重复)")
return kept
def _merge_facts(all_facts: list) -> list[tuple[str, list, dict]]:
"""
重组素材（Reduce）。
对每个故事段落的所有切片事实进行合并去重：
- 事件按原文顺序保留
- 情感去重
- 原话去重（不限制数量）
- 人物去重
输入: [(speaker, seg, hints, facts_list), ...]
输出: [(speaker, seg, material), ...]
material: {"events": [...], "emotions": [...], "quotes": [...], "key_people": [...], "story_hints": "..."}
"""
merged = []
for i, (speaker, seg, hints, facts_list) in enumerate(all_facts):
all_events, all_emotions, all_quotes, all_people = [], [], [], []
for facts in facts_list:
if isinstance(facts, dict):
all_events.extend(facts.get("events", []))
all_emotions.extend(facts.get("emotions", []))
all_quotes.extend(facts.get("quotes", []))
all_people.extend(facts.get("key_people", []))
# 情感去重（保持顺序）
all_emotions = list(dict.fromkeys(all_emotions))
# 人物去重（保持顺序）
all_people = list(dict.fromkeys(all_people))
# 原话去重（保持顺序，不限制数量）
# 第一轮：精确去重
seen_quotes = set()
unique_quotes = [q for q in all_quotes if q not in seen_quotes and not seen_quotes.add(q)]
# 第二轮：模糊去重（去除重复模式的语句，如释放语句只保留一条）
unique_quotes = _fuzzy_dedup_quotes(unique_quotes)
material = {
"events": all_events,
"emotions": all_emotions,
"quotes": unique_quotes,
"key_people": all_people,
"story_hints": hints,
}
logger.info(f" 重组故事 {i}: {speaker} - 事件:{len(all_events)}, 情感:{len(all_emotions)}, "
f"原话:{len(unique_quotes)}, 人物:{len(all_people)}")
merged.append((speaker, seg, material))
return merged
# ============================================================
# 第五步：生成最终故事
# ============================================================
async def _generate_story(material: dict) -> dict | None:
"""基于素材生成第三人称故事（5部分结构化输出）"""
prompt = f"""基于素材生成第三人称故事。只输出JSON，不要其他内容。
素材：
主题：{material['story_hints']}
事件：{json.dumps(material['events'], ensure_ascii=False)}
情感：{json.dumps(material['emotions'], ensure_ascii=False)}
原话：{json.dumps(material['quotes'], ensure_ascii=False)}
人物：{json.dumps(material['key_people'], ensure_ascii=False)}
写作手法（非常重要）：
【双时空结构】
故事有两个时空层：
- "当下时空"：学员在课堂上与卢慧老师的对话——这是故事的框架，用对话体呈现，保留师生互动的完整脉络
- "过往时空"：学员讲述的过去经历——这是故事的核心，用"场景重建"手法呈现，让读者穿越到那个时空
【场景重建手法（用于"过往的故事"部分）】
当学员讲述过去的经历时，不要写成"她说她小时候…"这种转述体，而是直接把读者拉进那个时空：
- 直接切入场景，用过去时态叙述。比如不要写"文晔回忆说她的儿子15岁去了美国"，而要写"十五岁那年，文晔的儿子独自登上了飞往美国的航班"
- 用五感描写构建画面：她看到了什么、听到了什么、身体感受到了什么（比如"手心出汗"、"脚发凉"这些身体反应要融入场景中）
- 用环境细节暗示情绪：不要直接说"她很痛苦"，而是通过场景让读者感受到痛苦
- 关键情感爆发处用直接引语（「」包裹原话），其他地方用间接引语和叙述自然融合
- 场景之间要有过渡，比如"然而，命运的转折发生在前年冬天——"
【对话体手法（用于"当下的困境"和"转变"部分）】
- 保留师生对话的现场感，写出谁说了什么、对方如何回应
- 在对话中穿插学员的内心活动、身体反应、情绪变化
- 老师的引导语要完整保留，这是疗愈过程的关键
【通用要求】
- 每个事件都要展开写，写出具体过程和细节，不要一句话概括
- 事件之间要有过渡和因果逻辑，不能跳跃
- 情感变化要写出过程：从什么情感转变到什么情感，中间经历了什么
- 不编造素材中没有的内容，但可以根据素材合理补充场景细节（如环境、氛围）
- 每个部分充分展开，不要限制长度，越详细越好
- 原文金句数组中的每条金句必须用英文双引号包裹，如 ["金句1", "金句2"]，不要用「」替代双引号
禁止事项：
- 禁止用一句话概括多个事件（如"她倾诉了挣扎"、"她经历了疗愈过程"）
- 禁止省略素材中的具体细节
- 禁止省略老师的引导和回应
- 禁止事件之间跳跃，要保持时间顺序和因果逻辑
- 禁止过度总结或抽象化描述
- 禁止在"过往的故事"部分全程使用转述体（"她说…她回忆…"），必须用场景重建手法让读者身临其境
- 禁止为了简洁而牺牲内容的丰富性
输出格式：{{"title":"标题","summary":"80字摘要","tags":["标签"],"speaker_nickname":"昵称","content":{{"讲述者":"...","当下的困境":"...","过往的故事":"...","转变":"...","原文金句":["原话1","原话2","原话3"]}}}}"""
try:
result = await llm_client.chat_json(
[{"role": "user", "content": prompt}],
temperature=0.7,
)
if not result:
return None
# chat_json 可能返回 list（解析错误），需要转为 dict
if isinstance(result, list):
for item in result:
if isinstance(item, dict) and "title" in item:
result = item
break
else:
logger.warning(f"chat_json returned list, cannot convert to dict: {result[:3]}")
return None
# 解析 content 纯文本为结构化字段
raw_content = result.get("content", "")
if isinstance(raw_content, str) and "|||" in raw_content:
parts = raw_content.split("|||")
content_dict = {}
keys = ["讲述者", "当下的困境", "过往的故事", "转变", "原文金句"]
for idx, key in enumerate(keys):
content_dict[key] = parts[idx].strip() if idx < len(parts) else ""
result["content"] = content_dict
elif isinstance(raw_content, dict):
for key in ["讲述者", "当下的困境", "过往的故事", "转变", "原文金句"]:
if key not in raw_content:
raw_content[key] = ""
result["content"] = raw_content
else:
result["content"] = {"完整故事": str(raw_content)}
if not result.get("tags"):
result["tags"] = []
return result
except Exception as e:
logger.warning(f"Generate story error: {e}")
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
raise
return None
# ============================================================
# 工具函数
# ============================================================
def _segment_to_text(segment_lines: list) -> str:
"""将段落行列表转为对话文本"""
parts = []
for line in segment_lines:
speaker = line.get("speaker", "")
content = line.get("content", "")
if speaker:
parts.append(f"{speaker}: {content}")
else:
parts.append(content)
return "\n".join(parts)
def _increment_files_completed(state: TaskState):
"""增加已完成文件计数"""
p = state.progress
files_completed = p.get("files_completed", 0) + 1
files_total = p.get("files_total", 1)
state.update(files_completed=files_completed)
def _estimate_duration(segment_lines: list) -> float:
"""根据时间戳估算故事时长（分钟）"""
timestamps = []
for line in segment_lines:
if isinstance(line, dict) and line.get("timestamp"):
try:
parts = line["timestamp"].split(":")
if len(parts) >= 2:
minutes = int(parts[-2]) + int(parts[-1]) / 60
if len(parts) == 3:
minutes += int(parts[0]) * 60
timestamps.append(minutes)
except (ValueError, IndexError):
continue
if len(timestamps) >= 2:
return round(max(timestamps) - min(timestamps), 1)
return 0
def _confidence_level(confidence: float) -> str:
if confidence >= 0.8:
return "high"
elif confidence >= 0.6:
return "medium"
return "low"
def _format_story_content(content_dict: dict) -> str:
"""将 LLM 返回的结构化故事内容转为可读的 Markdown 格式"""
parts = []
# 按顺序输出各个章节
sections = ["讲述者", "当下的困境", "过往的故事", "转变"]
for section_name in sections:
if content_dict.get(section_name):
parts.append(f"## {section_name}\n\n{content_dict[section_name]}")
# 原文金句
quotes = content_dict.get("原文金句", [])
if not quotes:
quotes = content_dict.get("金句摘录", [])
if quotes and isinstance(quotes, list):
parts.append("## 原文金句")
for q in quotes:
parts.append(f"\n> {q}")
# 兼容旧格式：故事梗概
if content_dict.get("故事梗概") and "讲述者" not in content_dict:
parts.insert(0, f"## 故事梗概\n\n{content_dict['故事梗概']}")
# 兼容旧格式：关键要素
elements = content_dict.get("关键要素", [])
if elements and isinstance(elements, list) and "讲述者" not in content_dict:
parts.append("## 关键要素")
for elem in elements:
if isinstance(elem, dict):
title = elem.get("标题", "")
desc = elem.get("描述", "")
parts.append(f"\n### {title}\n\n{desc}")
# 如果上面都没匹配到，按通用 key-value 格式输出
if not parts:
for key, value in content_dict.items():
if isinstance(value, str):
parts.append(f"## {key}\n\n{value}")
elif isinstance(value, list):
parts.append(f"## {key}")
for item in value:
if isinstance(item, dict):
for k, v in item.items():
parts.append(f"\n**{k}**：{v}")
else:
parts.append(f"\n- {item}")
return "\n\n".join(parts) if parts else json.dumps(content_dict, ensure_ascii=False)

63
backend/app/services/file_service.py Normal file
View File

@@ -0,0 +1,63 @@
import os
import uuid
import shutil
from fastapi import UploadFile
from app.config import settings
class FileService:
"""文件上传、存储、删除"""
def __init__(self):
os.makedirs(settings.UPLOAD_DIR, exist_ok=True)
os.makedirs(os.path.join(settings.UPLOAD_DIR, "transcripts"), exist_ok=True)
os.makedirs(os.path.join(settings.UPLOAD_DIR, "persons"), exist_ok=True)
os.makedirs(settings.EXPORT_DIR, exist_ok=True)
async def save_transcript(self, file: UploadFile) -> dict:
"""保存上传的文字稿"""
file_id = str(uuid.uuid4())
ext = os.path.splitext(file.filename)[1]
save_name = f"{file_id}{ext}"
save_dir = os.path.join(settings.UPLOAD_DIR, "transcripts")
save_path = os.path.join(save_dir, save_name)
content = await file.read()
with open(save_path, "wb") as f:
f.write(content)
return {
"id": file_id,
"filename": file.filename,
"file_path": save_path,
"file_size": len(content),
}
async def save_person(self, file: UploadFile) -> dict:
"""保存上传的讲述者信息"""
file_id = str(uuid.uuid4())
ext = os.path.splitext(file.filename)[1]
save_name = f"{file_id}{ext}"
save_dir = os.path.join(settings.UPLOAD_DIR, "persons")
save_path = os.path.join(save_dir, save_name)
content = await file.read()
with open(save_path, "wb") as f:
f.write(content)
return {
"id": file_id,
"filename": file.filename,
"file_path": save_path,
"file_size": len(content),
}
def delete_file(self, file_path: str) -> bool:
"""删除文件"""
if os.path.exists(file_path):
os.remove(file_path)
return True
return False
file_service = FileService()

287
backend/app/services/llm_client.py Normal file
View File

@@ -0,0 +1,287 @@
import json
import logging
import re
import time
from openai import AsyncOpenAI
import anthropic
from app.config import settings
logger = logging.getLogger(__name__)
class LLMClient:
"""LLM 客户端，支持 OpenAI 和 Anthropic SDK"""
def __init__(self):
self._openai_client = None
self._anthropic_client = None
self._api_key = settings.LLM_API_KEY
self._base_url = settings.LLM_BASE_URL
self._model = settings.LLM_MODEL
self._provider = settings.LLM_PROVIDER
def _get_openai_client(self) -> AsyncOpenAI:
if self._openai_client is None:
self._openai_client = AsyncOpenAI(
api_key=self._api_key,
base_url=self._base_url,
max_retries=0,
timeout=300.0,
)
return self._openai_client
def _get_anthropic_client(self) -> anthropic.AsyncAnthropic:
if self._anthropic_client is None:
# MiniMax Anthropic 端点
base_url = self._base_url.replace("/v1", "/anthropic").replace("api.minimax.chat", "api.minimaxi.com")
if not base_url.endswith("/anthropic"):
# 如果 base_url 不是标准格式，手动拼接
base_url = "https://api.minimaxi.com/anthropic"
self._anthropic_client = anthropic.AsyncAnthropic(
api_key=self._api_key,
base_url=base_url,
timeout=300.0,
)
return self._anthropic_client
def _use_anthropic(self) -> bool:
"""判断是否使用 Anthropic SDK"""
return False # MiniMax thinking 占用太多 token，统一用 OpenAI SDK
def update_config(self, base_url: str = None, api_key: str = None, model: str = None):
"""更新配置（设置页面保存后调用）"""
if base_url:
self._base_url = base_url
if api_key:
self._api_key = api_key
if model:
self._model = model
# 重置客户端
self._openai_client = None
self._anthropic_client = None
# 更新 provider 判断
if base_url:
self._provider = "minimax" if "minimax" in base_url.lower() else "openai"
async def chat(self, messages: list[dict], temperature: float = None, max_tokens: int = 16384, timeout: float = 300.0) -> str:
"""发送对话请求，返回文本内容（不含思考过程）"""
if temperature is None:
temperature = settings.TEMPERATURE
if self._use_anthropic():
return await self._chat_anthropic(messages, temperature, max_tokens, timeout)
else:
result = await self._chat_openai(messages, temperature, max_tokens, timeout)
return result["content"]
async def chat_detail(self, messages: list[dict], temperature: float = None, max_tokens: int = 16384, timeout: float = 300.0) -> dict:
"""发送对话请求，返回详细信息（content + reasoning + finish_reason + usage + elapsed）"""
if temperature is None:
temperature = settings.TEMPERATURE
if self._use_anthropic():
# Anthropic 暂不支持 detail
content = await self._chat_anthropic(messages, temperature, max_tokens, timeout)
return {"content": content, "reasoning_content": "", "finish_reason": "stop", "usage": {}, "elapsed": 0}
else:
return await self._chat_openai(messages, temperature, max_tokens, timeout)
async def _chat_openai(self, messages: list[dict], temperature: float, max_tokens: int, timeout: float = 300.0) -> dict:
"""OpenAI SDK 调用，返回详细信息 dict"""
client = self._get_openai_client()
t0 = time.time()
response = await client.chat.completions.create(
model=self._model,
messages=messages,
temperature=temperature,
max_tokens=max_tokens,
timeout=timeout,
)
elapsed = time.time() - t0
content = response.choices[0].message.content or ""
# 提取 reasoning_content（thinking）
reasoning = ""
msg = response.choices[0].message
if hasattr(msg, 'reasoning_content') and msg.reasoning_content:
reasoning = msg.reasoning_content
# 有些 SDK 把 reasoning 放在 model_extra 里
if not reasoning and hasattr(msg, 'model_extra') and isinstance(msg.model_extra, dict):
reasoning = msg.model_extra.get("reasoning_content", "")
# 检查是否有多个 choice
if len(response.choices) > 1:
print(f"[LLM] 警告: 返回了 {len(response.choices)} 个 choices")
# 检查 finish_reason
reason = response.choices[0].finish_reason if response.choices else "unknown"
if reason != "stop":
print(f"[LLM] 警告: finish_reason={reason}, 可能被截断")
# usage 信息
usage = {}
if hasattr(response, 'usage') and response.usage:
usage = {
"prompt_tokens": response.usage.prompt_tokens,
"completion_tokens": response.usage.completion_tokens,
"total_tokens": response.usage.total_tokens,
}
if hasattr(response.usage, 'completion_tokens_details') and response.usage.completion_tokens_details:
details = response.usage.completion_tokens_details
usage["reasoning_tokens"] = getattr(details, 'reasoning_tokens', 0) or 0
# 保存完整响应到文件，供调试
try:
resp_dict = response.model_dump() if hasattr(response, 'model_dump') else str(response)
debug_path = f"/tmp/llm_response_{id(response) % 100000}.json"
with open(debug_path, "w", encoding="utf-8") as _f:
json.dump(resp_dict, _f, ensure_ascii=False, indent=2, default=str)
print(f"[LLM] 完整响应已保存: {debug_path}")
except Exception:
pass
return {
"content": content,
"reasoning_content": reasoning,
"finish_reason": reason,
"usage": usage,
"elapsed": round(elapsed, 2),
}
async def _chat_anthropic(self, messages: list[dict], temperature: float, max_tokens: int, timeout: float = 300.0) -> str:
"""Anthropic SDK 调用 - thinking 和 text 自动分离"""
client = self._get_anthropic_client()
# 转换消息格式：OpenAI -> Anthropic
system_msg = ""
anthropic_messages = []
for msg in messages:
role = msg.get("role", "user")
content = msg.get("content", "")
if role == "system":
system_msg = content
elif role in ("user", "assistant"):
anthropic_messages.append({"role": role, "content": content})
kwargs = {
"model": self._model,
"max_tokens": max_tokens,
"messages": anthropic_messages,
"thinking": {"type": "disabled"}, # 禁用 thinking，避免占用 token
}
if system_msg:
kwargs["system"] = system_msg
if temperature is not None:
kwargs["temperature"] = temperature
response = await client.messages.create(**kwargs)
# 从 response 中提取文本内容，thinking 自动分离在 block.thinking 中
text_parts = []
for block in response.content:
if hasattr(block, 'text') and block.type == 'text':
text_parts.append(block.text)
# thinking 内容自动忽略
result = "\n".join(text_parts)
return result
async def chat_json(self, messages: list[dict], temperature: float = None, max_tokens: int = 16384) -> dict:
"""发送对话请求，期望返回 JSON"""
raw = await self.chat(messages, temperature, max_tokens=max_tokens)
return _parse_json(raw)
async def test_connection(self) -> dict:
"""测试 LLM 连接"""
try:
response = await self.chat([{"role": "user", "content": "请回复：连接成功"}])
return {"success": True, "message": f"连接成功，模型回复：{response[:50]}"}
except Exception as e:
return {"success": False, "message": f"连接失败：{str(e)}"}
def _parse_json(raw: str) -> dict:
"""健壮的 JSON 解析，处理 LLM 返回的各种格式问题"""
print(f"[_parse_json] raw长度={len(raw)}, 前100字={raw[:100]}")
print(f"[_parse_json] raw后100字={raw[-100:]}")
raw = raw.strip()
# 去掉 markdown 代码块
if raw.startswith("```"):
raw = re.sub(r'^```\w*\n?', '', raw)
raw = re.sub(r'\n?```$', '', raw)
raw = raw.strip()
# 找 JSON 结构的位置
# 优先找第一个 { }（JSON 对象），用括号匹配确保完整
first_brace = raw.find('{')
last_bracket = raw.rfind('[')
start = -1
if first_brace >= 0:
start = first_brace
# 从第一个 { 开始，找匹配的 }
depth = 0
end = -1
for i in range(start, len(raw)):
if raw[i] == '{':
depth += 1
elif raw[i] == '}':
depth -= 1
if depth == 0:
end = i + 1
break
if end <= start:
start = -1
elif last_bracket >= 0:
start = last_bracket
end = raw.rfind(']') + 1
if end <= start:
start = -1
if start < 0:
start = raw.find('{')
end = raw.rfind('}') + 1
if start < 0 or end <= start:
raise ValueError(f"无法找到 JSON: {raw[:200]}")
json_str = raw[start:end]
# 修复常见问题：中文引号替换为英文引号（在JSON值内部的需要转义）
# 先把 JSON 结构中的英文引号保护起来，再替换中文引号
json_str = json_str.replace('\u201c', '\u201c').replace('\u201d', '\u201d') # 先保持不变
json_str = json_str.replace('\u2018', "'").replace('\u2019', "'")
json_str = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', '', json_str)
json_str = re.sub(r',\s*([}\]])', r'\1', json_str)
try:
return json.loads(json_str)
except json.JSONDecodeError as e:
# 如果失败，尝试将中文引号替换为转义的英文引号
json_str2 = json_str.replace('\u201c', '\\"').replace('\u201d', '\\"')
try:
return json.loads(json_str2)
except json.JSONDecodeError:
pass
# 尝试把换行替换为 \n
json_str2 = json_str.replace('\n', '\\n').replace('\r', '\\r')
json_str2 = json_str2.replace('\\\\n', '\\n')
try:
return json.loads(json_str2)
except json.JSONDecodeError:
pass
# 尝试自动补全缺失的闭合括号（模型输出被截断时常见）
for extra in ['}', '}}', '}}}', '}]', '}]}']:
try:
return json.loads(json_str + extra)
except json.JSONDecodeError:
continue
raise ValueError(f"无法解析 JSON: {json_str[:300]}")
llm_client = LLMClient()

220
backend/app/services/preprocessor.py Normal file
View File

@@ -0,0 +1,220 @@
import re
class Preprocessor:
"""
预处理器：从原始对话中提取包含故事的段落
核心思路：
1. 过滤纯课堂管理内容（签到、纪律等）
2. 识别"个案对话段"——老师与某位学员的深度互动
3. 保留完整对话上下文（师生互动），不过度过滤
4. 按话题边界切分为合理的段落
"""
# 课堂管理关键词（出现这些词的整行直接过滤）
MANAGEMENT_KEYWORDS = ["签到", "打卡", "禁言", "扣三个一", "进组", "会议室链接"]
# 纯课堂管理发言模式（整条消息都是管理内容）
MANAGEMENT_PATTERNS = [
re.compile(r"大家.*签到"),
re.compile(r"请.*进组"),
re.compile(r"会议室"),
re.compile(r"纪律"),
re.compile(r"昵称.*改为"),
re.compile(r"欢迎.*进入"),
]
# 老师关键词
TEACHER_KEYWORDS = ["卢慧老师", "静静", "督导"]
def is_teacher(self, speaker: str) -> bool:
"""判断是否为老师/助教"""
return any(kw in speaker for kw in self.TEACHER_KEYWORDS)
def is_management_line(self, line: dict) -> bool:
"""判断是否为课堂管理行"""
content = line.get("content", "")
# 短内容 + 管理关键词 = 管理行
if len(content) < 30:
return any(kw in content for kw in self.MANAGEMENT_KEYWORDS)
return any(p.search(content) for p in self.MANAGEMENT_PATTERNS)
def preprocess(self, paragraphs: list[dict]) -> list[dict]:
"""
完整预处理流程：
1. 过滤课堂管理
2. 识别个案对话段（保留师生互动）
3. 按主要讲述者拆分（每个段落只包含一个主要讲述者）
"""
# 第一步：过滤课堂管理行
filtered = [p for p in paragraphs if not self.is_management_line(p)]
# 第二步：识别个案对话段
segments = self._extract_story_segments(filtered)
# 第三步：按主要讲述者拆分
segments = self._split_by_speaker(segments)
return segments
def _split_by_speaker(self, segments: list[list[dict]]) -> list[list[dict]]:
"""
将每个段落按讲述者拆分成子段落。
同一个讲述者的所有对话（即使中间隔了其他人的插话）收集在一起，
老师发言分配给前一个活跃的学员作为上下文。
"""
result = []
for seg in segments:
# 按讲述者收集对话，保留老师发言作为上下文
speaker_blocks = {} # speaker -> [lines]
last_student = None
for line in seg:
speaker = line.get("speaker", "")
if self.is_teacher(speaker):
# 老师发言归入前一个活跃的学员
if last_student and last_student in speaker_blocks:
speaker_blocks[last_student].append(line)
else:
if speaker not in speaker_blocks:
speaker_blocks[speaker] = []
speaker_blocks[speaker].append(line)
last_student = speaker
# 每个讲述者的对话作为一个子段落
for speaker, lines in speaker_blocks.items():
student_content = sum(
len(l.get("content", ""))
for l in lines
if not self.is_teacher(l.get("speaker", ""))
)
# 过滤掉内容太少的（可能是插话而非讲述者）
if student_content > 80 and len(lines) >= 3:
# 计算该讲述者在自己段落中的字数占比
# 如果老师发言占比超过50%，说明这主要是老师在讲课，不是学员故事
teacher_chars = sum(
len(l.get("content", ""))
for l in lines
if self.is_teacher(l.get("speaker", ""))
)
total_chars = student_content + teacher_chars
if total_chars > 0 and teacher_chars / total_chars > 0.5:
continue # 老师讲太多，跳过
result.append(lines)
return result
def _extract_story_segments(self, paragraphs: list[dict]) -> list[list[dict]]:
"""
从过滤后的对话中提取可能包含故事的段落。
策略：
- 找到学员的"长发言"（>50字）作为故事线索
- 向前向后扩展，包含老师的引导和互动
- 直到遇到另一位学员的长发言或长时间沉默
- 最小段落 10 行，最大段落 200 行
"""
segments = []
i = 0
used = set() # 已分配的行索引
while i < len(paragraphs):
p = paragraphs[i]
# 跳过老师单独发言（非互动）
if self.is_teacher(p.get("speaker", "")):
i += 1
continue
# 跳过短发言（< 30字不太可能开启故事）
if len(p.get("content", "")) < 30:
i += 1
continue
# 跳过已分配的行
if p.get("index", i) in used:
i += 1
continue
# 找到一个学员发言，开始扩展
segment_start = i
segment_end = i
# 向前扩展（找老师之前的引导）
expand_start = max(0, i - 5)
for j in range(i - 1, expand_start - 1, -1):
if j in used:
break
prev = paragraphs[j]
# 如果前一个是老师发言，包含
if self.is_teacher(prev.get("speaker", "")):
segment_start = j
# 如果前一个是同一学员的发言，包含
elif prev.get("speaker") == p.get("speaker"):
segment_start = j
else:
# 其他学员发言，停止
break
# 向后扩展（找老师的回应和后续互动）
current_speaker = p.get("speaker", "")
for j in range(i + 1, min(len(paragraphs), i + 200)):
if j in used:
segment_end = j - 1
break
curr = paragraphs[j]
curr_speaker = curr.get("speaker", "")
curr_content = curr.get("content", "")
# 老师发言：包含（互动引导）
if self.is_teacher(curr_speaker):
segment_end = j
continue
# 同一学员继续发言：包含
if curr_speaker == current_speaker:
segment_end = j
continue
# 其他学员短回应（< 20字）：可能是插话，跳过但不终止
if len(curr_content) < 20:
continue
# 其他学员长发言：新的话题开始，终止
break
# 提取段落
segment = paragraphs[segment_start:segment_end + 1]
# 过滤掉太短的段落（< 8行）
if len(segment) >= 8:
# 检查是否有足够的学员实质性内容（非纯疗愈练习）
student_content_len = sum(
len(s.get("content", ""))
for s in segment
if not self.is_teacher(s.get("speaker", ""))
and len(s.get("content", "")) > 15
)
# 学员实质内容 > 100字才认为是故事
if student_content_len > 100:
segments.append(segment)
for s in segment:
used.add(s.get("index", 0))
i = segment_end + 1
continue
i += 1
return segments
def is_teacher(speaker: str) -> bool:
"""判断是否为老师/助教发言"""
teacher_keywords = ["卢慧老师", "静静", "督导", "老师"]
return any(kw in speaker for kw in teacher_keywords)
preprocessor = Preprocessor()