958 lines
40 KiB
Python
958 lines
40 KiB
Python
"""
|
||
故事提取器 - v4 五步流水线策略
|
||
第一步：代码预处理（识别老师→过滤杂音→时间范围截取→过滤短段落/非主讲述人）
|
||
第二步：判断故事 + 提取事实（短段落直接提取，长段落先判断再切片提取）
|
||
第三步：切片提取事实（按自然段切分，并行提取）
|
||
第四步：重组（纯代码，事件/情感/原话/人物去重）
|
||
第五步：生成最终故事（基于素材生成5部分结构化故事）
|
||
"""
|
||
import json
|
||
import asyncio
|
||
import logging
|
||
|
||
from sqlalchemy import select
|
||
|
||
from app.database import async_session
|
||
from app.models.story import Story
|
||
from app.models.transcript import Transcript
|
||
from app.services.docx_parser import parse_transcript
|
||
from app.services.preprocessor import Preprocessor
|
||
from app.services.llm_client import llm_client
|
||
from app.state import TaskState
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 并发控制
|
||
MAX_CONCURRENT_FILES = 3
|
||
MAX_CONCURRENT_SEGMENTS = 5
|
||
|
||
# 短段落阈值（字数）：<=4000 字直接提取事实，>4000 字先判断是否有故事
|
||
SHORT_THRESHOLD = 4000
|
||
|
||
# 切片参数
|
||
CHUNK_MAX_CHARS = 3000
|
||
CHUNK_OVERLAP_PARAGRAPHS = 5
|
||
|
||
|
||
# ============================================================
|
||
# 主入口
|
||
# ============================================================
|
||
async def run_extraction(state: TaskState):
|
||
"""主提取流程：文件间串行（避免 API 限流），文件内各步骤并行"""
|
||
state.is_running = True
|
||
try:
|
||
async with async_session() as db:
|
||
result = await db.execute(
|
||
select(Transcript).where(Transcript.status == "pending")
|
||
)
|
||
transcripts = result.scalars().all()
|
||
|
||
if not transcripts:
|
||
state.update(status="completed", percent=100)
|
||
return
|
||
|
||
file_total = len(transcripts)
|
||
state.update(
|
||
status="running", percent=0,
|
||
files_total=file_total, files_completed=0,
|
||
stories_found=0,
|
||
current_file="", action=f"共发现 {file_total} 份文字稿，开始处理...",
|
||
phase="preprocess", phase_percent=0, phase_detail="准备中",
|
||
)
|
||
|
||
total_stories = [0]
|
||
|
||
# 文件间串行处理（避免 API 限流）
|
||
for file_index, transcript in enumerate(transcripts):
|
||
await _process_file(state, file_index, file_total, transcript, total_stories)
|
||
|
||
state.update(
|
||
status="completed", percent=100,
|
||
files_completed=file_total,
|
||
stories_found=total_stories[0],
|
||
current_file="", action="全部处理完成！",
|
||
phase="preprocess", phase_percent=100, phase_detail="全部完成",
|
||
)
|
||
state.is_running = False
|
||
|
||
except asyncio.CancelledError:
|
||
state.update(status="stopped", action="提取已停止")
|
||
state.is_running = False
|
||
except Exception as e:
|
||
logger.exception("Extraction failed")
|
||
state.update(status="error", action=str(e))
|
||
state.is_running = False
|
||
|
||
|
||
# ============================================================
|
||
# 单文件处理（五步流水线）
|
||
# ============================================================
|
||
async def _process_file(state: TaskState, file_index: int, file_total: int,
|
||
transcript: Transcript, total_stories: list):
|
||
"""处理单个文件 - 五步流水线"""
|
||
try:
|
||
async with async_session() as db:
|
||
t = await db.get(Transcript, transcript.id)
|
||
if not t:
|
||
_increment_files_completed(state)
|
||
return
|
||
|
||
t.status = "processing"
|
||
await db.commit()
|
||
|
||
# 计算该文件在总进度中的权重区间
|
||
# 预处理 0-5%, 判断+提取 5-30%, 切片提取 30-55%, 重组 55-60%, 生成 60-95%, 完成 100%
|
||
# 每个文件占 95/file_total 的权重
|
||
file_weight = 95.0 / max(file_total, 1)
|
||
file_start_percent = file_index * file_weight
|
||
|
||
def _file_percent(phase_start_ratio, phase_end_ratio, inner_ratio=1.0):
|
||
"""计算当前文件在总进度中的百分比"""
|
||
phase_range = phase_end_ratio - phase_start_ratio
|
||
return min(99, int(file_start_percent + file_weight * (phase_start_ratio + phase_range * inner_ratio)))
|
||
|
||
state.update(
|
||
current_file=t.filename, action="正在读取文件...",
|
||
phase="preprocess", phase_percent=0, phase_detail="读取文件",
|
||
)
|
||
|
||
# 解析 DOCX
|
||
lines = await asyncio.to_thread(parse_transcript, t.file_path)
|
||
if not lines:
|
||
t.status = "completed"
|
||
t.error_message = "文件为空"
|
||
await db.commit()
|
||
_increment_files_completed(state)
|
||
return
|
||
|
||
# ========== 第一步：代码预处理（纯代码，不用 LLM）==========
|
||
state.update(
|
||
current_file=t.filename, action="正在分析对话结构...",
|
||
phase="preprocess", phase_percent=10, phase_detail="统计说话人",
|
||
)
|
||
|
||
preprocessor = Preprocessor()
|
||
filtered_lines = [p for p in lines if not preprocessor.is_management_line(p)]
|
||
|
||
if not filtered_lines:
|
||
t.status = "completed"
|
||
t.error_message = "过滤后无有效对话内容"
|
||
await db.commit()
|
||
_increment_files_completed(state)
|
||
return
|
||
|
||
segments, teacher = _preprocess_segments(filtered_lines)
|
||
logger.info(f"预处理完成: 识别老师={teacher}, 有效段落={len(segments)}")
|
||
|
||
if not segments:
|
||
t.status = "completed"
|
||
t.error_message = "未发现学员对话段落"
|
||
await db.commit()
|
||
_increment_files_completed(state)
|
||
return
|
||
|
||
state.update(
|
||
current_file=t.filename,
|
||
action=f"发现 {len(segments)} 个学员段落，开始识别故事...",
|
||
phase="preprocess", phase_percent=100, phase_detail="预处理完成",
|
||
percent=_file_percent(0, 0.05),
|
||
)
|
||
|
||
# ========== 第二步：判断故事 + 提取事实（并行）==========
|
||
state.update(
|
||
current_file=t.filename,
|
||
action=f"正在判断 {len(segments)} 个段落是否包含故事...",
|
||
phase="identify", phase_percent=0,
|
||
phase_detail=f"准备处理 {len(segments)} 个段落",
|
||
)
|
||
|
||
seg_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
|
||
identify_results = [None] * len(segments)
|
||
identify_errors = []
|
||
|
||
async def process_segment_for_identify(seg_idx, speaker, seg):
|
||
"""对单个段落进行判断+提取"""
|
||
async with seg_semaphore:
|
||
text = _segment_to_text(seg)
|
||
text_len = len(text)
|
||
|
||
# 更新进度
|
||
inner_pct = int((seg_idx + 1) / len(segments) * 100)
|
||
state.update(
|
||
phase="identify", phase_percent=inner_pct,
|
||
phase_detail=f"正在处理段落 {seg_idx + 1}/{len(segments)}（{speaker}, {text_len}字）",
|
||
percent=_file_percent(0.05, 0.30, (seg_idx + 1) / len(segments)),
|
||
)
|
||
|
||
if text_len <= SHORT_THRESHOLD:
|
||
# 短段落：跳过判断，直接提取事实
|
||
logger.info(f" 段落 {seg_idx}: {speaker} ({text_len}字) - 短段落，直接提取事实")
|
||
facts = await _extract_facts_from_chunk(text, "")
|
||
if facts:
|
||
hints = facts.get("story_hints", "") or f"{speaker}的故事"
|
||
return (speaker, seg, hints, [facts])
|
||
else:
|
||
logger.info(f" 段落 {seg_idx}: 提取失败，跳过")
|
||
return None
|
||
else:
|
||
# 长段落：先判断是否有故事
|
||
is_story, hints = await _identify_story(text)
|
||
logger.info(f" 段落 {seg_idx}: {speaker} ({text_len}字) - is_story={is_story}, hints={hints}")
|
||
if is_story:
|
||
return (speaker, seg, hints, None) # None 表示需要第三步切片提取
|
||
else:
|
||
logger.info(f" 段落 {seg_idx}: 无故事，跳过")
|
||
return None
|
||
|
||
tasks = [
|
||
process_segment_for_identify(i, speaker, seg)
|
||
for i, (speaker, seg) in enumerate(segments)
|
||
]
|
||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||
|
||
# 收集结果：分为两类
|
||
# - 直接提取到事实的（短段落）
|
||
# - 需要切片提取的（长段落，有故事）
|
||
all_facts = [] # 最终的 (speaker, seg, hints, facts_list) 列表
|
||
needs_chunking = [] # 需要进入第三步的 (speaker, seg, hints)
|
||
|
||
for i, r in enumerate(results):
|
||
if isinstance(r, Exception):
|
||
error_str = str(r)
|
||
logger.warning(f"段落 {i} 处理异常: {error_str}")
|
||
identify_errors.append(error_str)
|
||
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
|
||
raise r
|
||
continue
|
||
if r is None:
|
||
continue
|
||
speaker, seg, hints, facts_list = r
|
||
if facts_list is not None:
|
||
# 短段落已直接提取到事实
|
||
all_facts.append((speaker, seg, hints, facts_list))
|
||
else:
|
||
# 长段落有故事，需要切片提取
|
||
needs_chunking.append((speaker, seg, hints))
|
||
|
||
if not all_facts and not needs_chunking:
|
||
t.status = "completed"
|
||
t.error_message = "未发现包含个人故事的段落"
|
||
await db.commit()
|
||
_increment_files_completed(state)
|
||
state.update(
|
||
percent=_file_percent(0.05, 0.30, 1.0),
|
||
action="未发现故事",
|
||
)
|
||
return
|
||
|
||
logger.info(f"第二步完成: 直接提取={len(all_facts)}, 需切片={len(needs_chunking)}")
|
||
|
||
# 第二步完成时已知道发现了多少个故事，立即更新
|
||
discovered = len(all_facts) + len(needs_chunking)
|
||
total_stories[0] += discovered
|
||
state.update(stories_found=total_stories[0])
|
||
|
||
# ========== 第三步：切片提取事实（并行）==========
|
||
if needs_chunking:
|
||
state.update(
|
||
current_file=t.filename,
|
||
action=f"正在切片提取 {len(needs_chunking)} 个故事...",
|
||
phase="chunk", phase_percent=0,
|
||
phase_detail=f"准备切片 {len(needs_chunking)} 个故事段落",
|
||
)
|
||
|
||
chunk_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
|
||
|
||
async def process_chunked_story(story_idx, speaker, seg, hints):
|
||
"""对单个长段落进行切片提取"""
|
||
async with chunk_semaphore:
|
||
text = _segment_to_text(seg)
|
||
chunks = _chunk_text_by_paragraphs(text, max_chars=CHUNK_MAX_CHARS, overlap_paragraphs=CHUNK_OVERLAP_PARAGRAPHS)
|
||
logger.info(f" 故事 {story_idx}: {speaker} - 切成 {len(chunks)} 块")
|
||
|
||
facts_list = []
|
||
chunk_facts = await asyncio.gather(
|
||
*[_extract_facts_from_chunk(chunk, hints) for chunk in chunks],
|
||
return_exceptions=True,
|
||
)
|
||
for j, facts in enumerate(chunk_facts):
|
||
inner_pct = int((story_idx * 10 + j + 1) / (len(needs_chunking) * 10) * 100)
|
||
state.update(
|
||
phase="chunk", phase_percent=inner_pct,
|
||
phase_detail=f"正在切片提取 故事{story_idx + 1}/{len(needs_chunking)} 块{j + 1}/{len(chunks)}",
|
||
percent=_file_percent(0.30, 0.55, (story_idx + j / max(len(chunks), 1)) / len(needs_chunking)),
|
||
)
|
||
if isinstance(facts, Exception):
|
||
logger.warning(f" 块{j}提取异常: {facts}")
|
||
continue
|
||
if facts:
|
||
facts_list.append(facts)
|
||
|
||
return (speaker, seg, hints, facts_list)
|
||
|
||
chunk_tasks = [
|
||
process_chunked_story(i, speaker, seg, hints)
|
||
for i, (speaker, seg, hints) in enumerate(needs_chunking)
|
||
]
|
||
chunk_results = await asyncio.gather(*chunk_tasks, return_exceptions=True)
|
||
|
||
for r in chunk_results:
|
||
if isinstance(r, Exception):
|
||
error_str = str(r)
|
||
logger.warning(f"切片提取异常: {error_str}")
|
||
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
|
||
raise r
|
||
continue
|
||
if r is not None:
|
||
all_facts.append(r)
|
||
|
||
logger.info(f"第三步完成: 总共 {len(all_facts)} 个故事段落完成事实提取")
|
||
|
||
state.update(
|
||
current_file=t.filename,
|
||
action="切片提取完成，正在重组素材...",
|
||
phase="chunk", phase_percent=100,
|
||
phase_detail="切片提取完成",
|
||
percent=_file_percent(0.30, 0.55, 1.0),
|
||
)
|
||
|
||
# ========== 第四步：重组（纯代码，无需 LLM）==========
|
||
state.update(
|
||
current_file=t.filename,
|
||
action="正在重组素材...",
|
||
phase="merge", phase_percent=50,
|
||
phase_detail="去重合并素材",
|
||
percent=_file_percent(0.55, 0.60, 0.5),
|
||
)
|
||
|
||
merged = _merge_facts(all_facts)
|
||
logger.info(f"第四步完成: 重组出 {len(merged)} 个故事素材")
|
||
|
||
if not merged:
|
||
t.status = "completed"
|
||
t.error_message = "素材重组后为空"
|
||
await db.commit()
|
||
_increment_files_completed(state)
|
||
return
|
||
|
||
state.update(
|
||
current_file=t.filename,
|
||
action=f"重组完成，正在生成 {len(merged)} 个故事...",
|
||
phase="merge", phase_percent=100,
|
||
phase_detail="重组完成",
|
||
percent=_file_percent(0.55, 0.60, 1.0),
|
||
)
|
||
|
||
# ========== 第五步：生成最终故事（并行）==========
|
||
state.update(
|
||
current_file=t.filename,
|
||
action=f"正在生成 {len(merged)} 个故事...",
|
||
phase="generate", phase_percent=0,
|
||
phase_detail=f"准备生成 {len(merged)} 个故事",
|
||
percent=_file_percent(0.60, 0.95, 0),
|
||
)
|
||
|
||
gen_semaphore = asyncio.Semaphore(MAX_CONCURRENT_SEGMENTS)
|
||
|
||
async def generate_one_story(story_idx, speaker, seg, material):
|
||
"""生成单个故事并保存到数据库"""
|
||
async with gen_semaphore:
|
||
inner_pct = int((story_idx + 1) / len(merged) * 100)
|
||
state.update(
|
||
phase="generate", phase_percent=inner_pct,
|
||
phase_detail=f"正在生成故事 {story_idx + 1}/{len(merged)}（{speaker}）",
|
||
percent=_file_percent(0.60, 0.95, (story_idx + 1) / len(merged)),
|
||
)
|
||
|
||
story_data = await _generate_story(material)
|
||
if not story_data:
|
||
logger.warning(f" 故事 {story_idx}: 生成失败")
|
||
return None
|
||
|
||
# 保存到数据库
|
||
raw_content = story_data.get("content", {})
|
||
if isinstance(raw_content, dict):
|
||
content_str = _format_story_content(raw_content)
|
||
else:
|
||
content_str = str(raw_content)
|
||
|
||
story = Story(
|
||
title=story_data.get("title", "未命名故事"),
|
||
summary=story_data.get("summary", ""),
|
||
tags=json.dumps(story_data.get("tags", []), ensure_ascii=False),
|
||
content=content_str,
|
||
raw_material=json.dumps(material, ensure_ascii=False),
|
||
speaker_nickname=story_data.get("speaker_nickname", ""),
|
||
source_transcript_id=t.id,
|
||
source_lines=json.dumps([l.get("index", 0) for l in seg if isinstance(l, dict)]),
|
||
duration_minutes=_estimate_duration(seg),
|
||
confidence=story_data.get("confidence", 0.7),
|
||
confidence_level=_confidence_level(story_data.get("confidence", 0.7)),
|
||
)
|
||
db.add(story)
|
||
await db.commit()
|
||
|
||
state.update(
|
||
action=f"生成第 {total_stories[0]} 个故事：{story.title}",
|
||
)
|
||
logger.info(f" 故事 {story_idx}: 保存成功 - {story.title}")
|
||
|
||
return story
|
||
|
||
gen_tasks = [
|
||
generate_one_story(i, speaker, seg, material)
|
||
for i, (speaker, seg, material) in enumerate(merged)
|
||
]
|
||
gen_results = await asyncio.gather(*gen_tasks, return_exceptions=True)
|
||
|
||
for r in gen_results:
|
||
if isinstance(r, Exception):
|
||
error_str = str(r)
|
||
logger.warning(f"故事生成异常: {error_str}")
|
||
if "529" in error_str or "overloaded" in error_str.lower() or "timed out" in error_str.lower():
|
||
raise r
|
||
|
||
t.status = "completed"
|
||
await db.commit()
|
||
_increment_files_completed(state)
|
||
state.update(
|
||
current_file=t.filename,
|
||
action=f"文件处理完成，共生成 {total_stories[0]} 个故事",
|
||
phase="generate", phase_percent=100,
|
||
phase_detail="文件处理完成",
|
||
percent=_file_percent(0.60, 0.95, 1.0),
|
||
)
|
||
|
||
except Exception as e:
|
||
error_str = str(e)
|
||
logger.exception(f"Error processing file {transcript.filename}")
|
||
try:
|
||
async with async_session() as db:
|
||
t = await db.get(Transcript, transcript.id)
|
||
if t:
|
||
t.status = "error"
|
||
t.error_message = error_str
|
||
await db.commit()
|
||
except Exception:
|
||
pass
|
||
_increment_files_completed(state)
|
||
|
||
if "529" in error_str or "overloaded" in error_str.lower():
|
||
state.update(status="error", error="AI 服务暂时过载（529），请稍后再试")
|
||
elif "timed out" in error_str.lower():
|
||
state.update(status="error", error="AI 服务请求超时，请稍后再试")
|
||
else:
|
||
state.update(status="error", error=f"处理文件时出错：{error_str[:200]}")
|
||
|
||
|
||
# ============================================================
|
||
# 第一步：代码预处理
|
||
# ============================================================
|
||
def _preprocess_segments(lines: list[dict]) -> tuple[list[tuple[str, list[dict]]], str]:
|
||
"""
|
||
代码预处理（v3：时间范围截取，允许重叠）。
|
||
识别老师 → 过滤杂音（<20句）→ 按时间范围截取 → 过滤短段落（<15句/<80字）→ 过滤非主讲述人（占比<10%）
|
||
|
||
返回: (segments, teacher)
|
||
segments: [(speaker, segment_lines), ...]
|
||
teacher: 老师名称
|
||
"""
|
||
# 1. 统计每个说话人
|
||
speaker_stats = {}
|
||
for i, line in enumerate(lines):
|
||
speaker = line.get("speaker", "")
|
||
if not speaker:
|
||
continue
|
||
if speaker not in speaker_stats:
|
||
speaker_stats[speaker] = {"lines": 0, "chars": 0, "first": i, "last": i}
|
||
speaker_stats[speaker]["lines"] += 1
|
||
speaker_stats[speaker]["chars"] += len(line.get("content", ""))
|
||
speaker_stats[speaker]["last"] = i
|
||
|
||
# 2. 识别老师（发言最多的人）
|
||
teacher = max(speaker_stats, key=lambda s: speaker_stats[s]["lines"])
|
||
logger.info(f"识别老师: {teacher} ({speaker_stats[teacher]['lines']}行, {speaker_stats[teacher]['chars']}字)")
|
||
|
||
# 3. 过滤杂音（< 20句的说话人）+ 排除老师
|
||
students = []
|
||
noise_speakers = []
|
||
for s, stats in speaker_stats.items():
|
||
if s == teacher:
|
||
continue
|
||
if stats["lines"] < 20:
|
||
noise_speakers.append(f"{s} ({stats['lines']}句)")
|
||
continue
|
||
students.append(s)
|
||
|
||
if noise_speakers:
|
||
logger.info(f"杂音过滤: {', '.join(noise_speakers)}")
|
||
logger.info(f"有效学员: {students}")
|
||
|
||
# 4. 按时间范围截取：每个学员从第一句到最后一句，中间所有行全部截取
|
||
segments = []
|
||
for speaker in students:
|
||
stats = speaker_stats[speaker]
|
||
first_idx = stats["first"]
|
||
last_idx = stats["last"]
|
||
seg = lines[first_idx:last_idx + 1]
|
||
segments.append((speaker, seg))
|
||
|
||
# 5. 过滤：短段落 + 当事人占比过低（非主讲述人）
|
||
final_segments = []
|
||
for speaker, seg in segments:
|
||
student_lines = [l for l in seg if l.get("speaker", "") == speaker]
|
||
if len(student_lines) < 15:
|
||
logger.info(f"过滤: {speaker} ({len(student_lines)}句<15)")
|
||
continue
|
||
student_chars = sum(len(l.get("content", "")) for l in student_lines)
|
||
if student_chars < 80:
|
||
logger.info(f"过滤: {speaker} ({student_chars}字<80)")
|
||
continue
|
||
# 当事人说话占比 < 10% → 非主讲述人，丢弃
|
||
ratio = len(student_lines) / len(seg) if seg else 0
|
||
if ratio < 0.1:
|
||
logger.info(f"过滤: {speaker} (占比{ratio:.1%}<10%)")
|
||
continue
|
||
final_segments.append((speaker, seg))
|
||
|
||
logger.info(f"最终: {len(final_segments)} 个学员段落")
|
||
for i, (speaker, seg) in enumerate(final_segments):
|
||
student_lines = [l for l in seg if l.get("speaker", "") == speaker]
|
||
student_chars = sum(len(l.get("content", "")) for l in student_lines)
|
||
ratio = len(student_lines) / len(seg) if seg else 0
|
||
logger.info(f" 段落{i}: {speaker}, 截取{len(seg)}行, 学员{len(student_lines)}句/{student_chars}字, 占比{ratio:.1%}")
|
||
|
||
return final_segments, teacher
|
||
|
||
|
||
# ============================================================
|
||
# 第二步：判断故事
|
||
# ============================================================
|
||
async def _identify_story(text: str) -> tuple[bool, str]:
|
||
"""判断对话段落是否包含个人故事"""
|
||
prompt = """判断这段对话是否包含学员的个人故事。直接输出JSON，不要分析。
|
||
|
||
个人故事特征（满足至少2条）：学员讲述具体经历、包含时间地点人物细节、表达深层情感、老师深入引导疗愈、揭示心理或家庭问题。
|
||
|
||
非故事内容：纯课堂讲解、简单问答寒暄、课堂管理、纯疗愈引导词。
|
||
|
||
直接输出JSON：
|
||
{"is_story": true或false, "confidence": 0.0到1.0, "story_hints": "一句话描述故事主题和讲述者"}
|
||
|
||
对话内容：
|
||
```
|
||
""" + text[:4000] + """
|
||
```"""
|
||
|
||
try:
|
||
result = await llm_client.chat_json([{"role": "user", "content": prompt}], temperature=0.2)
|
||
if result:
|
||
is_story = result.get("is_story", False)
|
||
confidence = result.get("confidence", 0)
|
||
hints = result.get("story_hints", "")
|
||
logger.info(f"Story identification: is_story={is_story}, confidence={confidence}, hints={hints[:80]}")
|
||
return bool(is_story) and confidence >= 0.5, hints
|
||
except Exception as e:
|
||
logger.warning(f"Story identification error: {e}")
|
||
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
|
||
raise
|
||
|
||
return False, ""
|
||
|
||
|
||
# ============================================================
|
||
# 第三步：切片提取事实
|
||
# ============================================================
|
||
def _chunk_text_by_paragraphs(text: str, max_chars: int = 3000, overlap_paragraphs: int = 5) -> list[str]:
|
||
"""按自然段切分，在 max_chars 附近找段落结尾切，重叠 overlap_paragraphs 个自然段"""
|
||
paragraphs = [p for p in text.split("\n") if p.strip()]
|
||
if not paragraphs:
|
||
return [text]
|
||
if len(text) <= max_chars:
|
||
return [text]
|
||
|
||
chunks = []
|
||
i = 0
|
||
while i < len(paragraphs):
|
||
# 从当前位置开始，累加段落直到超过 max_chars
|
||
current = []
|
||
current_len = 0
|
||
j = i
|
||
while j < len(paragraphs):
|
||
para = paragraphs[j]
|
||
# 超过 max_chars 且已有内容 → 在这个段落前切断（段落结尾切）
|
||
if current_len + len(para) > max_chars and current:
|
||
break
|
||
current.append(para)
|
||
current_len += len(para)
|
||
j += 1
|
||
|
||
chunks.append("\n".join(current))
|
||
|
||
# 到末尾了就结束
|
||
if j >= len(paragraphs):
|
||
break
|
||
|
||
# 下一次起点：当前切到的位置 - 重叠段落数
|
||
next_i = j - overlap_paragraphs
|
||
# 防止死循环：确保至少前进1个段落
|
||
if next_i <= i:
|
||
next_i = i + 1
|
||
i = next_i
|
||
|
||
return chunks
|
||
|
||
|
||
async def _extract_facts_from_chunk(text: str, hints: str) -> dict | None:
|
||
"""从对话切片中提取关键信息（事件、情感、原话、人物）"""
|
||
prompt = f"""从这段对话中提取关键信息。只输出JSON。
|
||
|
||
故事主题：{hints}
|
||
|
||
提取以下内容：
|
||
- events: 发生的事件（尽量详细，保留具体细节）
|
||
- emotions: 表达的情感
|
||
- quotes: 讲述者的原话（保留原话，不要改写，尽量多提取；完全相同的重复语句只保留一条）
|
||
- key_people: 提到的人物
|
||
|
||
{{"events":["事件1","事件2"],"emotions":["情感1","情感2"],"quotes":["原话1","原话2","原话3"],"key_people":["人物1"]}}
|
||
|
||
对话内容：
|
||
```
|
||
{text}
|
||
```"""
|
||
|
||
try:
|
||
result = await llm_client.chat_json([{"role": "user", "content": prompt}], temperature=0.1)
|
||
if not result:
|
||
return None
|
||
# chat_json 可能返回 list（解析错误），需要转为 dict
|
||
if isinstance(result, list):
|
||
for item in result:
|
||
if isinstance(item, dict) and ("events" in item or "quotes" in item):
|
||
result = item
|
||
break
|
||
else:
|
||
logger.warning(f"chat_json returned list, cannot convert to dict: {result[:3]}")
|
||
return None
|
||
return result
|
||
except Exception as e:
|
||
logger.warning(f"Extract facts error: {e}")
|
||
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
|
||
raise
|
||
|
||
return None
|
||
|
||
|
||
# ============================================================
|
||
# 第四步：重组（纯代码，无需 LLM）
|
||
# ============================================================
|
||
def _fuzzy_dedup_quotes(quotes: list[str]) -> list[str]:
|
||
"""
|
||
模糊去重：对高度相似的语句只保留第一条。
|
||
策略：提取每条语句的核心部分（去掉常见前缀后缀），相似度超过阈值的视为重复。
|
||
"""
|
||
import re
|
||
|
||
def normalize(s: str) -> str:
|
||
"""提取语句核心，去掉语气词和细微差异"""
|
||
s = s.strip()
|
||
# 去掉「」包裹
|
||
if s.startswith("「") and s.endswith("」"):
|
||
s = s[1:-1]
|
||
# 去掉常见前缀
|
||
for prefix in ["嗯", "啊", "呃", "就是", "然后", "那个", "所以"]:
|
||
if s.startswith(prefix):
|
||
s = s[len(prefix):].strip()
|
||
# 去掉常见尾缀
|
||
for suffix in ["啊", "吧", "呢", "嘛", "哦", "呀"]:
|
||
if s.endswith(suffix) and len(s) > 4:
|
||
s = s[:-1]
|
||
# 去掉多余空白和重复字（如"我我我"、"的的的"）
|
||
s = re.sub(r'(.)\1{2,}', r'\1', s)
|
||
return s
|
||
|
||
def similarity(a: str, b: str) -> float:
|
||
"""简单的字符级相似度（Jaccard on character bigrams）"""
|
||
if not a or not b:
|
||
return 0.0
|
||
a_bigrams = set(a[i:i+2] for i in range(len(a)-1))
|
||
b_bigrams = set(b[i:i+2] for i in range(len(b)-1))
|
||
if not a_bigrams or not b_bigrams:
|
||
return 0.0
|
||
intersection = a_bigrams & b_bigrams
|
||
union = a_bigrams | b_bigrams
|
||
return len(intersection) / len(union)
|
||
|
||
kept = []
|
||
kept_normalized = []
|
||
for q in quotes:
|
||
nq = normalize(q)
|
||
is_dup = False
|
||
for kn in kept_normalized:
|
||
# 短语句用高阈值，长语句用低阈值
|
||
threshold = 0.8 if len(nq) < 15 else 0.7
|
||
if similarity(nq, kn) >= threshold:
|
||
is_dup = True
|
||
break
|
||
if not is_dup:
|
||
kept.append(q)
|
||
kept_normalized.append(nq)
|
||
|
||
removed = len(quotes) - len(kept)
|
||
if removed > 0:
|
||
logger.info(f" 模糊去重: {len(quotes)} -> {len(kept)} (去除 {removed} 条相似重复)")
|
||
|
||
return kept
|
||
|
||
|
||
def _merge_facts(all_facts: list) -> list[tuple[str, list, dict]]:
|
||
"""
|
||
重组素材（Reduce）。
|
||
对每个故事段落的所有切片事实进行合并去重：
|
||
- 事件按原文顺序保留
|
||
- 情感去重
|
||
- 原话去重（不限制数量）
|
||
- 人物去重
|
||
|
||
输入: [(speaker, seg, hints, facts_list), ...]
|
||
输出: [(speaker, seg, material), ...]
|
||
material: {"events": [...], "emotions": [...], "quotes": [...], "key_people": [...], "story_hints": "..."}
|
||
"""
|
||
merged = []
|
||
for i, (speaker, seg, hints, facts_list) in enumerate(all_facts):
|
||
all_events, all_emotions, all_quotes, all_people = [], [], [], []
|
||
for facts in facts_list:
|
||
if isinstance(facts, dict):
|
||
all_events.extend(facts.get("events", []))
|
||
all_emotions.extend(facts.get("emotions", []))
|
||
all_quotes.extend(facts.get("quotes", []))
|
||
all_people.extend(facts.get("key_people", []))
|
||
|
||
# 情感去重（保持顺序）
|
||
all_emotions = list(dict.fromkeys(all_emotions))
|
||
# 人物去重（保持顺序）
|
||
all_people = list(dict.fromkeys(all_people))
|
||
# 原话去重（保持顺序，不限制数量）
|
||
# 第一轮：精确去重
|
||
seen_quotes = set()
|
||
unique_quotes = [q for q in all_quotes if q not in seen_quotes and not seen_quotes.add(q)]
|
||
# 第二轮：模糊去重（去除重复模式的语句，如释放语句只保留一条）
|
||
unique_quotes = _fuzzy_dedup_quotes(unique_quotes)
|
||
|
||
material = {
|
||
"events": all_events,
|
||
"emotions": all_emotions,
|
||
"quotes": unique_quotes,
|
||
"key_people": all_people,
|
||
"story_hints": hints,
|
||
}
|
||
|
||
logger.info(f" 重组故事 {i}: {speaker} - 事件:{len(all_events)}, 情感:{len(all_emotions)}, "
|
||
f"原话:{len(unique_quotes)}, 人物:{len(all_people)}")
|
||
|
||
merged.append((speaker, seg, material))
|
||
|
||
return merged
|
||
|
||
|
||
# ============================================================
|
||
# 第五步：生成最终故事
|
||
# ============================================================
|
||
async def _generate_story(material: dict) -> dict | None:
|
||
"""基于素材生成第三人称故事（5部分结构化输出）"""
|
||
prompt = f"""基于素材生成第三人称故事。只输出JSON，不要其他内容。
|
||
|
||
素材：
|
||
主题：{material['story_hints']}
|
||
事件：{json.dumps(material['events'], ensure_ascii=False)}
|
||
情感：{json.dumps(material['emotions'], ensure_ascii=False)}
|
||
原话：{json.dumps(material['quotes'], ensure_ascii=False)}
|
||
人物：{json.dumps(material['key_people'], ensure_ascii=False)}
|
||
|
||
写作手法（非常重要）：
|
||
|
||
【双时空结构】
|
||
故事有两个时空层：
|
||
- "当下时空"：学员在课堂上与卢慧老师的对话——这是故事的框架，用对话体呈现，保留师生互动的完整脉络
|
||
- "过往时空"：学员讲述的过去经历——这是故事的核心，用"场景重建"手法呈现，让读者穿越到那个时空
|
||
|
||
【场景重建手法（用于"过往的故事"部分）】
|
||
当学员讲述过去的经历时，不要写成"她说她小时候…"这种转述体，而是直接把读者拉进那个时空：
|
||
- 直接切入场景，用过去时态叙述。比如不要写"文晔回忆说她的儿子15岁去了美国"，而要写"十五岁那年，文晔的儿子独自登上了飞往美国的航班"
|
||
- 用五感描写构建画面：她看到了什么、听到了什么、身体感受到了什么（比如"手心出汗"、"脚发凉"这些身体反应要融入场景中）
|
||
- 用环境细节暗示情绪：不要直接说"她很痛苦"，而是通过场景让读者感受到痛苦
|
||
- 关键情感爆发处用直接引语（「」包裹原话），其他地方用间接引语和叙述自然融合
|
||
- 场景之间要有过渡，比如"然而，命运的转折发生在前年冬天——"
|
||
|
||
【对话体手法（用于"当下的困境"和"转变"部分）】
|
||
- 保留师生对话的现场感，写出谁说了什么、对方如何回应
|
||
- 在对话中穿插学员的内心活动、身体反应、情绪变化
|
||
- 老师的引导语要完整保留，这是疗愈过程的关键
|
||
|
||
【通用要求】
|
||
- 每个事件都要展开写，写出具体过程和细节，不要一句话概括
|
||
- 事件之间要有过渡和因果逻辑，不能跳跃
|
||
- 情感变化要写出过程：从什么情感转变到什么情感，中间经历了什么
|
||
- 不编造素材中没有的内容，但可以根据素材合理补充场景细节（如环境、氛围）
|
||
- 每个部分充分展开，不要限制长度，越详细越好
|
||
- 原文金句数组中的每条金句必须用英文双引号包裹，如 ["金句1", "金句2"]，不要用「」替代双引号
|
||
|
||
禁止事项：
|
||
- 禁止用一句话概括多个事件（如"她倾诉了挣扎"、"她经历了疗愈过程"）
|
||
- 禁止省略素材中的具体细节
|
||
- 禁止省略老师的引导和回应
|
||
- 禁止事件之间跳跃，要保持时间顺序和因果逻辑
|
||
- 禁止过度总结或抽象化描述
|
||
- 禁止在"过往的故事"部分全程使用转述体（"她说…她回忆…"），必须用场景重建手法让读者身临其境
|
||
- 禁止为了简洁而牺牲内容的丰富性
|
||
|
||
输出格式：{{"title":"标题","summary":"80字摘要","tags":["标签"],"speaker_nickname":"昵称","content":{{"讲述者":"...","当下的困境":"...","过往的故事":"...","转变":"...","原文金句":["原话1","原话2","原话3"]}}}}"""
|
||
|
||
try:
|
||
result = await llm_client.chat_json(
|
||
[{"role": "user", "content": prompt}],
|
||
temperature=0.7,
|
||
)
|
||
if not result:
|
||
return None
|
||
# chat_json 可能返回 list（解析错误），需要转为 dict
|
||
if isinstance(result, list):
|
||
for item in result:
|
||
if isinstance(item, dict) and "title" in item:
|
||
result = item
|
||
break
|
||
else:
|
||
logger.warning(f"chat_json returned list, cannot convert to dict: {result[:3]}")
|
||
return None
|
||
|
||
# 解析 content 纯文本为结构化字段
|
||
raw_content = result.get("content", "")
|
||
if isinstance(raw_content, str) and "|||" in raw_content:
|
||
parts = raw_content.split("|||")
|
||
content_dict = {}
|
||
keys = ["讲述者", "当下的困境", "过往的故事", "转变", "原文金句"]
|
||
for idx, key in enumerate(keys):
|
||
content_dict[key] = parts[idx].strip() if idx < len(parts) else ""
|
||
result["content"] = content_dict
|
||
elif isinstance(raw_content, dict):
|
||
for key in ["讲述者", "当下的困境", "过往的故事", "转变", "原文金句"]:
|
||
if key not in raw_content:
|
||
raw_content[key] = ""
|
||
result["content"] = raw_content
|
||
else:
|
||
result["content"] = {"完整故事": str(raw_content)}
|
||
|
||
if not result.get("tags"):
|
||
result["tags"] = []
|
||
|
||
return result
|
||
except Exception as e:
|
||
logger.warning(f"Generate story error: {e}")
|
||
if "529" in str(e) or "overloaded" in str(e).lower() or "timed out" in str(e).lower():
|
||
raise
|
||
|
||
return None
|
||
|
||
|
||
# ============================================================
|
||
# 工具函数
|
||
# ============================================================
|
||
def _segment_to_text(segment_lines: list) -> str:
|
||
"""将段落行列表转为对话文本"""
|
||
parts = []
|
||
for line in segment_lines:
|
||
speaker = line.get("speaker", "")
|
||
content = line.get("content", "")
|
||
if speaker:
|
||
parts.append(f"{speaker}: {content}")
|
||
else:
|
||
parts.append(content)
|
||
return "\n".join(parts)
|
||
|
||
|
||
def _increment_files_completed(state: TaskState):
|
||
"""增加已完成文件计数"""
|
||
p = state.progress
|
||
files_completed = p.get("files_completed", 0) + 1
|
||
files_total = p.get("files_total", 1)
|
||
state.update(files_completed=files_completed)
|
||
|
||
|
||
def _estimate_duration(segment_lines: list) -> float:
|
||
"""根据时间戳估算故事时长（分钟）"""
|
||
timestamps = []
|
||
for line in segment_lines:
|
||
if isinstance(line, dict) and line.get("timestamp"):
|
||
try:
|
||
parts = line["timestamp"].split(":")
|
||
if len(parts) >= 2:
|
||
minutes = int(parts[-2]) + int(parts[-1]) / 60
|
||
if len(parts) == 3:
|
||
minutes += int(parts[0]) * 60
|
||
timestamps.append(minutes)
|
||
except (ValueError, IndexError):
|
||
continue
|
||
|
||
if len(timestamps) >= 2:
|
||
return round(max(timestamps) - min(timestamps), 1)
|
||
return 0
|
||
|
||
|
||
def _confidence_level(confidence: float) -> str:
|
||
if confidence >= 0.8:
|
||
return "high"
|
||
elif confidence >= 0.6:
|
||
return "medium"
|
||
return "low"
|
||
|
||
|
||
def _format_story_content(content_dict: dict) -> str:
|
||
"""将 LLM 返回的结构化故事内容转为可读的 Markdown 格式"""
|
||
parts = []
|
||
|
||
# 按顺序输出各个章节
|
||
sections = ["讲述者", "当下的困境", "过往的故事", "转变"]
|
||
for section_name in sections:
|
||
if content_dict.get(section_name):
|
||
parts.append(f"## {section_name}\n\n{content_dict[section_name]}")
|
||
|
||
# 原文金句
|
||
quotes = content_dict.get("原文金句", [])
|
||
if not quotes:
|
||
quotes = content_dict.get("金句摘录", [])
|
||
if quotes and isinstance(quotes, list):
|
||
parts.append("## 原文金句")
|
||
for q in quotes:
|
||
parts.append(f"\n> {q}")
|
||
|
||
# 兼容旧格式：故事梗概
|
||
if content_dict.get("故事梗概") and "讲述者" not in content_dict:
|
||
parts.insert(0, f"## 故事梗概\n\n{content_dict['故事梗概']}")
|
||
|
||
# 兼容旧格式：关键要素
|
||
elements = content_dict.get("关键要素", [])
|
||
if elements and isinstance(elements, list) and "讲述者" not in content_dict:
|
||
parts.append("## 关键要素")
|
||
for elem in elements:
|
||
if isinstance(elem, dict):
|
||
title = elem.get("标题", "")
|
||
desc = elem.get("描述", "")
|
||
parts.append(f"\n### {title}\n\n{desc}")
|
||
|
||
# 如果上面都没匹配到，按通用 key-value 格式输出
|
||
if not parts:
|
||
for key, value in content_dict.items():
|
||
if isinstance(value, str):
|
||
parts.append(f"## {key}\n\n{value}")
|
||
elif isinstance(value, list):
|
||
parts.append(f"## {key}")
|
||
for item in value:
|
||
if isinstance(item, dict):
|
||
for k, v in item.items():
|
||
parts.append(f"\n**{k}**：{v}")
|
||
else:
|
||
parts.append(f"\n- {item}")
|
||
|
||
return "\n\n".join(parts) if parts else json.dumps(content_dict, ensure_ascii=False)
|