feat: add question insight analytics
This commit is contained in:
@@ -19,6 +19,7 @@ from app.models.chat import ChatMessage, ChatSession
|
||||
from app.models.logs import AiRequestLog, OperationLog
|
||||
from app.models.user import User
|
||||
from app.api.pagination import page_result
|
||||
from app.services.question_insight_service import QuestionInsightService
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -222,6 +223,30 @@ def ai_log_detail(
|
||||
return api_success(_ai_log_dict(log, include_prompt=True))
|
||||
|
||||
|
||||
@router.get("/question-insights/summary")
|
||||
def question_insights(
|
||||
dateFrom: datetime | None = Query(default=None),
|
||||
dateTo: datetime | None = Query(default=None),
|
||||
minCount: int = Query(default=2, ge=1, le=50),
|
||||
maxMessages: int = Query(default=5000, ge=100, le=20000),
|
||||
page: int = Query(default=1, ge=1),
|
||||
pageSize: int = Query(default=20, ge=10, le=50),
|
||||
db: Session = Depends(get_db),
|
||||
current_admin: Admin = Depends(get_current_admin),
|
||||
) -> dict:
|
||||
return api_success(
|
||||
QuestionInsightService.summarize(
|
||||
db,
|
||||
date_from=dateFrom,
|
||||
date_to=dateTo,
|
||||
min_count=minCount,
|
||||
max_messages=maxMessages,
|
||||
page=page,
|
||||
page_size=pageSize,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _chat_query(
|
||||
*,
|
||||
keyword: str,
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models.chat import ChatMessage, ChatSession
|
||||
from app.models.user import User
|
||||
|
||||
|
||||
LOW_VALUE_EXACT = {
|
||||
"你好",
|
||||
"您好",
|
||||
"老师好",
|
||||
"在吗",
|
||||
"好的",
|
||||
"好",
|
||||
"嗯",
|
||||
"嗯嗯",
|
||||
"收到",
|
||||
"谢谢",
|
||||
"感谢",
|
||||
"明白",
|
||||
"可以",
|
||||
"ok",
|
||||
"OK",
|
||||
}
|
||||
|
||||
COURTESY_PREFIXES = (
|
||||
"老师你好",
|
||||
"老师您好",
|
||||
"老师好",
|
||||
"你好",
|
||||
"您好",
|
||||
"请问一下",
|
||||
"请问",
|
||||
"想问一下",
|
||||
"麻烦问下",
|
||||
"我想问一下",
|
||||
"我想问问",
|
||||
)
|
||||
|
||||
COURTESY_SUFFIXES = (
|
||||
"谢谢老师",
|
||||
"谢谢",
|
||||
"感谢老师",
|
||||
"感谢",
|
||||
"麻烦老师",
|
||||
"辛苦老师",
|
||||
)
|
||||
|
||||
SYNONYM_RULES = (
|
||||
(re.compile(r"(作业|练习|功课|课后任务|课后练习)"), "功课"),
|
||||
(re.compile(r"(回放|录播|视频回看|回看)"), "回放"),
|
||||
(re.compile(r"(会议链接|会议号|直播链接|上课链接|腾讯会议|飞书会议)"), "会议链接"),
|
||||
(re.compile(r"(助教|助理|班主任|辅导老师)"), "课程助理"),
|
||||
(re.compile(r"(上课|直播|带练|带领练习)"), "上课安排"),
|
||||
(re.compile(r"(怎么做|如何做|咋做|具体步骤|操作步骤|怎么操作|具体操作)"), "怎么做"),
|
||||
(re.compile(r"(是什么|什么意思|啥意思|定义|区别)"), "是什么"),
|
||||
)
|
||||
|
||||
NOISE_PATTERN = re.compile(r"[\s\u3000,，。！？!?；;：:、“”\"'‘’（）()\[\]【】<>《》]+")
|
||||
QUESTION_SPLIT_PATTERN = re.compile(
|
||||
r"(?:\n+|[？?]\s*|(?:^|\n|\s)[0-9一二三四五六七八九十]+[、.．]\s*)"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CleanedQuestion:
|
||||
raw: str
|
||||
text: str
|
||||
normalized: str
|
||||
user_id: int
|
||||
user_name: str
|
||||
user_phone: str
|
||||
session_id: int
|
||||
message_id: int
|
||||
created_at: datetime
|
||||
tokens: set[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class QuestionCluster:
|
||||
title: str
|
||||
normalized: str
|
||||
tokens: set[str]
|
||||
questions: list[CleanedQuestion] = field(default_factory=list)
|
||||
|
||||
def add(self, question: CleanedQuestion) -> None:
|
||||
self.questions.append(question)
|
||||
if len(question.normalized) < len(self.normalized) or _looks_more_question_like(question.text, self.title):
|
||||
self.title = question.text
|
||||
self.normalized = question.normalized
|
||||
self.tokens = _merge_tokens(self.tokens, question.tokens)
|
||||
|
||||
|
||||
class QuestionInsightService:
|
||||
@staticmethod
|
||||
def summarize(
|
||||
db: Session,
|
||||
*,
|
||||
date_from: datetime | None = None,
|
||||
date_to: datetime | None = None,
|
||||
min_count: int = 2,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
max_messages: int = 5000,
|
||||
) -> dict:
|
||||
messages = _load_user_messages(db, date_from=date_from, date_to=date_to, limit=max_messages)
|
||||
cleaned, filtered_count = _clean_messages(messages)
|
||||
clusters = _cluster_questions(cleaned)
|
||||
visible_clusters = [cluster for cluster in clusters if len(cluster.questions) >= min_count]
|
||||
visible_clusters.sort(key=lambda item: (len(item.questions), item.questions[-1].created_at), reverse=True)
|
||||
|
||||
total = len(visible_clusters)
|
||||
offset = (page - 1) * page_size
|
||||
page_clusters = visible_clusters[offset : offset + page_size]
|
||||
|
||||
return {
|
||||
"range": {
|
||||
"dateFrom": date_from,
|
||||
"dateTo": date_to,
|
||||
"maxMessages": max_messages,
|
||||
},
|
||||
"summary": {
|
||||
"scannedMessages": len(messages),
|
||||
"cleanedQuestions": len(cleaned),
|
||||
"filteredMessages": filtered_count,
|
||||
"clusterCount": len(clusters),
|
||||
"visibleClusterCount": total,
|
||||
"minCount": min_count,
|
||||
},
|
||||
"items": [_cluster_dict(index + offset + 1, cluster) for index, cluster in enumerate(page_clusters)],
|
||||
"total": total,
|
||||
"page": page,
|
||||
"pageSize": page_size,
|
||||
}
|
||||
|
||||
|
||||
def _load_user_messages(
|
||||
db: Session,
|
||||
*,
|
||||
date_from: datetime | None,
|
||||
date_to: datetime | None,
|
||||
limit: int,
|
||||
) -> list[tuple[ChatMessage, ChatSession | None, User | None]]:
|
||||
query = (
|
||||
select(ChatMessage, ChatSession, User)
|
||||
.join(ChatSession, ChatSession.id == ChatMessage.session_id, isouter=True)
|
||||
.join(User, User.id == ChatMessage.user_id, isouter=True)
|
||||
.where(ChatMessage.role == "user")
|
||||
)
|
||||
if date_from is not None:
|
||||
query = query.where(ChatMessage.created_at >= date_from.replace(tzinfo=None))
|
||||
if date_to is not None:
|
||||
query = query.where(ChatMessage.created_at <= date_to.replace(tzinfo=None))
|
||||
return list(
|
||||
db.execute(
|
||||
query.order_by(ChatMessage.created_at.desc(), ChatMessage.id.desc()).limit(limit)
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def _clean_messages(messages: Iterable[tuple[ChatMessage, ChatSession | None, User | None]]) -> tuple[list[CleanedQuestion], int]:
|
||||
cleaned: list[CleanedQuestion] = []
|
||||
filtered_count = 0
|
||||
seen_parts: set[tuple[int, str]] = set()
|
||||
for message, _session, user in messages:
|
||||
parts = _split_questions(message.content)
|
||||
accepted = 0
|
||||
for part in parts:
|
||||
text = _clean_text(part)
|
||||
if _is_low_value(text):
|
||||
continue
|
||||
normalized = _normalize_question(text)
|
||||
if len(normalized) < 3:
|
||||
continue
|
||||
dedupe_key = (message.id, normalized)
|
||||
if dedupe_key in seen_parts:
|
||||
continue
|
||||
seen_parts.add(dedupe_key)
|
||||
accepted += 1
|
||||
cleaned.append(
|
||||
CleanedQuestion(
|
||||
raw=message.content,
|
||||
text=text,
|
||||
normalized=normalized,
|
||||
user_id=message.user_id,
|
||||
user_name=user.name if user else "",
|
||||
user_phone=user.phone if user else "",
|
||||
session_id=message.session_id,
|
||||
message_id=message.id,
|
||||
created_at=message.created_at,
|
||||
tokens=_tokens(normalized),
|
||||
)
|
||||
)
|
||||
if accepted == 0:
|
||||
filtered_count += 1
|
||||
return cleaned, filtered_count
|
||||
|
||||
|
||||
def _split_questions(content: str) -> list[str]:
|
||||
text = (content or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
rough_parts = [part.strip() for part in QUESTION_SPLIT_PATTERN.split(text) if part.strip()]
|
||||
if len(rough_parts) <= 1:
|
||||
return [text]
|
||||
merged: list[str] = []
|
||||
for part in rough_parts:
|
||||
if len(part) <= 2 and merged:
|
||||
merged[-1] = f"{merged[-1]} {part}"
|
||||
else:
|
||||
merged.append(part)
|
||||
return merged
|
||||
|
||||
|
||||
def _clean_text(text: str) -> str:
|
||||
value = re.sub(r"\s+", " ", text.strip())
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for prefix in COURTESY_PREFIXES:
|
||||
if value.startswith(prefix):
|
||||
value = value[len(prefix) :].lstrip(" ，,。:：")
|
||||
changed = True
|
||||
for suffix in COURTESY_SUFFIXES:
|
||||
if value.endswith(suffix):
|
||||
value = value[: -len(suffix)].rstrip(" ，,。:：")
|
||||
changed = True
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _is_low_value(text: str) -> bool:
|
||||
compact = NOISE_PATTERN.sub("", text)
|
||||
if not compact:
|
||||
return True
|
||||
if compact in LOW_VALUE_EXACT:
|
||||
return True
|
||||
if len(compact) <= 2:
|
||||
return True
|
||||
return len(compact) <= 6 and not any(marker in compact for marker in ("吗", "么", "哪", "谁", "怎么", "如何", "什么", "为啥", "为什么"))
|
||||
|
||||
|
||||
def _normalize_question(text: str) -> str:
|
||||
value = text.lower()
|
||||
for pattern, replacement in SYNONYM_RULES:
|
||||
value = pattern.sub(replacement, value)
|
||||
value = re.sub(r"(吗|呢|呀|啊|嘛)+$", "", value)
|
||||
return NOISE_PATTERN.sub("", value)
|
||||
|
||||
|
||||
def _cluster_questions(questions: list[CleanedQuestion]) -> list[QuestionCluster]:
|
||||
clusters: list[QuestionCluster] = []
|
||||
for question in questions:
|
||||
best_cluster: QuestionCluster | None = None
|
||||
best_score = 0.0
|
||||
for cluster in clusters:
|
||||
score = _similarity(question, cluster)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_cluster = cluster
|
||||
if best_cluster is not None and best_score >= 0.62:
|
||||
best_cluster.add(question)
|
||||
else:
|
||||
clusters.append(QuestionCluster(title=question.text, normalized=question.normalized, tokens=set(question.tokens), questions=[question]))
|
||||
return clusters
|
||||
|
||||
|
||||
def _similarity(question: CleanedQuestion, cluster: QuestionCluster) -> float:
|
||||
if question.normalized == cluster.normalized:
|
||||
return 1.0
|
||||
token_score = _jaccard(question.tokens, cluster.tokens)
|
||||
sequence_score = SequenceMatcher(None, question.normalized, cluster.normalized).ratio()
|
||||
containment_score = _containment(question.normalized, cluster.normalized)
|
||||
return max(token_score, sequence_score * 0.88, containment_score)
|
||||
|
||||
|
||||
def _tokens(text: str) -> set[str]:
|
||||
tokens = {item for item in re.split(r"[^\w\u4e00-\u9fff]+", text) if len(item) >= 2}
|
||||
compact = NOISE_PATTERN.sub("", text)
|
||||
for size in (2, 3):
|
||||
tokens.update(compact[index : index + size] for index in range(max(len(compact) - size + 1, 0)))
|
||||
return tokens
|
||||
|
||||
|
||||
def _jaccard(left: set[str], right: set[str]) -> float:
|
||||
if not left or not right:
|
||||
return 0.0
|
||||
return len(left & right) / len(left | right)
|
||||
|
||||
|
||||
def _containment(left: str, right: str) -> float:
|
||||
if not left or not right:
|
||||
return 0.0
|
||||
short, long = (left, right) if len(left) <= len(right) else (right, left)
|
||||
if len(short) < 4:
|
||||
return 0.0
|
||||
return 0.92 if short in long else 0.0
|
||||
|
||||
|
||||
def _merge_tokens(left: set[str], right: set[str]) -> set[str]:
|
||||
if len(left) > 260:
|
||||
return set(Counter(left).keys())
|
||||
return left | right
|
||||
|
||||
|
||||
def _looks_more_question_like(candidate: str, current: str) -> bool:
|
||||
markers = ("什么", "怎么", "如何", "为什么", "区别", "能不能", "可以")
|
||||
return any(marker in candidate for marker in markers) and not any(marker in current for marker in markers)
|
||||
|
||||
|
||||
def _cluster_dict(rank: int, cluster: QuestionCluster) -> dict:
|
||||
questions = sorted(cluster.questions, key=lambda item: item.created_at, reverse=True)
|
||||
users = {item.user_id for item in questions}
|
||||
sessions = {item.session_id for item in questions}
|
||||
variants = Counter(item.text for item in questions).most_common(6)
|
||||
terms = _top_terms(questions)
|
||||
return {
|
||||
"rank": rank,
|
||||
"title": cluster.title,
|
||||
"normalized": cluster.normalized,
|
||||
"count": len(questions),
|
||||
"userCount": len(users),
|
||||
"sessionCount": len(sessions),
|
||||
"firstSeenAt": min(item.created_at for item in questions),
|
||||
"lastSeenAt": max(item.created_at for item in questions),
|
||||
"topTerms": terms,
|
||||
"variants": [{"text": text, "count": count} for text, count in variants],
|
||||
"samples": [
|
||||
{
|
||||
"messageId": item.message_id,
|
||||
"sessionId": item.session_id,
|
||||
"userId": item.user_id,
|
||||
"userName": item.user_name,
|
||||
"userPhone": item.user_phone,
|
||||
"raw": item.raw,
|
||||
"cleaned": item.text,
|
||||
"createdAt": item.created_at,
|
||||
}
|
||||
for item in questions[:5]
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _top_terms(questions: list[CleanedQuestion]) -> list[str]:
|
||||
counter: Counter[str] = Counter()
|
||||
for question in questions:
|
||||
for token in question.tokens:
|
||||
if len(token) >= 2 and not token.isdigit():
|
||||
counter[token] += 1
|
||||
return [term for term, _count in counter.most_common(8)]
|
||||
Reference in New Issue
Block a user