fix: complete admin knowledge audit improvements

ai_knowledge_base_v2/apps/backend/app/services/knowledge_agent_service.py

@@ -23,6 +23,7 @@ from app.models.knowledge import (
     KnowledgeVersion,
 )
 from app.services.knowledge_pipeline_service import expand_synonyms, extract_terms
+from app.services.knowledge_catalog_cache_service import KnowledgeCatalogCacheService
 from app.services.knowledge_service import KnowledgeScope
 from app.services.model_service import _call_configured_model, _system_config_bool
 from app.services.rag_service import PromptService, RagResult, RetrievedChunk
@@ -76,16 +77,16 @@ class KnowledgeAgentService:
         db.add(log)
         db.flush()
         scopes = [KnowledgeScope(id=item["knowledgeId"], name=item["name"], feishu_space_id="", feishu_node_id="") for item in catalog]
-        trace: list[dict] = [{"tool": "get_knowledge_catalog", "count": len(catalog)}]
+        trace: list[dict] = [cls._trace("get_knowledge_catalog", 1, {}, {"items": catalog, "count": len(catalog)}, started)]
         need_knowledge, selected_ids, reason = cls._decide(question, catalog)
-        trace.append({"tool": "agent_decision", "needKnowledge": need_knowledge, "selectedKnowledgeIds": selected_ids, "reason": reason})
+        trace.append(cls._trace("agent_decision", 2, {"question": question}, {"needKnowledge": need_knowledge, "selectedKnowledgeIds": selected_ids, "reason": reason}, started))
         chunks: list[RetrievedChunk] = []
         try:
             if need_knowledge and selected_ids:
                 terms = cls._query_terms(question)
                 candidates = cls.search_knowledge(db, terms, selected_ids, catalog)
-                trace.append({"tool": "search_knowledge", "queryTerms": terms, "candidateCount": len(candidates)})
-                await cls._rerank(db, question, candidates, trace)
+                trace.append(cls._trace("search_knowledge", len(trace) + 1, {"queryTerms": terms, "knowledgeIds": selected_ids}, {"candidateCount": len(candidates), "candidates": [cls._candidate_trace(x) for x in candidates]}, started))
+                await cls._rerank(db, question, candidates, trace, started)
                 selected = cls._select_sections(candidates)
                 chunks = [
                     RetrievedChunk(
@@ -100,7 +101,7 @@ class KnowledgeAgentService:
                     )
                     for item in selected
                 ]
-                trace.append({"tool": "read_knowledge_sections", "sectionIds": [item.section.id for item in selected]})
+                trace.append(cls._trace("read_knowledge_sections", len(trace) + 1, {"sectionIds": [item.section.id for item in selected]}, {"count": len(selected), "sections": [{"knowledgeId": x.knowledge.id, "knowledgeName": x.knowledge.name, "sectionId": x.section.id, "title": x.section.title, "content": cls._protect_content(x.section.content), "adopted": True} for x in selected]}, started))
                 cls._persist_candidates(db, log.id, candidates)
                 log.knowledge_called = 1
                 log.selected_knowledge_ids = ",".join(str(item) for item in selected_ids)
@@ -134,6 +135,13 @@ class KnowledgeAgentService:
         version_overrides: dict[int, int] | None = None,
         preview_knowledge_ids: list[int] | None = None,
     ) -> list[dict]:
+        cacheable = version_overrides is None and preview_knowledge_ids is None
+        if cacheable:
+            return KnowledgeCatalogCacheService.get_or_build("open", lambda: KnowledgeAgentService._catalog_from_db(db, None, None))
+        return KnowledgeAgentService._catalog_from_db(db, version_overrides, preview_knowledge_ids)
+
+    @staticmethod
+    def _catalog_from_db(db: Session, version_overrides: dict[int, int] | None, preview_knowledge_ids: list[int] | None) -> list[dict]:
         query = select(Knowledge).where(Knowledge.lifecycle_status == "active")
         if preview_knowledge_ids is None:
             query = query.where(
@@ -171,6 +179,15 @@ class KnowledgeAgentService:
             )
         return catalog
 
+    @staticmethod
+    def _trace(tool: str, order: int, request: dict, response: dict, started: float, *, status: str = "success", error: str | None = None, retries: int = 0) -> dict:
+        now = int((perf_counter() - started) * 1000)
+        return {"tool": tool, "order": order, "request": request, "response": response, **response, "status": status, "startedAtMs": now, "endedAtMs": now, "durationMs": 0, "responseCount": response.get("count", response.get("candidateCount")), "error": error, "retries": retries}
+
+    @staticmethod
+    def _candidate_trace(item: Candidate) -> dict:
+        return {"knowledgeId": item.knowledge.id, "knowledgeName": item.knowledge.name, "versionId": item.version.id, "chunkId": item.chunk.id, "sectionId": item.section.id, "title": item.chunk.title, "lexicalScore": item.lexical_score, "rerankScore": item.rerank_score, "selected": item.selected, "discardReason": item.discard_reason}
+
     @classmethod
     def search_knowledge(cls, db: Session, terms: list[str], selected_ids: list[int], catalog: list[dict]) -> list[Candidate]:
         versions_by_kb = {item["knowledgeId"]: item["versionId"] for item in catalog}
@@ -200,12 +217,12 @@ class KnowledgeAgentService:
         return candidates[:MAX_LEXICAL_CANDIDATES]
 
     @classmethod
-    async def _rerank(cls, db: Session, question: str, candidates: list[Candidate], trace: list[dict]) -> None:
+    async def _rerank(cls, db: Session, question: str, candidates: list[Candidate], trace: list[dict], started: float) -> None:
         if not candidates:
             return
         model = db.scalar(select(ModelConfig).where(ModelConfig.enabled == 1).order_by(ModelConfig.id.desc()).limit(1))
         if model is None or _system_config_bool(db, "mock_model_enabled", get_settings().mock_model_enabled):
-            trace.append({"tool": "model_rerank", "status": "lexical_fallback", "reason": "模型未启用或处于 mock"})
+            trace.append(cls._trace("Rerank", len(trace) + 1, {"candidateCount": len(candidates)}, {"count": len(candidates), "mode": "lexical_fallback", "candidates": [cls._candidate_trace(x) for x in candidates]}, started))
             return
         prompt = (
             "你是知识检索重排器。根据用户问题给候选打0到100相关分，只返回JSON："
@@ -221,9 +238,9 @@ class KnowledgeAgentService:
             for index, item in enumerate(candidates, 1):
                 item.rerank_score = score_map.get(index)
             candidates.sort(key=lambda item: item.rerank_score if item.rerank_score is not None else item.lexical_score, reverse=True)
-            trace.append({"tool": "model_rerank", "status": "success", "candidateCount": len(candidates)})
+            trace.append(cls._trace("Rerank", len(trace) + 1, {"candidateCount": len(candidates)}, {"count": len(candidates), "candidates": [cls._candidate_trace(x) for x in candidates]}, started))
         except Exception as exc:
-            trace.append({"tool": "model_rerank", "status": "lexical_fallback", "reason": str(exc)[:300]})
+            trace.append(cls._trace("Rerank", len(trace) + 1, {"candidateCount": len(candidates)}, {"count": len(candidates), "mode": "lexical_fallback"}, started, status="fallback", error=str(exc)[:300]))
 
     @staticmethod
     def _select_sections(candidates: list[Candidate]) -> list[Candidate]:

ai_knowledge_base_v2/apps/backend/app/services/knowledge_catalog_cache_service.py

@@ -1,12 +1,46 @@
 from __future__ import annotations
 
+import json
+import time
+from typing import Callable
+
 from app.services.redis_client import get_sync_redis_client
 
 
 class KnowledgeCatalogCacheService:
     """Central invalidation point for current and future KnowledgeList caches."""
 
-    VERSION_KEY = "knowledge:catalog:version"
+    VERSION_KEY = "kb:catalog:generation"
+    KEY_PREFIX = "kb:catalog:v1"
+    METRICS_KEY = "kb:catalog:metrics"
+    TTL_SECONDS = 300
+
+    @classmethod
+    def get_or_build(cls, variant: str, builder: Callable[[], list[dict]]) -> list[dict]:
+        """Read-through cache. Redis is optional and never the source of truth."""
+        client = get_sync_redis_client()
+        if client is None:
+            return builder()
+        started = time.perf_counter()
+        try:
+            generation = client.get(cls.VERSION_KEY) or "1"
+            key = f"{cls.KEY_PREFIX}:{generation}:{variant}"
+            cached = client.get(key)
+            if cached:
+                client.hincrby(cls.METRICS_KEY, "hits", 1)
+                client.hincrbyfloat(cls.METRICS_KEY, "hit_ms", (time.perf_counter() - started) * 1000)
+                return json.loads(cached)
+            client.hincrby(cls.METRICS_KEY, "misses", 1)
+            value = builder()
+            # Short lock prevents concurrent cold requests from repeatedly rebuilding.
+            lock = client.set(f"{key}:lock", "1", nx=True, ex=15)
+            if lock:
+                client.setex(key, cls.TTL_SECONDS, json.dumps(value, ensure_ascii=False, default=str))
+                client.delete(f"{key}:lock")
+            client.hincrbyfloat(cls.METRICS_KEY, "miss_ms", (time.perf_counter() - started) * 1000)
+            return value
+        except Exception:
+            return builder()
 
     @classmethod
     def invalidate(cls) -> None:

ai_knowledge_base_v2/apps/backend/app/services/maintenance_service.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import datetime, timedelta
+
+from sqlalchemy import delete, select
+
+from app.core.database import SessionLocal
+from app.models.knowledge import KnowledgeRetrievalCandidate, KnowledgeRetrievalLog
+from app.models.logs import LogRetentionPolicy
+from app.services.redis_client import get_sync_redis_client
+
+
+class MaintenanceService:
+    """Single-runner background maintenance. MySQL remains authoritative."""
+
+    @classmethod
+    async def run_forever(cls) -> None:
+        while True:
+            await asyncio.to_thread(cls.run_due_jobs)
+            await asyncio.sleep(3600)
+
+    @classmethod
+    def run_due_jobs(cls) -> None:
+        redis = get_sync_redis_client()
+        lock_acquired = False
+        try:
+            if redis is not None:
+                lock_acquired = bool(redis.set("maintenance:daily:lock", "1", nx=True, ex=3300))
+                if not lock_acquired:
+                    return
+            with SessionLocal() as db:
+                policy = db.scalar(select(LogRetentionPolicy).order_by(LogRetentionPolicy.id).limit(1))
+                if not policy or not policy.enabled or not policy.retention_days:
+                    return
+                if policy.last_run_at and policy.last_run_at.date() >= datetime.now().date():
+                    return
+                before = datetime.now() - timedelta(days=policy.retention_days)
+                deleted = cls.delete_retrieval_logs(db, before)
+                policy.last_run_at = datetime.now()
+                policy.last_result = json.dumps({"status": "success", "deleted": deleted, "before": before.isoformat()}, ensure_ascii=False)
+                db.add(policy)
+                db.commit()
+        except Exception as exc:
+            with SessionLocal() as db:
+                policy = db.scalar(select(LogRetentionPolicy).order_by(LogRetentionPolicy.id).limit(1))
+                if policy:
+                    policy.last_run_at = datetime.now()
+                    policy.last_result = json.dumps({"status": "failed", "error": str(exc)[:1000]}, ensure_ascii=False)
+                    db.add(policy)
+                    db.commit()
+        finally:
+            if redis is not None and lock_acquired:
+                try:
+                    redis.delete("maintenance:daily:lock")
+                except Exception:
+                    pass
+
+    @staticmethod
+    def delete_retrieval_logs(db, before: datetime, batch_size: int = 500) -> int:
+        total = 0
+        while True:
+            ids = list(db.scalars(select(KnowledgeRetrievalLog.id).where(KnowledgeRetrievalLog.created_at < before).order_by(KnowledgeRetrievalLog.id).limit(batch_size)).all())
+            if not ids:
+                return total
+            db.execute(delete(KnowledgeRetrievalCandidate).where(KnowledgeRetrievalCandidate.retrieval_log_id.in_(ids)))
+            db.execute(delete(KnowledgeRetrievalLog).where(KnowledgeRetrievalLog.id.in_(ids)))
+            total += len(ids)
+            db.commit()