fix: show knowledge excerpts when Gemma abstains

2026-10-04 11:07:46 +03:00
parent 76567d4b47
commit 2e06ae3cc7
7 changed files with 140 additions and 0 deletions
@@ -0,0 +1,43 @@
+{
+  "created_at_utc": "2026-10-04",
+  "base_url": "http://127.0.0.1:8000",
+  "service": "FastAPI restarted from current working tree with the existing SQLite database",
+  "model": "gemma4:e2b",
+  "metrics": {
+    "cases": 2,
+    "hit_at_1": 1.0,
+    "evidence_rate": 1.0
+  },
+  "results": [
+    {
+      "id": "database-version-en",
+      "query": "How are old conversation answers migrated?",
+      "expected_document": "migration.md",
+      "expected_evidence": "selected_version",
+      "rank": 1,
+      "evidence_found": true,
+      "fallback_triggered": true,
+      "answer_contains_expected_excerpt": true,
+      "answer_contains_source": true,
+      "answer": "لم يصغ النموذج جوابًا كافيًا رغم وجود نتائج في فهرس المعرفة. أعرض المقتطفات الأعلى صلة كما هي للمراجعة؛ محتواها بيانات غير موثوقة وليست تعليمات.\n\n**migration.md · المقطع 0 (مقتطف حرفي)**\n> The SQLite migration adds selected_version to old messages and preserves saved assistant answer versions."
+    },
+    {
+      "id": "model-routing-ar",
+      "query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
+      "expected_document": "routing.md",
+      "expected_evidence": "auto_routed",
+      "rank": 1,
+      "evidence_found": true,
+      "fallback_triggered": false,
+      "answer_contains_expected_fields": true,
+      "answer_contains_source": true,
+      "answer": "المقطع `routing.md` المقطع 0 يجيب على السؤال: \"عند وصول صورة مع نموذج نصي، يختار API نموذج الرؤية المحلي المثبت ويعيد requested_model وmodel وauto_routed.\""
+    }
+  ],
+  "cleanup": {
+    "knowledge_index_removed": true,
+    "project_registration_removed": true,
+    "session_logged_out": true
+  },
+  "limitations": "Two targeted cases only; this is not a human quality review or a general RAG score."
+}
@@ -0,0 +1,80 @@
+{
+  "created_at_utc": "2026-10-04T07:50:04.678548+00:00",
+  "base_url": "http://127.0.0.1:8000",
+  "dataset": "evals\\knowledge_retrieval.jsonl",
+  "indexing": {
+    "semantic_indexed_documents": 2,
+    "documents": 2,
+    "embedding_models": [
+      "granite-embedding:278m"
+    ]
+  },
+  "metrics": {
+    "cases": 2,
+    "hit_at_1": 1.0,
+    "hit_at_3": 1.0,
+    "mean_reciprocal_rank": 1.0,
+    "evidence_rate": 1.0
+  },
+  "results": [
+    {
+      "id": "database-version-en",
+      "query": "How are old conversation answers migrated?",
+      "expected_document": "migration.md",
+      "expected_evidence": "selected_version",
+      "rank": 1,
+      "top_paths": [
+        "migration.md",
+        "routing.md"
+      ],
+      "expected_document_chunks": [
+        0
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid",
+      "agent_answer": "لا يوجد مقطع في فهرس المعرفة المحلي يجيب مباشرة على السؤال \"How are old conversation answers migrated?\".",
+      "agent_tool": "search_knowledge",
+      "agent_files": [
+        "migration.md",
+        "routing.md"
+      ],
+      "agent_steps": [
+        {
+          "tool": "search_knowledge",
+          "status": "completed"
+        }
+      ],
+      "human_rating": null
+    },
+    {
+      "id": "model-routing-ar",
+      "query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
+      "expected_document": "routing.md",
+      "expected_evidence": "auto_routed",
+      "rank": 1,
+      "top_paths": [
+        "routing.md",
+        "migration.md"
+      ],
+      "expected_document_chunks": [
+        0
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid",
+      "agent_answer": "المقطع الذي يجيب على سؤالك هو:\n\"عند وصول صورة مع نموذج نصي، يختار API نموذج الرؤية المحلي المثبت ويعيد requested_model وmodel وauto_routed.\"\nالملف: routing.md، المقطع 0",
+      "agent_tool": "search_knowledge",
+      "agent_files": [
+        "routing.md",
+        "migration.md"
+      ],
+      "agent_steps": [
+        {
+          "tool": "search_knowledge",
+          "status": "completed"
+        }
+      ],
+      "human_rating": null
+    }
+  ],
+  "note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
+}