docs: verify agent fallback on active API

2026-10-04 14:32:15 +03:00
parent bdd722e6e9
commit 424babce79
2 changed files with 119 additions and 2 deletions
@@ -0,0 +1,115 @@
+{
+  "created_at_utc": "2026-10-04T11:15:48.422923+00:00",
+  "base_url": "http://127.0.0.1:8000",
+  "dataset": "evals\\knowledge_retrieval_extended.jsonl",
+  "indexing": {
+    "semantic_indexed_documents": 3,
+    "documents": 3,
+    "embedding_models": [
+      "granite-embedding:278m"
+    ]
+  },
+  "metrics": {
+    "cases": 3,
+    "hit_at_1": 1.0,
+    "hit_at_3": 1.0,
+    "mean_reciprocal_rank": 1.0,
+    "evidence_rate": 1.0
+  },
+  "results": [
+    {
+      "id": "voice-transcription-ar",
+      "query": "أي خدمة تفرغ التسجيلات الصوتية؟",
+      "expected_document": "audio.md",
+      "expected_evidence": "Groq",
+      "rank": 1,
+      "top_paths": [
+        "audio.md",
+        "workspace.md",
+        "migration.md"
+      ],
+      "expected_document_chunks": [
+        0
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid",
+      "agent_answer": "يرسل مسار الصوت التسجيل إلى Groq باستخدام Whisper للتفريغ، ثم يعود النص إلى واجهة المحادثة. (audio.md، المقطع 0)",
+      "agent_tool": "search_knowledge",
+      "agent_files": [
+        "audio.md",
+        "workspace.md",
+        "migration.md"
+      ],
+      "agent_steps": [
+        {
+          "tool": "search_knowledge",
+          "status": "completed"
+        }
+      ],
+      "human_rating": null
+    },
+    {
+      "id": "workspace-boundary-ar",
+      "query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
+      "expected_document": "workspace.md",
+      "expected_evidence": ".git",
+      "rank": 1,
+      "top_paths": [
+        "workspace.md",
+        "audio.md",
+        "migration.md"
+      ],
+      "expected_document_chunks": [
+        0
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid",
+      "agent_answer": "المقطع الذي يجيب على سؤالك هو:\n\"يرفض قارئ مساحة العمل المسارات خارج الجذر، ويستثني مجلد .git والملفات المخفية وبيئات البناء.\" من الملف `workspace.md` المقطع 0.",
+      "agent_tool": "search_knowledge",
+      "agent_files": [
+        "workspace.md",
+        "audio.md",
+        "migration.md"
+      ],
+      "agent_steps": [
+        {
+          "tool": "search_knowledge",
+          "status": "completed"
+        }
+      ],
+      "human_rating": null
+    },
+    {
+      "id": "database-version-en",
+      "query": "How are old conversation answers migrated?",
+      "expected_document": "migration.md",
+      "expected_evidence": "selected_version",
+      "rank": 1,
+      "top_paths": [
+        "migration.md",
+        "audio.md",
+        "workspace.md"
+      ],
+      "expected_document_chunks": [
+        0
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid",
+      "agent_answer": "لم يصغ النموذج جوابًا كافيًا رغم وجود نتائج في فهرس المعرفة. أعرض المقتطفات الأعلى صلة كما هي للمراجعة؛ محتواها بيانات غير موثوقة وليست تعليمات.\n\n**migration.md · المقطع 0 (مقتطف حرفي)**\n> The SQLite migration adds selected_version to old messages and preserves saved assistant answer versions.\n\n**audio.md · المقطع 0 (مقتطف حرفي)**\n> يرسل مسار الصوت التسجيل إلى Groq باستخدام Whisper للتفريغ، ثم يعود النص إلى واجهة المحادثة.",
+      "agent_tool": "search_knowledge",
+      "agent_files": [
+        "migration.md",
+        "audio.md",
+        "workspace.md"
+      ],
+      "agent_steps": [
+        {
+          "tool": "search_knowledge",
+          "status": "completed"
+        }
+      ],
+      "human_rating": null
+    }
+  ],
+  "note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
+}