Record retrieval and Gemma evaluation results

2026-10-03 18:01:18 +03:00
parent b5a48c5a96
commit 1f8014f226
6 changed files with 363 additions and 15 deletions
@@ -0,0 +1,141 @@
+{
+  "created_at_utc": "2026-10-03T14:53:57.594426+00:00",
+  "base_url": "http://127.0.0.1:8000",
+  "dataset": "evals\\knowledge_retrieval_project.jsonl",
+  "indexing": {
+    "semantic_indexed_documents": 4,
+    "documents": 4,
+    "embedding_models": [
+      "granite-embedding:278m"
+    ]
+  },
+  "metrics": {
+    "cases": 9,
+    "hit_at_1": 0.6666666666666666,
+    "hit_at_3": 1.0,
+    "mean_reciprocal_rank": 0.8333333333333334,
+    "evidence_rate": 0.8888888888888888
+  },
+  "results": [
+    {
+      "id": "project-path-confinement",
+      "query": "Does resolving a requested source path guarantee it stays under the selected root?",
+      "expected_document": "fixture_workspace.py",
+      "expected_evidence": "relative_to(root)",
+      "rank": 1,
+      "top_paths": [
+        "fixture_workspace.py",
+        "fixture_main.py"
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid"
+    },
+    {
+      "id": "project-preview-stale-file",
+      "query": "How does applying an approved preview detect that the original file changed after review?",
+      "expected_document": "fixture_workspace.py",
+      "expected_evidence": "expected_hash",
+      "rank": 2,
+      "top_paths": [
+        "fixture_main.py",
+        "fixture_workspace.py"
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid"
+    },
+    {
+      "id": "project-index-stale-source",
+      "query": "Which file hash is compared with the stored content hash before returning a search result?",
+      "expected_document": "fixture_knowledge.py",
+      "expected_evidence": "actual_hash == row[\"content_hash\"]",
+      "rank": 1,
+      "top_paths": [
+        "fixture_knowledge.py",
+        "fixture_main.py",
+        "fixture_workspace.py"
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid"
+    },
+    {
+      "id": "project-chunk-overlap",
+      "query": "What are the configured chunk size and overlap for local knowledge indexing?",
+      "expected_document": "fixture_knowledge.py",
+      "expected_evidence": "CHUNK_OVERLAP = 120",
+      "rank": 1,
+      "top_paths": [
+        "fixture_knowledge.py",
+        "fixture_main.py"
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid"
+    },
+    {
+      "id": "project-arabic-audio-variants",
+      "query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
+      "expected_document": "fixture_knowledge.py",
+      "expected_evidence": "QUERY_SYNONYMS",
+      "rank": 1,
+      "top_paths": [
+        "fixture_knowledge.py",
+        "fixture_main.py"
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid"
+    },
+    {
+      "id": "project-pdf-page-bound",
+      "query": "What is the maximum number of PDF pages accepted during local extraction?",
+      "expected_document": "fixture_pdf.py",
+      "expected_evidence": "MAX_PDF_PAGES = 30",
+      "rank": 1,
+      "top_paths": [
+        "fixture_pdf.py",
+        "fixture_main.py"
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid"
+    },
+    {
+      "id": "project-pdf-empty-scan",
+      "query": "ما الذي يحدث إذا رفع المستخدم ملف PDF ممسوحًا ولا يحتوي نصًا قابلًا للاستخراج؟",
+      "expected_document": "fixture_pdf.py",
+      "expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
+      "rank": 2,
+      "top_paths": [
+        "fixture_main.py",
+        "fixture_pdf.py"
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid"
+    },
+    {
+      "id": "project-skill-tool-filter",
+      "query": "How is a tool call blocked when it is outside the selected skill permissions?",
+      "expected_document": "fixture_main.py",
+      "expected_evidence": "tool_name not in selected_skill.allowed_tools",
+      "rank": 2,
+      "top_paths": [
+        "fixture_workspace.py",
+        "fixture_main.py"
+      ],
+      "evidence_found": false,
+      "search_mode": "hybrid"
+    },
+    {
+      "id": "project-agent-index-limits",
+      "query": "What file count and total size limits apply when indexing local knowledge?",
+      "expected_document": "fixture_knowledge.py",
+      "expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
+      "rank": 1,
+      "top_paths": [
+        "fixture_knowledge.py",
+        "fixture_main.py",
+        "fixture_workspace.py"
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid"
+    }
+  ],
+  "note": "Small deterministic fixture set; measures configured retrieval (keyword or hybrid) and evidence presence, not answer quality or general RAG quality."
+}