Validate local OCR retrieval for scanned PDFs

2026-10-03 19:29:39 +03:00
parent 332f43aaf0
commit 0587347157
5 changed files with 326 additions and 3 deletions
@@ -0,0 +1,49 @@
+{
+  "created_at_utc": "2026-10-03T16:18:22.317786+00:00",
+  "base_url": "http://127.0.0.1:8130",
+  "dataset": "evals\\knowledge_retrieval_extended.jsonl",
+  "indexing": {
+    "semantic_indexed_documents": 1,
+    "documents": 1,
+    "embedding_models": [
+      "granite-embedding:278m"
+    ]
+  },
+  "metrics": {
+    "cases": 1,
+    "hit_at_1": 1.0,
+    "hit_at_3": 1.0,
+    "mean_reciprocal_rank": 1.0,
+    "evidence_rate": 1.0
+  },
+  "results": [
+    {
+      "id": "scanned-pdf-ocr-deadline-en",
+      "query": "What calendar date ends the archive restoration approval window?",
+      "expected_document": "scanned_approval_notice.pdf",
+      "expected_evidence": "17 October 2026",
+      "rank": 1,
+      "top_paths": [
+        "scanned_approval_notice.pdf"
+      ],
+      "expected_document_chunks": [
+        0
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid",
+      "agent_answer": "تنتهي فترة موافقة استعادة الأرشيف في 17 أكتوبر 2026. (scanned_approval_notice.pdf · المقطع 0)",
+      "agent_tool": "search_knowledge",
+      "agent_files": [
+        "scanned_approval_notice.pdf"
+      ],
+      "agent_steps": [
+        {
+          "tool": "search_knowledge",
+          "status": "completed"
+        }
+      ],
+      "human_rating": null
+    }
+  ],
+  "note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
+}