Validate local OCR retrieval for scanned PDFs
This commit is contained in:
Hamza Ayed
Binary file not shown.
@@ -11,3 +11,4 @@
|
||||
{"id": "skill-permissions-en", "query": "How does the server restrict tools for a selected skill?", "document": "skills.md", "expected_evidence": "allowed_tools", "text": "Each local agent skill declares an allowed_tools set. The server filters the advertised tools and rejects any model call outside that set."}
|
||||
{"id": "long-guide-deep-approval-en", "query": "What must happen before a proposed production change can be saved?", "document": "long_technical_guide.md", "expected_evidence": "two-stage approval", "source_path": "evals/fixtures/long_technical_guide.md"}
|
||||
{"id": "pdf-archive-page-limit-en", "query": "What is the maximum number of pages accepted when indexing one PDF?", "document": "archive_retention_policy.pdf", "expected_evidence": "30 pages", "source_path": "evals/fixtures/archive_retention_policy.pdf"}
|
||||
{"id": "scanned-pdf-ocr-deadline-en", "query": "What calendar date ends the archive restoration approval window?", "document": "scanned_approval_notice.pdf", "expected_evidence": "17 October 2026", "source_path": "evals/fixtures/scanned_approval_notice.pdf"}
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-03T16:27:51.027447+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"dataset": "evals\\knowledge_retrieval_extended.jsonl",
|
||||
"indexing": {
|
||||
"semantic_indexed_documents": 14,
|
||||
"documents": 14,
|
||||
"embedding_models": [
|
||||
"granite-embedding:278m"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"cases": 14,
|
||||
"hit_at_1": 1.0,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 1.0,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "conversation-storage-ar",
|
||||
"query": "أين تحفظ المحادثات؟",
|
||||
"expected_document": "storage.md",
|
||||
"expected_evidence": "SQLite",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"storage.md",
|
||||
"long_technical_guide.md",
|
||||
"workspace.md",
|
||||
"scanned_approval_notice.pdf"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "image-model-ar",
|
||||
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
|
||||
"expected_document": "images.md",
|
||||
"expected_evidence": "Ministral 3:3b",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"images.md",
|
||||
"routing.md",
|
||||
"timeout.md",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "preview-safety-ar",
|
||||
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
|
||||
"expected_document": "preview.md",
|
||||
"expected_evidence": "SHA256",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"preview.md",
|
||||
"long_technical_guide.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"workspace.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "pdf-extraction-ar",
|
||||
"query": "كيف نستخرج النص من صفحات PDF؟",
|
||||
"expected_document": "pdf.md",
|
||||
"expected_evidence": "pypdf",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"pdf.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"images.md",
|
||||
"audio.md",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "voice-transcription-ar",
|
||||
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
|
||||
"expected_document": "audio.md",
|
||||
"expected_evidence": "Groq",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"audio.md",
|
||||
"scanned_approval_notice.pdf",
|
||||
"archive_retention_policy.pdf",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "workspace-boundary-ar",
|
||||
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
|
||||
"expected_document": "workspace.md",
|
||||
"expected_evidence": ".git",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"workspace.md",
|
||||
"preview.md",
|
||||
"long_technical_guide.md",
|
||||
"archive_retention_policy.pdf"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "web-search-en",
|
||||
"query": "Which provider searches multiple web sources?",
|
||||
"expected_document": "search.md",
|
||||
"expected_evidence": "DuckDuckGo",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"search.md",
|
||||
"long_technical_guide.md",
|
||||
"skills.md",
|
||||
"routing.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "database-version-en",
|
||||
"query": "How are old conversation answers migrated?",
|
||||
"expected_document": "migration.md",
|
||||
"expected_evidence": "selected_version",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"migration.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "model-routing-ar",
|
||||
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
|
||||
"expected_document": "routing.md",
|
||||
"expected_evidence": "auto_routed",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"routing.md",
|
||||
"images.md",
|
||||
"storage.md",
|
||||
"timeout.md",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "api-timeout-ar",
|
||||
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
|
||||
"expected_document": "timeout.md",
|
||||
"expected_evidence": "504",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"timeout.md",
|
||||
"routing.md",
|
||||
"long_technical_guide.md",
|
||||
"skills.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "skill-permissions-en",
|
||||
"query": "How does the server restrict tools for a selected skill?",
|
||||
"expected_document": "skills.md",
|
||||
"expected_evidence": "allowed_tools",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"skills.md",
|
||||
"long_technical_guide.md",
|
||||
"migration.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "long-guide-deep-approval-en",
|
||||
"query": "What must happen before a proposed production change can be saved?",
|
||||
"expected_document": "long_technical_guide.md",
|
||||
"expected_evidence": "two-stage approval",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"long_technical_guide.md",
|
||||
"scanned_approval_notice.pdf",
|
||||
"migration.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
17,
|
||||
4,
|
||||
8
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "pdf-archive-page-limit-en",
|
||||
"query": "What is the maximum number of pages accepted when indexing one PDF?",
|
||||
"expected_document": "archive_retention_policy.pdf",
|
||||
"expected_evidence": "30 pages",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"archive_retention_policy.pdf",
|
||||
"pdf.md",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "scanned-pdf-ocr-deadline-en",
|
||||
"query": "What calendar date ends the archive restoration approval window?",
|
||||
"expected_document": "scanned_approval_notice.pdf",
|
||||
"expected_evidence": "17 October 2026",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"scanned_approval_notice.pdf",
|
||||
"archive_retention_policy.pdf",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-03T16:18:22.317786+00:00",
|
||||
"base_url": "http://127.0.0.1:8130",
|
||||
"dataset": "evals\\knowledge_retrieval_extended.jsonl",
|
||||
"indexing": {
|
||||
"semantic_indexed_documents": 1,
|
||||
"documents": 1,
|
||||
"embedding_models": [
|
||||
"granite-embedding:278m"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"cases": 1,
|
||||
"hit_at_1": 1.0,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 1.0,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "scanned-pdf-ocr-deadline-en",
|
||||
"query": "What calendar date ends the archive restoration approval window?",
|
||||
"expected_document": "scanned_approval_notice.pdf",
|
||||
"expected_evidence": "17 October 2026",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"scanned_approval_notice.pdf"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid",
|
||||
"agent_answer": "تنتهي فترة موافقة استعادة الأرشيف في 17 أكتوبر 2026. (scanned_approval_notice.pdf · المقطع 0)",
|
||||
"agent_tool": "search_knowledge",
|
||||
"agent_files": [
|
||||
"scanned_approval_notice.pdf"
|
||||
],
|
||||
"agent_steps": [
|
||||
{
|
||||
"tool": "search_knowledge",
|
||||
"status": "completed"
|
||||
}
|
||||
],
|
||||
"human_rating": null
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
|
||||
}
|
||||
Reference in New Issue
Block a user