Expand retrieval evaluation to long docs and PDF
This commit is contained in:
Hamza Ayed
@@ -0,0 +1,266 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-03T16:06:06.517530+00:00",
|
||||
"base_url": "http://127.0.0.1:8129",
|
||||
"dataset": "evals\\knowledge_retrieval_extended.jsonl",
|
||||
"indexing": {
|
||||
"semantic_indexed_documents": 13,
|
||||
"documents": 13,
|
||||
"embedding_models": [
|
||||
"granite-embedding:278m"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"cases": 13,
|
||||
"hit_at_1": 1.0,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 1.0,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "conversation-storage-ar",
|
||||
"query": "أين تحفظ المحادثات؟",
|
||||
"expected_document": "storage.md",
|
||||
"expected_evidence": "SQLite",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"storage.md",
|
||||
"long_technical_guide.md",
|
||||
"workspace.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"preview.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "image-model-ar",
|
||||
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
|
||||
"expected_document": "images.md",
|
||||
"expected_evidence": "Ministral 3:3b",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"images.md",
|
||||
"routing.md",
|
||||
"timeout.md",
|
||||
"long_technical_guide.md",
|
||||
"archive_retention_policy.pdf"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "preview-safety-ar",
|
||||
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
|
||||
"expected_document": "preview.md",
|
||||
"expected_evidence": "SHA256",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"preview.md",
|
||||
"long_technical_guide.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"workspace.md",
|
||||
"pdf.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "pdf-extraction-ar",
|
||||
"query": "كيف نستخرج النص من صفحات PDF؟",
|
||||
"expected_document": "pdf.md",
|
||||
"expected_evidence": "pypdf",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"pdf.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"images.md",
|
||||
"audio.md",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "voice-transcription-ar",
|
||||
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
|
||||
"expected_document": "audio.md",
|
||||
"expected_evidence": "Groq",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"audio.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"long_technical_guide.md",
|
||||
"workspace.md",
|
||||
"skills.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "workspace-boundary-ar",
|
||||
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
|
||||
"expected_document": "workspace.md",
|
||||
"expected_evidence": ".git",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"workspace.md",
|
||||
"preview.md",
|
||||
"long_technical_guide.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"pdf.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "web-search-en",
|
||||
"query": "Which provider searches multiple web sources?",
|
||||
"expected_document": "search.md",
|
||||
"expected_evidence": "DuckDuckGo",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"search.md",
|
||||
"long_technical_guide.md",
|
||||
"skills.md",
|
||||
"routing.md",
|
||||
"archive_retention_policy.pdf"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "database-version-en",
|
||||
"query": "How are old conversation answers migrated?",
|
||||
"expected_document": "migration.md",
|
||||
"expected_evidence": "selected_version",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"migration.md",
|
||||
"archive_retention_policy.pdf",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "model-routing-ar",
|
||||
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
|
||||
"expected_document": "routing.md",
|
||||
"expected_evidence": "auto_routed",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"routing.md",
|
||||
"images.md",
|
||||
"storage.md",
|
||||
"timeout.md",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "api-timeout-ar",
|
||||
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
|
||||
"expected_document": "timeout.md",
|
||||
"expected_evidence": "504",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"timeout.md",
|
||||
"routing.md",
|
||||
"long_technical_guide.md",
|
||||
"skills.md",
|
||||
"images.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "skill-permissions-en",
|
||||
"query": "How does the server restrict tools for a selected skill?",
|
||||
"expected_document": "skills.md",
|
||||
"expected_evidence": "allowed_tools",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"skills.md",
|
||||
"long_technical_guide.md",
|
||||
"migration.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "long-guide-deep-approval-en",
|
||||
"query": "What must happen before a proposed production change can be saved?",
|
||||
"expected_document": "long_technical_guide.md",
|
||||
"expected_evidence": "two-stage approval",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"long_technical_guide.md",
|
||||
"migration.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
17,
|
||||
4,
|
||||
8,
|
||||
11,
|
||||
13,
|
||||
16,
|
||||
3
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "pdf-archive-page-limit-en",
|
||||
"query": "What is the maximum number of pages accepted when indexing one PDF?",
|
||||
"expected_document": "archive_retention_policy.pdf",
|
||||
"expected_evidence": "30 pages",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"archive_retention_policy.pdf",
|
||||
"pdf.md",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-03T16:04:32.525939+00:00",
|
||||
"base_url": "http://127.0.0.1:8129",
|
||||
"dataset": "evals\\knowledge_retrieval_extended.jsonl",
|
||||
"indexing": {
|
||||
"semantic_indexed_documents": 2,
|
||||
"documents": 2,
|
||||
"embedding_models": [
|
||||
"granite-embedding:278m"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"cases": 2,
|
||||
"hit_at_1": 1.0,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 1.0,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "long-guide-deep-approval-en",
|
||||
"query": "What must happen before a proposed production change can be saved?",
|
||||
"expected_document": "long_technical_guide.md",
|
||||
"expected_evidence": "two-stage approval",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"long_technical_guide.md",
|
||||
"archive_retention_policy.pdf"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
17,
|
||||
4,
|
||||
14,
|
||||
9,
|
||||
11,
|
||||
2,
|
||||
16
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid",
|
||||
"agent_answer": "للحفظ أي تغيير مقترح في مساحة العمل، يجب أن تحدث مراجعة من مرحلتين: أولاً، يقوم المستخدم بمراجعة الفرق المقترح بالكامل، ثم يتحقق الخادم من تجزئة الملف الأصلية ويتلقى تأكيدًا صريحًا قبل الكتابة. لا يمكن لأي إجابة من النموذج أو استدعاء الأداة وحده أن يحقق مرحلة المراجعة.\n\nالمصدر: long_technical_guide.md · المقطع 17",
|
||||
"agent_tool": "search_knowledge",
|
||||
"agent_files": [
|
||||
"long_technical_guide.md",
|
||||
"archive_retention_policy.pdf"
|
||||
],
|
||||
"agent_steps": [
|
||||
{
|
||||
"tool": "search_knowledge",
|
||||
"status": "completed"
|
||||
}
|
||||
],
|
||||
"human_rating": null
|
||||
},
|
||||
{
|
||||
"id": "pdf-archive-page-limit-en",
|
||||
"query": "What is the maximum number of pages accepted when indexing one PDF?",
|
||||
"expected_document": "archive_retention_policy.pdf",
|
||||
"expected_evidence": "30 pages",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"archive_retention_policy.pdf",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid",
|
||||
"agent_answer": "الحد الأقصى لعدد الصفحات المقبولة عند فهرسة ملف PDF واحد هو 30 صفحة.\nالمصدر: archive_retention_policy.pdf · المقطع 0",
|
||||
"agent_tool": "search_knowledge",
|
||||
"agent_files": [
|
||||
"archive_retention_policy.pdf",
|
||||
"long_technical_guide.md"
|
||||
],
|
||||
"agent_steps": [
|
||||
{
|
||||
"tool": "search_knowledge",
|
||||
"status": "completed"
|
||||
}
|
||||
],
|
||||
"human_rating": null
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
|
||||
}
|
||||
Reference in New Issue
Block a user