Improve retrieval of deep workspace evidence
This commit is contained in:
Hamza Ayed
@@ -0,0 +1,142 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-03T15:45:47.531644+00:00",
|
||||
"base_url": "http://127.0.0.1:8129",
|
||||
"dataset": "evals\\knowledge_retrieval_project.jsonl",
|
||||
"indexing": {
|
||||
"semantic_indexed_documents": 4,
|
||||
"documents": 4,
|
||||
"embedding_models": [
|
||||
"granite-embedding:278m"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"cases": 9,
|
||||
"hit_at_1": 0.8888888888888888,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 0.9444444444444444,
|
||||
"evidence_rate": 0.8888888888888888
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "project-path-confinement",
|
||||
"query": "Does resolving a requested source path guarantee it stays under the selected root?",
|
||||
"expected_document": "fixture_workspace.py",
|
||||
"expected_evidence": "relative_to(root)",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-preview-stale-file",
|
||||
"query": "How does applying an approved preview detect that the original file changed after review?",
|
||||
"expected_document": "fixture_workspace.py",
|
||||
"expected_evidence": "expected_hash",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-index-stale-source",
|
||||
"query": "Which file hash is compared with the stored content hash before returning a search result?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "actual_hash == row[\"content_hash\"]",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-chunk-overlap",
|
||||
"query": "What are the configured chunk size and overlap for local knowledge indexing?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "CHUNK_OVERLAP = 120",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-arabic-audio-variants",
|
||||
"query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "QUERY_SYNONYMS",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-pdf-page-bound",
|
||||
"query": "What is the maximum number of PDF pages accepted during local extraction?",
|
||||
"expected_document": "fixture_pdf.py",
|
||||
"expected_evidence": "MAX_PDF_PAGES = 30",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_pdf.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-pdf-empty-scan",
|
||||
"query": "ما الذي يحدث إذا رفع المستخدم ملف PDF ممسوحًا ولا يحتوي نصًا قابلًا للاستخراج؟",
|
||||
"expected_document": "fixture_pdf.py",
|
||||
"expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_pdf.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-skill-tool-filter",
|
||||
"query": "How is a tool call blocked when it is outside the selected skill permissions?",
|
||||
"expected_document": "fixture_main.py",
|
||||
"expected_evidence": "tool_name not in selected_skill.allowed_tools",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_main.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": false,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-agent-index-limits",
|
||||
"query": "What file count and total size limits apply when indexing local knowledge?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set; measures configured retrieval (keyword or hybrid) and evidence presence, not answer quality or general RAG quality."
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-03T15:54:47.273463+00:00",
|
||||
"base_url": "http://127.0.0.1:8129",
|
||||
"dataset": "evals\\knowledge_retrieval.jsonl",
|
||||
"indexing": {
|
||||
"semantic_indexed_documents": 11,
|
||||
"documents": 11,
|
||||
"embedding_models": [
|
||||
"granite-embedding:278m"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"cases": 11,
|
||||
"hit_at_1": 1.0,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 1.0,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "conversation-storage-ar",
|
||||
"query": "أين تحفظ المحادثات؟",
|
||||
"expected_document": "storage.md",
|
||||
"expected_evidence": "SQLite",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"storage.md",
|
||||
"workspace.md",
|
||||
"preview.md",
|
||||
"images.md",
|
||||
"audio.md",
|
||||
"routing.md",
|
||||
"skills.md",
|
||||
"timeout.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "image-model-ar",
|
||||
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
|
||||
"expected_document": "images.md",
|
||||
"expected_evidence": "Ministral 3:3b",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"images.md",
|
||||
"timeout.md",
|
||||
"routing.md",
|
||||
"workspace.md",
|
||||
"pdf.md",
|
||||
"preview.md",
|
||||
"skills.md",
|
||||
"search.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "preview-safety-ar",
|
||||
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
|
||||
"expected_document": "preview.md",
|
||||
"expected_evidence": "SHA256",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"preview.md",
|
||||
"workspace.md",
|
||||
"pdf.md",
|
||||
"storage.md",
|
||||
"routing.md",
|
||||
"timeout.md",
|
||||
"images.md",
|
||||
"skills.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "pdf-extraction-ar",
|
||||
"query": "كيف نستخرج النص من صفحات PDF؟",
|
||||
"expected_document": "pdf.md",
|
||||
"expected_evidence": "pypdf",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"pdf.md",
|
||||
"images.md",
|
||||
"audio.md",
|
||||
"workspace.md",
|
||||
"search.md",
|
||||
"preview.md",
|
||||
"routing.md",
|
||||
"timeout.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "voice-transcription-ar",
|
||||
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
|
||||
"expected_document": "audio.md",
|
||||
"expected_evidence": "Groq",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"audio.md",
|
||||
"workspace.md",
|
||||
"skills.md",
|
||||
"pdf.md",
|
||||
"preview.md",
|
||||
"images.md",
|
||||
"search.md",
|
||||
"storage.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "workspace-boundary-ar",
|
||||
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
|
||||
"expected_document": "workspace.md",
|
||||
"expected_evidence": ".git",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"workspace.md",
|
||||
"preview.md",
|
||||
"pdf.md",
|
||||
"images.md",
|
||||
"skills.md",
|
||||
"storage.md",
|
||||
"audio.md",
|
||||
"timeout.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "web-search-en",
|
||||
"query": "Which provider searches multiple web sources?",
|
||||
"expected_document": "search.md",
|
||||
"expected_evidence": "DuckDuckGo",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"search.md",
|
||||
"skills.md",
|
||||
"routing.md",
|
||||
"timeout.md",
|
||||
"workspace.md",
|
||||
"images.md",
|
||||
"pdf.md",
|
||||
"audio.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "database-version-en",
|
||||
"query": "How are old conversation answers migrated?",
|
||||
"expected_document": "migration.md",
|
||||
"expected_evidence": "selected_version",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"migration.md",
|
||||
"storage.md",
|
||||
"images.md",
|
||||
"audio.md",
|
||||
"preview.md",
|
||||
"workspace.md",
|
||||
"timeout.md",
|
||||
"skills.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "model-routing-ar",
|
||||
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
|
||||
"expected_document": "routing.md",
|
||||
"expected_evidence": "auto_routed",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"routing.md",
|
||||
"images.md",
|
||||
"storage.md",
|
||||
"timeout.md",
|
||||
"skills.md",
|
||||
"preview.md",
|
||||
"search.md",
|
||||
"migration.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "api-timeout-ar",
|
||||
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
|
||||
"expected_document": "timeout.md",
|
||||
"expected_evidence": "504",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"timeout.md",
|
||||
"routing.md",
|
||||
"skills.md",
|
||||
"images.md",
|
||||
"storage.md",
|
||||
"preview.md",
|
||||
"audio.md",
|
||||
"search.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "skill-permissions-en",
|
||||
"query": "How does the server restrict tools for a selected skill?",
|
||||
"expected_document": "skills.md",
|
||||
"expected_evidence": "allowed_tools",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"skills.md",
|
||||
"migration.md",
|
||||
"workspace.md",
|
||||
"preview.md",
|
||||
"storage.md",
|
||||
"routing.md",
|
||||
"timeout.md",
|
||||
"images.md"
|
||||
],
|
||||
"expected_document_chunks": [
|
||||
0
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set; measures configured retrieval (keyword or hybrid) and evidence presence, not answer quality or general RAG quality."
|
||||
}
|
||||
Reference in New Issue
Block a user