408 lines
12 KiB
JSON
408 lines
12 KiB
JSON
{
|
|
"created_at_utc": "2026-10-03T19:08:14.238708+00:00",
|
|
"base_url": "http://127.0.0.1:8000",
|
|
"dataset": "evals\\knowledge_retrieval.jsonl",
|
|
"indexing": {
|
|
"semantic_indexed_documents": 11,
|
|
"documents": 11,
|
|
"embedding_models": [
|
|
"granite-embedding:278m"
|
|
]
|
|
},
|
|
"metrics": {
|
|
"cases": 11,
|
|
"hit_at_1": 1.0,
|
|
"hit_at_3": 1.0,
|
|
"mean_reciprocal_rank": 1.0,
|
|
"evidence_rate": 1.0
|
|
},
|
|
"results": [
|
|
{
|
|
"id": "conversation-storage-ar",
|
|
"query": "أين تحفظ المحادثات؟",
|
|
"expected_document": "storage.md",
|
|
"expected_evidence": "SQLite",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"storage.md",
|
|
"workspace.md",
|
|
"preview.md",
|
|
"images.md",
|
|
"audio.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "المحادثات تحفظ في قاعدة SQLite المحلية كي تبقى متاحة بعد إغلاق التطبيق أو تحديث الصفحة. (storage.md، المقطع 0)",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"storage.md",
|
|
"routing.md",
|
|
"workspace.md",
|
|
"images.md",
|
|
"preview.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "image-model-ar",
|
|
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
|
|
"expected_document": "images.md",
|
|
"expected_evidence": "Ministral 3:3b",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"images.md",
|
|
"routing.md",
|
|
"timeout.md",
|
|
"workspace.md",
|
|
"pdf.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "المصدر: images.md، المقطع 0\nالصور المرفقة تحول تلقائيًا إلى نموذج Ministral 3:3b المحلي لتحليل الصورة واستخراج النص منها.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"images.md",
|
|
"timeout.md",
|
|
"routing.md",
|
|
"pdf.md",
|
|
"storage.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "preview-safety-ar",
|
|
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
|
|
"expected_document": "preview.md",
|
|
"expected_evidence": "SHA256",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"preview.md",
|
|
"workspace.md",
|
|
"pdf.md",
|
|
"storage.md",
|
|
"routing.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "المصدر: preview.md، المقطع 0\nنُربط معاينة الملف ببصمة SHA256، ويجب أن يوافق المستخدم قبل تطبيق التغيير على مساحة العمل.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"preview.md",
|
|
"storage.md",
|
|
"pdf.md",
|
|
"workspace.md",
|
|
"routing.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "pdf-extraction-ar",
|
|
"query": "كيف نستخرج النص من صفحات PDF؟",
|
|
"expected_document": "pdf.md",
|
|
"expected_evidence": "pypdf",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"pdf.md",
|
|
"images.md",
|
|
"audio.md",
|
|
"workspace.md",
|
|
"search.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "المصدر: pdf.md، المقطع 0\nيستخرج pypdf النص الرقمي من صفحات PDF محليًا، ويحافظ على رقم الصفحة ضمن المقتطف المرسل للنموذج.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"pdf.md",
|
|
"images.md",
|
|
"audio.md",
|
|
"search.md",
|
|
"routing.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "voice-transcription-ar",
|
|
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
|
|
"expected_document": "audio.md",
|
|
"expected_evidence": "Groq",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"audio.md",
|
|
"workspace.md",
|
|
"skills.md",
|
|
"pdf.md",
|
|
"preview.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "المصدر: audio.md، المقطع 0\nيرسل مسار الصوت التسجيل إلى Groq باستخدام Whisper للتفريغ، ثم يعود النص إلى واجهة المحادثة.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"audio.md",
|
|
"pdf.md",
|
|
"workspace.md",
|
|
"storage.md",
|
|
"images.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "workspace-boundary-ar",
|
|
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
|
|
"expected_document": "workspace.md",
|
|
"expected_evidence": ".git",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"workspace.md",
|
|
"preview.md",
|
|
"pdf.md",
|
|
"images.md",
|
|
"skills.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "يستطيع قارئ مساحة العمل رفض المسارات خارج الجذر، ويستثني مجلد `.git` والملفات المخفية وبيئات البناء.\\nالملف: workspace.md، المقطع: 0",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"workspace.md",
|
|
"preview.md",
|
|
"pdf.md",
|
|
"images.md",
|
|
"storage.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "web-search-en",
|
|
"query": "Which provider searches multiple web sources?",
|
|
"expected_document": "search.md",
|
|
"expected_evidence": "DuckDuckGo",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"search.md",
|
|
"skills.md",
|
|
"routing.md",
|
|
"timeout.md",
|
|
"workspace.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "الملف: search.md، المقطع: 0\nThe experimental multi-source web search adapter queries DuckDuckGo and returns source URLs with the summary.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"search.md",
|
|
"routing.md",
|
|
"timeout.md",
|
|
"images.md",
|
|
"pdf.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "database-version-en",
|
|
"query": "How are old conversation answers migrated?",
|
|
"expected_document": "migration.md",
|
|
"expected_evidence": "selected_version",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"migration.md",
|
|
"storage.md",
|
|
"images.md",
|
|
"audio.md",
|
|
"preview.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "المعلومات حول كيفية ترحيل إجابات المحادثات القديمة غير متوفرة في المقاطع المسترجعة.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"migration.md",
|
|
"storage.md",
|
|
"images.md",
|
|
"routing.md",
|
|
"audio.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "model-routing-ar",
|
|
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
|
|
"expected_document": "routing.md",
|
|
"expected_evidence": "auto_routed",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"routing.md",
|
|
"images.md",
|
|
"storage.md",
|
|
"timeout.md",
|
|
"skills.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "الحقول التي يعيدها API بعد تحويل الصور هي `requested_model` و`auto_routed`.\nالمصدر: routing.md، المقطع 0",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"routing.md",
|
|
"images.md",
|
|
"storage.md",
|
|
"timeout.md",
|
|
"search.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "api-timeout-ar",
|
|
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
|
|
"expected_document": "timeout.md",
|
|
"expected_evidence": "504",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"timeout.md",
|
|
"routing.md",
|
|
"skills.md",
|
|
"images.md",
|
|
"storage.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "المصدر: timeout.md، المقطع 0\n\nعند انتهاء مهلة النموذج المحلي، يرجع FastAPI حالة HTTP 504 مع رسالة عربية موحدة ومعرّف الطلب.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"routing.md",
|
|
"timeout.md",
|
|
"storage.md",
|
|
"search.md",
|
|
"images.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
},
|
|
{
|
|
"id": "skill-permissions-en",
|
|
"query": "How does the server restrict tools for a selected skill?",
|
|
"expected_document": "skills.md",
|
|
"expected_evidence": "allowed_tools",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"skills.md",
|
|
"migration.md",
|
|
"workspace.md",
|
|
"preview.md",
|
|
"storage.md"
|
|
],
|
|
"expected_document_chunks": [
|
|
0
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "المصدر: skills.md، المقطع 0\nالخادم يقوم بتصفية الأدوات المُعلن عنها ويرفض أي استدعاء للنموذج خارج تلك المجموعة.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"skills.md",
|
|
"migration.md",
|
|
"workspace.md",
|
|
"preview.md",
|
|
"storage.md"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
}
|
|
],
|
|
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
|
|
}
|