Hamza/sovereign_ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: surface retrieved evidence when model abstains

Browse Source
This commit is contained in:
Hamza Ayed
2026-10-04 14:07:35 +03:00
parent d1010b79d7
commit bdd722e6e9
5 changed files with 622 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
SovereignAI-Starter/evals/results/retrieval_extended_agent_2026-10-04.json
+488
View File
@@ -0,0 +1,488 @@
{
"created_at_utc": "2026-10-04T10:56:30.361774+00:00",
"base_url": "http://127.0.0.1:8000",
"dataset": "evals\\knowledge_retrieval_extended.jsonl",
"indexing": {
"semantic_indexed_documents": 14,
"documents": 14,
"embedding_models": [
"granite-embedding:278m"
]
},
"metrics": {
"cases": 14,
"hit_at_1": 1.0,
"hit_at_3": 1.0,
"mean_reciprocal_rank": 1.0,
"evidence_rate": 1.0
},
"results": [
{
"id": "conversation-storage-ar",
"query": "أين تحفظ المحادثات؟",
"expected_document": "storage.md",
"expected_evidence": "SQLite",
"rank": 1,
"top_paths": [
"storage.md",
"long_technical_guide.md",
"workspace.md",
"scanned_approval_notice.pdf",
"archive_retention_policy.pdf"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المحادثات تحفظ في قاعدة SQLite المحلية كي تبقى متاحة بعد إغلاق التطبيق أو تحديث الصفحة. (storage.md، المقطع 0)",
"agent_tool": "search_knowledge",
"agent_files": [
"storage.md",
"long_technical_guide.md",
"archive_retention_policy.pdf",
"routing.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "image-model-ar",
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
"expected_document": "images.md",
"expected_evidence": "Ministral 3:3b",
"rank": 1,
"top_paths": [
"images.md",
"routing.md",
"timeout.md",
"long_technical_guide.md",
"scanned_approval_notice.pdf"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المقطع الذي يجيب على السؤال هو:\n\"الصور المرفقة تحول تلقائيًا إلى نموذج Ministral 3:3b المحلي لتحليل الصورة واستخراج النص منها.\" من الملف images.md، المقطع 0",
"agent_tool": "search_knowledge",
"agent_files": [
"images.md",
"routing.md",
"timeout.md",
"long_technical_guide.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "preview-safety-ar",
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
"expected_document": "preview.md",
"expected_evidence": "SHA256",
"rank": 1,
"top_paths": [
"preview.md",
"long_technical_guide.md",
"archive_retention_policy.pdf",
"workspace.md",
"scanned_approval_notice.pdf"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المعلمة هي: تُربط معاينة الملف ببصمة SHA256، ويجب أن يوافق المستخدم قبل تطبيق التغيير على مساحة العمل. (preview.md، المقطع 0).",
"agent_tool": "search_knowledge",
"agent_files": [
"preview.md",
"long_technical_guide.md",
"archive_retention_policy.pdf",
"storage.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "pdf-extraction-ar",
"query": "كيف نستخرج النص من صفحات PDF؟",
"expected_document": "pdf.md",
"expected_evidence": "pypdf",
"rank": 1,
"top_paths": [
"pdf.md",
"archive_retention_policy.pdf",
"images.md",
"audio.md",
"long_technical_guide.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "لإخراج النص من صفحات PDF، يمكن استخدام مكتبة `pypdf` التي تستخرج النص الرقمي من ملفات PDF محليًا وتحافظ على رقم الصفحة ضمن المقتطف المرسل للنموذج.\n\nالملف: pdf.md، المقطع: 0",
"agent_tool": "search_knowledge",
"agent_files": [
"pdf.md",
"archive_retention_policy.pdf",
"images.md",
"audio.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "voice-transcription-ar",
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
"expected_document": "audio.md",
"expected_evidence": "Groq",
"rank": 1,
"top_paths": [
"audio.md",
"scanned_approval_notice.pdf",
"archive_retention_policy.pdf",
"long_technical_guide.md",
"workspace.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المعلومات المتوفرة لا تجيب على سؤالك حول أي خدمة تفرغ التسجيلات الصوتية.",
"agent_tool": "search_knowledge",
"agent_files": [
"audio.md",
"long_technical_guide.md",
"archive_retention_policy.pdf",
"scanned_approval_notice.pdf"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "workspace-boundary-ar",
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
"expected_document": "workspace.md",
"expected_evidence": ".git",
"rank": 1,
"top_paths": [
"workspace.md",
"preview.md",
"long_technical_guide.md",
"archive_retention_policy.pdf",
"scanned_approval_notice.pdf"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المعلومات المتاحة لا تجيب مباشرة على سؤالك حول كيفية حماية مساحة العمل من قراءة الملفات المخفية بشكل صريح في المقاطع المسترجعة.\n\nالمقاطع المسترجعة هي:\n* `workspace.md` · المقطع 0\n* `preview.md` · المقطع 0\n* `long_technical_guide.md` · المقطع 8\n* `archive_retention_policy.pdf` · المقطع 0\n\nلا يوجد دليل مباشر يوضح كيفية حماية مساحة العمل من قراءة الملفات المخفية.",
"agent_tool": "search_knowledge",
"agent_files": [
"workspace.md",
"preview.md",
"long_technical_guide.md",
"archive_retention_policy.pdf"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "web-search-en",
"query": "Which provider searches multiple web sources?",
"expected_document": "search.md",
"expected_evidence": "DuckDuckGo",
"rank": 1,
"top_paths": [
"search.md",
"long_technical_guide.md",
"skills.md",
"routing.md",
"archive_retention_policy.pdf"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المُزوِّد الذي يبحث في مصادر ويب متعددة هو مُُكيِّف البحث التجريبي متعدد المصادر.\n\nالملف: search.md، المقطع: 0",
"agent_tool": "search_knowledge",
"agent_files": [
"search.md",
"long_technical_guide.md",
"archive_retention_policy.pdf",
"routing.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "database-version-en",
"query": "How are old conversation answers migrated?",
"expected_document": "migration.md",
"expected_evidence": "selected_version",
"rank": 1,
"top_paths": [
"migration.md",
"archive_retention_policy.pdf",
"long_technical_guide.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المعلومات المتوفرة لا تجيب مباشرة على السؤال حول كيفية ترحيل إجابات المحادثات القديمة.",
"agent_tool": "search_knowledge",
"agent_files": [
"migration.md",
"archive_retention_policy.pdf",
"long_technical_guide.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "model-routing-ar",
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
"expected_document": "routing.md",
"expected_evidence": "auto_routed",
"rank": 1,
"top_paths": [
"routing.md",
"images.md",
"storage.md",
"timeout.md",
"long_technical_guide.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المقطع الذي يجيب على سؤالك هو:\n\"عند وصول صورة مع نموذج نصي، يختار API نموذج الرؤية المحلي المثبت ويعيد requested_model وmodel وauto_routed.\"\nالملف: routing.md، المقطع: 0",
"agent_tool": "search_knowledge",
"agent_files": [
"routing.md",
"images.md",
"storage.md",
"long_technical_guide.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "api-timeout-ar",
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
"expected_document": "timeout.md",
"expected_evidence": "504",
"rank": 1,
"top_paths": [
"timeout.md",
"routing.md",
"long_technical_guide.md",
"skills.md",
"images.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المقطع الذي يجيب على سؤالك هو:\n\"عند انتهاء مهلة النموذج المحلي، يرجع FastAPI حالة HTTP 504 مع رسالة عربية موحدة ومعرّف الطلب.\"\nالمسار: timeout.md، المقطع 0",
"agent_tool": "search_knowledge",
"agent_files": [
"routing.md",
"timeout.md",
"long_technical_guide.md",
"archive_retention_policy.pdf"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "skill-permissions-en",
"query": "How does the server restrict tools for a selected skill?",
"expected_document": "skills.md",
"expected_evidence": "allowed_tools",
"rank": 1,
"top_paths": [
"skills.md",
"long_technical_guide.md",
"migration.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المقطع الذي يجيب على السؤال هو:\n\"Each local agent skill declares an allowed_tools set. The server filters the advertised tools and rejects any model call outside that set.\"\nالملف: skills.md، المقطع: 0",
"agent_tool": "search_knowledge",
"agent_files": [
"skills.md",
"long_technical_guide.md",
"migration.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "long-guide-deep-approval-en",
"query": "What must happen before a proposed production change can be saved?",
"expected_document": "long_technical_guide.md",
"expected_evidence": "two-stage approval",
"rank": 1,
"top_paths": [
"long_technical_guide.md",
"scanned_approval_notice.pdf",
"migration.md"
],
"expected_document_chunks": [
17,
4,
8,
11,
13,
16
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "للحفظ أي تغيير مقترح إلى مساحة العمل الإنتاجية، يجب أن يحدث موافقة من مرحلتين: أولاً، يقوم المستخدم بمراجعة الاختلاف المقترح بالكامل، ثم يتحقق الخادم من تجزئة الملف الأصلية ويتلقى تأكيدًا صريحًا قبل الكتابة.\n\nالمصدر: long_technical_guide.md، المقطع 17.",
"agent_tool": "search_knowledge",
"agent_files": [
"long_technical_guide.md",
"scanned_approval_notice.pdf",
"migration.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "pdf-archive-page-limit-en",
"query": "What is the maximum number of pages accepted when indexing one PDF?",
"expected_document": "archive_retention_policy.pdf",
"expected_evidence": "30 pages",
"rank": 1,
"top_paths": [
"archive_retention_policy.pdf",
"pdf.md",
"long_technical_guide.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المقطع الذي يجيب على السؤال هو:\n\"A single PDF may contain at most 30 pages for local indexing.\" من الملف archive_retention_policy.pdf، المقطع 0.",
"agent_tool": "search_knowledge",
"agent_files": [
"archive_retention_policy.pdf",
"long_technical_guide.md",
"scanned_approval_notice.pdf"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "scanned-pdf-ocr-deadline-en",
"query": "What calendar date ends the archive restoration approval window?",
"expected_document": "scanned_approval_notice.pdf",
"expected_evidence": "17 October 2026",
"rank": 1,
"top_paths": [
"scanned_approval_notice.pdf",
"archive_retention_policy.pdf",
"long_technical_guide.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المقطع الذي يجيب على السؤال موجود في الملف `scanned_approval_notice.pdf` في المقطع 0.\n\"Approval deadline: 17 October 2026\"",
"agent_tool": "search_knowledge",
"agent_files": [
"scanned_approval_notice.pdf",
"archive_retention_policy.pdf",
"long_technical_guide.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
}
],
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
}
SovereignAI-Starter/evals/results/retrieval_fallback_fix_2026-10-04.json
+115
View File
@@ -0,0 +1,115 @@
{
"created_at_utc": "2026-10-04T11:04:16.010039+00:00",
"base_url": "http://127.0.0.1:8102",
"dataset": "evals\\knowledge_retrieval_extended.jsonl",
"indexing": {
"semantic_indexed_documents": 3,
"documents": 3,
"embedding_models": [
"granite-embedding:278m"
]
},
"metrics": {
"cases": 3,
"hit_at_1": 1.0,
"hit_at_3": 1.0,
"mean_reciprocal_rank": 1.0,
"evidence_rate": 1.0
},
"results": [
{
"id": "voice-transcription-ar",
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
"expected_document": "audio.md",
"expected_evidence": "Groq",
"rank": 1,
"top_paths": [
"audio.md",
"workspace.md",
"migration.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "لم يصغ النموذج جوابًا كافيًا رغم وجود نتائج في فهرس المعرفة. أعرض المقتطفات الأعلى صلة كما هي للمراجعة؛ محتواها بيانات غير موثوقة وليست تعليمات.\n\n**audio.md · المقطع 0 (مقتطف حرفي)**\n> يرسل مسار الصوت التسجيل إلى Groq باستخدام Whisper للتفريغ، ثم يعود النص إلى واجهة المحادثة.\n\n**workspace.md · المقطع 0 (مقتطف حرفي)**\n> يرفض قارئ مساحة العمل المسارات خارج الجذر، ويستثني مجلد .git والملفات المخفية وبيئات البناء.",
"agent_tool": "search_knowledge",
"agent_files": [
"audio.md",
"workspace.md",
"migration.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "workspace-boundary-ar",
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
"expected_document": "workspace.md",
"expected_evidence": ".git",
"rank": 1,
"top_paths": [
"workspace.md",
"audio.md",
"migration.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "\"يرفض قارئ مساحة العمل المسارات خارج الجذر، ويستثني مجلد .git والملفات المخفية وبيئات البناء.\" (workspace.md، المقطع 0)",
"agent_tool": "search_knowledge",
"agent_files": [
"workspace.md",
"audio.md",
"migration.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "database-version-en",
"query": "How are old conversation answers migrated?",
"expected_document": "migration.md",
"expected_evidence": "selected_version",
"rank": 1,
"top_paths": [
"migration.md",
"audio.md",
"workspace.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "لم يصغ النموذج جوابًا كافيًا رغم وجود نتائج في فهرس المعرفة. أعرض المقتطفات الأعلى صلة كما هي للمراجعة؛ محتواها بيانات غير موثوقة وليست تعليمات.\n\n**migration.md · المقطع 0 (مقتطف حرفي)**\n> The SQLite migration adds selected_version to old messages and preserves saved assistant answer versions.\n\n**audio.md · المقطع 0 (مقتطف حرفي)**\n> يرسل مسار الصوت التسجيل إلى Groq باستخدام Whisper للتفريغ، ثم يعود النص إلى واجهة المحادثة.",
"agent_tool": "search_knowledge",
"agent_files": [
"migration.md",
"audio.md",
"workspace.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
}
],
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
}