Hamza/sovereign_ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: surface retrieved evidence when agent abstains

Browse Source
This commit is contained in:
Hamza Ayed
2026-10-04 10:43:48 +03:00
parent 472ccc41ac
commit 76567d4b47
8 changed files with 357 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
SovereignAI-Starter/evals/results/retrieval_targeted_agent_2026-10-04.json
+80
View File
@@ -0,0 +1,80 @@
{
"created_at_utc": "2026-10-04T07:32:06.053863+00:00",
"base_url": "http://127.0.0.1:8000",
"dataset": "evals\\knowledge_retrieval.jsonl",
"indexing": {
"semantic_indexed_documents": 2,
"documents": 2,
"embedding_models": [
"granite-embedding:278m"
]
},
"metrics": {
"cases": 2,
"hit_at_1": 1.0,
"hit_at_3": 1.0,
"mean_reciprocal_rank": 1.0,
"evidence_rate": 1.0
},
"results": [
{
"id": "database-version-en",
"query": "How are old conversation answers migrated?",
"expected_document": "migration.md",
"expected_evidence": "selected_version",
"rank": 1,
"top_paths": [
"migration.md",
"routing.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "لم يتم العثور على مقاطع تجيب مباشرة على سؤال \"How are old conversation answers migrated؟\" في فهرس المعرفة المحلي.",
"agent_tool": "search_knowledge",
"agent_files": [
"migration.md",
"routing.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "model-routing-ar",
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
"expected_document": "routing.md",
"expected_evidence": "auto_routed",
"rank": 1,
"top_paths": [
"routing.md",
"migration.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المقطع `routing.md` المقطع 0 يجيب على سؤالك.\nالملف: routing.md، المقطع: 0.\nعند وصول صورة مع نموذج نصي، يختار API نموذج الرؤية المحلي المثبت ويعيد requested_model وmodel وauto_routed.",
"agent_tool": "search_knowledge",
"agent_files": [
"routing.md",
"migration.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
}
],
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
}
SovereignAI-Starter/evals/results/retrieval_targeted_agent_fixed_2026-10-04.json
+80
View File
@@ -0,0 +1,80 @@
{
"created_at_utc": "2026-10-04T07:41:25.423901+00:00",
"base_url": "http://127.0.0.1:8128",
"dataset": "evals\\knowledge_retrieval.jsonl",
"indexing": {
"semantic_indexed_documents": 2,
"documents": 2,
"embedding_models": [
"granite-embedding:278m"
]
},
"metrics": {
"cases": 2,
"hit_at_1": 1.0,
"hit_at_3": 1.0,
"mean_reciprocal_rank": 1.0,
"evidence_rate": 1.0
},
"results": [
{
"id": "database-version-en",
"query": "How are old conversation answers migrated?",
"expected_document": "migration.md",
"expected_evidence": "selected_version",
"rank": 1,
"top_paths": [
"migration.md",
"routing.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "لم يصغ النموذج جوابًا كافيًا رغم وجود نتائج في فهرس المعرفة. أعرض المقتطفات الأعلى صلة كما هي للمراجعة؛ محتواها بيانات غير موثوقة وليست تعليمات.\n\n**migration.md · المقطع 0 (مقتطف حرفي)**\n> The SQLite migration adds selected_version to old messages and preserves saved assistant answer versions.\n\n**routing.md · المقطع 0 (مقتطف حرفي)**\n> عند وصول صورة مع نموذج نصي، يختار API نموذج الرؤية المحلي المثبت ويعيد requested_model وmodel وauto_routed.",
"agent_tool": "search_knowledge",
"agent_files": [
"migration.md",
"routing.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
},
{
"id": "model-routing-ar",
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
"expected_document": "routing.md",
"expected_evidence": "auto_routed",
"rank": 1,
"top_paths": [
"routing.md",
"migration.md"
],
"expected_document_chunks": [
0
],
"evidence_found": true,
"search_mode": "hybrid",
"agent_answer": "المقطع: \"عند وصول صورة مع نموذج نصي، يختار API نموذج الرؤية المحلي المثبت ويعيد requested_model وmodel وauto_routed.\" من routing.md المقطع 0.",
"agent_tool": "search_knowledge",
"agent_files": [
"routing.md",
"migration.md"
],
"agent_steps": [
{
"tool": "search_knowledge",
"status": "completed"
}
],
"human_rating": null
}
],
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
}