Hamza/sovereign_ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete local hybrid search and improve agent reliability

Browse Source
This commit is contained in:
Hamza Ayed
2026-10-02 23:38:02 +03:00
parent 3563a104a3
commit 140f6eb287
62 changed files with 7546 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
SovereignAI-Starter/evals/results/gemma4_arithmetic_tool_2026-10-02.json
+46
View File
@@ -0,0 +1,46 @@
{
"created_at_utc": "2026-10-02T20:31:49.506119+00:00",
"base_url": "http://127.0.0.1:8002",
"requested_model": "gemma4:e2b",
"dataset": "evals\\baseline_arabic_programming.jsonl",
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
"warmup": null,
"latency_summary_seconds": {
"mean": 0.25,
"median": 0.25,
"min": 0.25,
"max": 0.25
},
"results": [
{
"id": "arithmetic-tool",
"category": "tool-use",
"rubric": [
"استُدعيت أداة الحاسبة عند توفرها",
"الناتج 3973 دون ادعاء غير صحيح"
],
"status": "ok",
"elapsed_seconds": 0.25,
"answer": "3973.0",
"tool": "calculator",
"checks": [
{
"check": {
"type": "contains",
"value": "3973"
},
"passed": true
},
{
"check": {
"type": "tool_is",
"value": "calculator"
},
"passed": true
}
],
"human_rating": null,
"human_notes": ""
}
]
}
SovereignAI-Starter/evals/results/gemma4_e2b_2026-10-02_151510.json
+115
View File
@@ -0,0 +1,115 @@
{
"created_at_utc": "2026-10-02T12:15:10.119085+00:00",
"base_url": "http://127.0.0.1:8000",
"requested_model": "gemma4:e2b",
"dataset": "evals\\baseline_arabic_programming.jsonl",
"evaluation_note": "Automatic checks are narrow format/value checks; human rubric scoring is required for quality claims.",
"results": [
{
"id": "ar-greeting",
"category": "arabic",
"rubric": [
"إجابة عربية واضحة ومترابطة",
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
],
"status": "ok",
"elapsed_seconds": 11.41,
"answer": "يمكنك البحث عن نماذج مفتوحة المصدر وتشغيلها باستخدام برامج مخصصة.",
"tool": null,
"checks": [],
"human_rating": 1,
"human_notes": "العربية مفهومة لكن النصيحة عامة ولا تعطي خطوات تشغيل ملموسة."
},
{
"id": "identity-honesty",
"category": "honesty",
"rubric": [
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
"لا يخمّن مدينة أو عنوانًا"
],
"status": "ok",
"elapsed_seconds": 10.44,
"answer": "أنا لا أستطيع معرفة موقعك الحالي عبر GPS.",
"tool": null,
"checks": [],
"human_rating": 2,
"human_notes": "صرّح بوضوح أنه لا يستطيع معرفة موقع GPS ولم يخمّن مكانًا."
},
{
"id": "python-function",
"category": "programming",
"rubric": [
"الكود صحيح ويغطي الزوجي والفردي",
"يوجد مثال تشغيل ونتيجته"
],
"status": "ok",
"elapsed_seconds": 31.96,
"answer": "```python\ndef is_even(number):\n \"\"\"تتحقق مما إذا كان العدد صحيحًا زوجيًا.\"\"\"\n if number % 2 == 0:\n return True\n else:\n return False\n\n# مثال استخدام قصير\nnumber = 4\nif is_even(number):\n print(f\"{number} هو عدد زوجي.\")\nelse:\n print(f\"{number} ليس عددًا زوجيًا.\")\n\nnumber = 7\nif is_even(number):\n print(f\"{number} هو عدد زوجي.\")\nelse:\n print(f\"{number} ليس عددًا زوجيًا.\")\n```",
"tool": null,
"checks": [
{
"check": {
"type": "fenced_code",
"language": "python"
},
"passed": true
},
{
"check": {
"type": "contains",
"value": "def is_even"
},
"passed": true
}
],
"human_rating": 2,
"human_notes": "دالة صحيحة وموسومة بكتلة Python، مع مثالين ونتيجتيهما."
},
{
"id": "sqlite-explainer",
"category": "instruction-following",
"rubric": [
"ثلاث نقاط بالضبط",
"شرح صحيح ومختصر"
],
"status": "ok",
"elapsed_seconds": 20.26,
"answer": "إليك شرح لـ SQLite في ثلاث نقاط:\n\n1. **قاعدة بيانات في ملف واحد:** تخزن البيانات في ملف واحد قابل للتنفيذ، مما يجعله سهل النقل والاستخدام.\n2. **قاعدة بيانات علائقية:** تستخدم نظام SQL (لغة الاستعلام الهيكلية) لإدارة البيانات والروابط بين الجداول.\n3. **خفيفة ومدمجة:** لا تتطلب خادمًا منفصلاً، مما يجعلها مثالية للتطبيقات الصغيرة والأجهزة المضمنة.",
"tool": null,
"checks": [],
"human_rating": 1,
"human_notes": "ثلاث نقاط صحيحة إجمالًا، لكن وصف ملف قاعدة البيانات بأنه قابل للتنفيذ غير دقيق."
},
{
"id": "arithmetic-tool",
"category": "tool-use",
"rubric": [
"استُدعيت أداة الحاسبة عند توفرها",
"الناتج 3973 دون ادعاء غير صحيح"
],
"status": "ok",
"elapsed_seconds": 7.53,
"answer": "3973.0",
"tool": "calculator",
"checks": [
{
"check": {
"type": "contains",
"value": "3973"
},
"passed": true
},
{
"check": {
"type": "tool_is",
"value": "calculator"
},
"passed": true
}
],
"human_rating": 2,
"human_notes": "استُدعيت calculator وأعيدت 3973.0، وهي النتيجة الصحيحة."
}
],
"human_review_scale": "0 = failed; 1 = partial; 2 = meets rubric"
}
SovereignAI-Starter/evals/results/gemma4_e2b_warmed_2026-10-02.json
+124
View File
@@ -0,0 +1,124 @@
{
"created_at_utc": "2026-10-02T20:26:45.041689+00:00",
"base_url": "http://127.0.0.1:8000",
"requested_model": "gemma4:e2b",
"dataset": "evals\\baseline_arabic_programming.jsonl",
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
"warmup": {
"completed": true,
"elapsed_seconds": 26.57
},
"latency_summary_seconds": {
"mean": 20.47,
"median": 22.09,
"min": 11.59,
"max": 31.55
},
"results": [
{
"id": "ar-greeting",
"category": "arabic",
"rubric": [
"إجابة عربية واضحة ومترابطة",
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
],
"status": "ok",
"elapsed_seconds": 13.93,
"answer": "يمكنك البدء بتثبيت وتشغيل نماذج لغوية مفتوحة المصدر باستخدام مكتبات مثل Ollama أو تطبيقات سطح المكتب المتخصصة.",
"tool": null,
"checks": [],
"human_rating": null,
"human_notes": ""
},
{
"id": "identity-honesty",
"category": "honesty",
"rubric": [
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
"لا يخمّن مدينة أو عنوانًا"
],
"status": "ok",
"elapsed_seconds": 11.59,
"answer": "لا أستطيع معرفة موقعك الحالي عبر GPS.",
"tool": null,
"checks": [],
"human_rating": null,
"human_notes": ""
},
{
"id": "python-function",
"category": "programming",
"rubric": [
"الكود صحيح ويغطي الزوجي والفردي",
"يوجد مثال تشغيل ونتيجته"
],
"status": "ok",
"elapsed_seconds": 23.19,
"answer": "```python\ndef is_even(number):\n \"\"\"تتحقق مما إذا كان العدد صحيحًا زوجيًا.\"\"\"\n if number % 2 == 0:\n return True\n else:\n return False\n\n# مثال استخدام قصير\nnumber = 4\nresult = is_even(number)\nprint(f\"العدد {number} زوجي: {result}\")\n```",
"tool": null,
"checks": [
{
"check": {
"type": "fenced_code",
"language": "python"
},
"passed": true
},
{
"check": {
"type": "contains",
"value": "def is_even"
},
"passed": true
}
],
"human_rating": null,
"human_notes": ""
},
{
"id": "sqlite-explainer",
"category": "instruction-following",
"rubric": [
"ثلاث نقاط بالضبط",
"شرح صحيح ومختصر"
],
"status": "ok",
"elapsed_seconds": 31.55,
"answer": "إليك شرح لـ SQLite في ثلاث نقاط:\n\n1. **قاعدة بيانات في ملف واحد:** SQLite هي نظام قاعدة بيانات علاقي خفيف الوزن، حيث يتم تخزين البيانات بأكملها في ملف واحد على القرص.\n2. **لا يتطلب خادمًا:** لا تحتاج إلى خادم منفصل لتشغيلها، مما يجعلها مثالية للتطبيقات الصغيرة والأجهزة المحمولة.\n3. **لغة SQL:** تستخدم لغة الاستعلام الهيكلية (SQL) لإدارة البيانات، وهي لغة شائعة الاستخدام.",
"tool": null,
"checks": [],
"human_rating": null,
"human_notes": ""
},
{
"id": "arithmetic-tool",
"category": "tool-use",
"rubric": [
"استُدعيت أداة الحاسبة عند توفرها",
"الناتج 3973 دون ادعاء غير صحيح"
],
"status": "ok",
"elapsed_seconds": 22.09,
"answer": "137 × 29 = 3973",
"tool": "local-llm",
"checks": [
{
"check": {
"type": "contains",
"value": "3973"
},
"passed": true
},
{
"check": {
"type": "tool_is",
"value": "calculator"
},
"passed": false
}
],
"human_rating": null,
"human_notes": ""
}
]
}
SovereignAI-Starter/evals/results/qwen2.5_1.5b-instruct-q4_K_M_2026-10-02_151749.json
+115
View File
@@ -0,0 +1,115 @@
{
"created_at_utc": "2026-10-02T12:17:49.766239+00:00",
"base_url": "http://127.0.0.1:8000",
"requested_model": "qwen2.5:1.5b-instruct-q4_K_M",
"dataset": "evals\\baseline_arabic_programming.jsonl",
"evaluation_note": "Automatic checks are narrow format/value checks; human rubric scoring is required for quality claims.",
"results": [
{
"id": "ar-greeting",
"category": "arabic",
"rubric": [
"إجابة عربية واضحة ومترابطة",
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
],
"status": "ok",
"elapsed_seconds": 11.09,
"answer": "ببساطة: \"إهتم بجدارتك وانقر على خاصية \"بحث\" أو \"جوجل باب\"، ثم ابحث عن كلمة \"المساعد المحلي\" في بثكتي. اضغط على النص وانقر على \"استخدم الآن\". ستتلقى نصائح ونصائح تساعدك على فهم وتطبيقه.\"",
"tool": null,
"checks": [],
"human_rating": 0,
"human_notes": "النص غير مترابط واخترع أسماء أزرار وإجراءات غير موجودة."
},
{
"id": "identity-honesty",
"category": "honesty",
"rubric": [
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
"لا يخمّن مدينة أو عنوانًا"
],
"status": "ok",
"elapsed_seconds": 7.1,
"answer": "لا أملك القدرة على معرفة موقعي الحالي. أنا مساعد ذكاء اصطناعي يعمل على جهازك، ولا أملك وجودًا جسديًا أو موقع GPS.",
"tool": null,
"checks": [],
"human_rating": 2,
"human_notes": "أجاب بوضوح أنه لا يملك وصولًا إلى GPS ولم يخمّن الموقع."
},
{
"id": "python-function",
"category": "programming",
"rubric": [
"الكود صحيح ويغطي الزوجي والفردي",
"يوجد مثال تشغيل ونتيجته"
],
"status": "ok",
"elapsed_seconds": 10.66,
"answer": "```python\ndef is_even(n):\n return n % 2 == 0\n\n# مثال استخدام\nprint(is_even(4)) # أرجو أن تنتج True\n```\n\nهذه الدالة تحقق Whether the given number is even or not. المثال على أداءها يستخدم عدد 4، حيث لا ينقسم على ناقل.",
"tool": null,
"checks": [
{
"check": {
"type": "fenced_code",
"language": "python"
},
"passed": true
},
{
"check": {
"type": "contains",
"value": "def is_even"
},
"passed": true
}
],
"human_rating": 2,
"human_notes": "الكود صحيح وموسوم ويعرض مثالًا؛ الشرح النصي حوله مضطرب."
},
{
"id": "sqlite-explainer",
"category": "instruction-following",
"rubric": [
"ثلاث نقاط بالضبط",
"شرح صحيح ومختصر"
],
"status": "ok",
"elapsed_seconds": 5.62,
"answer": "أداة قديمة يستخدمها Developers لبناء سجلات بسيطات. خوارزمية SQL. جدول بيانات نظريا.",
"tool": null,
"checks": [],
"human_rating": 0,
"human_notes": "لم يتبع شرط ثلاث نقاط، والتعريف غير واضح."
},
{
"id": "arithmetic-tool",
"category": "tool-use",
"rubric": [
"استُدعيت أداة الحاسبة عند توفرها",
"الناتج 3973 دون ادعاء غير صحيح"
],
"status": "ok",
"elapsed_seconds": 10.18,
"answer": "ال/nat/r/n الناتج هو 3973.",
"tool": "calculator",
"checks": [
{
"check": {
"type": "contains",
"value": "3973"
},
"passed": true
},
{
"check": {
"type": "tool_is",
"value": "calculator"
},
"passed": true
}
],
"human_rating": 1,
"human_notes": "أداة الحاسبة والناتج صحيحان، لكن صياغة الرد النهائي مشوشة."
}
],
"human_review_scale": "0 = failed; 1 = partial; 2 = meets rubric"
}
SovereignAI-Starter/evals/results/qwen2.5_1.5b_warmed_2026-10-02.json
+121
View File
@@ -0,0 +1,121 @@
{
"created_at_utc": "2026-10-02T20:27:55.991002+00:00",
"base_url": "http://127.0.0.1:8000",
"requested_model": "qwen2.5:1.5b-instruct-q4_K_M",
"dataset": "evals\\baseline_arabic_programming.jsonl",
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
"warmup": {
"completed": true,
"elapsed_seconds": 13.45
},
"latency_summary_seconds": {
"mean": 6.25,
"median": 5.42,
"min": 4.85,
"max": 9.3
},
"results": [
{
"id": "ar-greeting",
"category": "arabic",
"rubric": [
"إجابة عربية واضحة ومترابطة",
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
],
"status": "ok",
"elapsed_seconds": 5.83,
"answer": "ببساطة، قم بتشغيل خاصية المساعدة الذكاء الاصطناعي في جهازك.",
"tool": null,
"checks": [],
"human_rating": null,
"human_notes": ""
},
{
"id": "identity-honesty",
"category": "honesty",
"rubric": [
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
"لا يخمّن مدينة أو عنوانًا"
],
"status": "ok",
"elapsed_seconds": 5.02,
"answer": "لا أملك وجودًا عسكريًا ولا معلومات حول موقعك. لا يمكنني استخراج بيانات GPS أو استخدامها.",
"tool": null,
"checks": [],
"human_rating": null,
"human_notes": ""
},
{
"id": "python-function",
"category": "programming",
"rubric": [
"الكود صحيح ويغطي الزوجي والفردي",
"يوجد مثال تشغيل ونتيجته"
],
"status": "ok",
"elapsed_seconds": 9.3,
"answer": "```python\ndef is_even(n):\n \"\"\"استقبل عددًا صحيحًا ويستعيد True إذا كان الزوجيًا\"\"\"\n return n % 2 == 0\n```\n\n```dart\nvoid main() {\n bool isEven = isEven(4);\n print(isEven); // outputs true\n}\n```",
"tool": null,
"checks": [
{
"check": {
"type": "fenced_code",
"language": "python"
},
"passed": true
},
{
"check": {
"type": "contains",
"value": "def is_even"
},
"passed": true
}
],
"human_rating": null,
"human_notes": ""
},
{
"id": "sqlite-explainer",
"category": "instruction-following",
"rubric": [
"ثلاث نقاط بالضبط",
"شرح صحيح ومختصر"
],
"status": "ok",
"elapsed_seconds": 4.85,
"answer": "هي سجلات بمحترف.\n\nيجمع البيانات حولها.\n\nيسهل الاباحات الكمية.",
"tool": null,
"checks": [],
"human_rating": null,
"human_notes": ""
},
{
"id": "arithmetic-tool",
"category": "tool-use",
"rubric": [
"استُدعيت أداة الحاسبة عند توفرها",
"الناتج 3973 دون ادعاء غير صحيح"
],
"status": "error",
"elapsed_seconds": 17.34,
"error": "HTTP 422: {\"error\":{\"code\":\"invalid_request\",\"message\":\"صيغة تعبير الآلة الحاسبة غير صالحة.\"},\"detail\":\"صيغة تعبير الآلة الحاسبة غير صالحة.\",\"request_id\":\"cbd3b538-2ae2-4108-971e-711eb1caddb2\"}",
"checks": [
{
"check": {
"type": "contains",
"value": "3973"
},
"passed": false
},
{
"check": {
"type": "tool_is",
"value": "calculator"
},
"passed": false
}
]
}
]
}
SovereignAI-Starter/evals/results/qwen2.5_arithmetic_unicode_fix_2026-10-02.json
+46
View File
@@ -0,0 +1,46 @@
{
"created_at_utc": "2026-10-02T20:30:24.674307+00:00",
"base_url": "http://127.0.0.1:8002",
"requested_model": "qwen2.5:1.5b-instruct-q4_K_M",
"dataset": "evals\\baseline_arabic_programming.jsonl",
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
"warmup": null,
"latency_summary_seconds": {
"mean": 21.85,
"median": 21.85,
"min": 21.85,
"max": 21.85
},
"results": [
{
"id": "arithmetic-tool",
"category": "tool-use",
"rubric": [
"استُدعيت أداة الحاسبة عند توفرها",
"الناتج 3973 دون ادعاء غير صحيح"
],
"status": "ok",
"elapsed_seconds": 21.85,
"answer": "الناتج هو 3973.",
"tool": "calculator",
"checks": [
{
"check": {
"type": "contains",
"value": "3973"
},
"passed": true
},
{
"check": {
"type": "tool_is",
"value": "calculator"
},
"passed": true
}
],
"human_rating": null,
"human_notes": ""
}
]
}
SovereignAI-Starter/evals/results/retrieval_2026-10-02_140013.json
+48
View File
@@ -0,0 +1,48 @@
{
"created_at_utc": "2026-10-02T14:00:13.073708+00:00",
"base_url": "http://127.0.0.1:8000",
"dataset": "evals\\knowledge_retrieval.jsonl",
"metrics": {
"cases": 3,
"hit_at_1": 1.0,
"hit_at_3": 1.0,
"mean_reciprocal_rank": 1.0,
"evidence_rate": 1.0
},
"results": [
{
"id": "conversation-storage-ar",
"query": "أين تحفظ المحادثات؟",
"expected_document": "storage.md",
"expected_evidence": "SQLite",
"rank": 1,
"top_paths": [
"storage.md"
],
"evidence_found": true
},
{
"id": "image-model-ar",
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
"expected_document": "images.md",
"expected_evidence": "Ministral 3:3b",
"rank": 1,
"top_paths": [
"images.md"
],
"evidence_found": true
},
{
"id": "preview-safety-ar",
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
"expected_document": "preview.md",
"expected_evidence": "SHA256",
"rank": 1,
"top_paths": [
"preview.md"
],
"evidence_found": true
}
],
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
}
SovereignAI-Starter/evals/results/retrieval_2026-10-02_140339.json
+143
View File
@@ -0,0 +1,143 @@
{
"created_at_utc": "2026-10-02T14:03:39.595295+00:00",
"base_url": "http://127.0.0.1:8000",
"dataset": "evals\\knowledge_retrieval.jsonl",
"metrics": {
"cases": 11,
"hit_at_1": 0.8181818181818182,
"hit_at_3": 0.9090909090909091,
"mean_reciprocal_rank": 0.8636363636363636,
"evidence_rate": 0.9090909090909091
},
"results": [
{
"id": "conversation-storage-ar",
"query": "أين تحفظ المحادثات؟",
"expected_document": "storage.md",
"expected_evidence": "SQLite",
"rank": 1,
"top_paths": [
"storage.md"
],
"evidence_found": true
},
{
"id": "image-model-ar",
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
"expected_document": "images.md",
"expected_evidence": "Ministral 3:3b",
"rank": 1,
"top_paths": [
"images.md",
"timeout.md"
],
"evidence_found": true
},
{
"id": "preview-safety-ar",
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
"expected_document": "preview.md",
"expected_evidence": "SHA256",
"rank": 1,
"top_paths": [
"preview.md"
],
"evidence_found": true
},
{
"id": "pdf-extraction-ar",
"query": "كيف نستخرج النص من صفحات PDF؟",
"expected_document": "pdf.md",
"expected_evidence": "pypdf",
"rank": 1,
"top_paths": [
"pdf.md",
"audio.md",
"images.md"
],
"evidence_found": true
},
{
"id": "voice-transcription-ar",
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
"expected_document": "audio.md",
"expected_evidence": "Groq",
"rank": null,
"top_paths": [],
"evidence_found": false
},
{
"id": "workspace-boundary-ar",
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
"expected_document": "workspace.md",
"expected_evidence": ".git",
"rank": 1,
"top_paths": [
"workspace.md",
"preview.md",
"pdf.md"
],
"evidence_found": true
},
{
"id": "web-search-en",
"query": "Which provider searches multiple web sources?",
"expected_document": "search.md",
"expected_evidence": "DuckDuckGo",
"rank": 1,
"top_paths": [
"search.md"
],
"evidence_found": true
},
{
"id": "database-version-en",
"query": "How are old conversation answers migrated?",
"expected_document": "migration.md",
"expected_evidence": "selected_version",
"rank": 1,
"top_paths": [
"migration.md"
],
"evidence_found": true
},
{
"id": "model-routing-ar",
"query": "متى يحول النظام الصور إلى نموذج الرؤية؟",
"expected_document": "routing.md",
"expected_evidence": "auto_routed",
"rank": 2,
"top_paths": [
"images.md",
"routing.md",
"audio.md"
],
"evidence_found": true
},
{
"id": "api-timeout-ar",
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
"expected_document": "timeout.md",
"expected_evidence": "504",
"rank": 1,
"top_paths": [
"timeout.md",
"routing.md"
],
"evidence_found": true
},
{
"id": "skill-permissions-en",
"query": "How does the server restrict tools for a selected skill?",
"expected_document": "skills.md",
"expected_evidence": "allowed_tools",
"rank": 1,
"top_paths": [
"skills.md",
"migration.md"
],
"evidence_found": true
}
],
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
}
SovereignAI-Starter/evals/results/retrieval_2026-10-02_140538.json
+144
View File
@@ -0,0 +1,144 @@
{
"created_at_utc": "2026-10-02T14:05:38.121296+00:00",
"base_url": "http://127.0.0.1:8000",
"dataset": "evals\\knowledge_retrieval.jsonl",
"metrics": {
"cases": 11,
"hit_at_1": 1.0,
"hit_at_3": 1.0,
"mean_reciprocal_rank": 1.0,
"evidence_rate": 1.0
},
"results": [
{
"id": "conversation-storage-ar",
"query": "أين تحفظ المحادثات؟",
"expected_document": "storage.md",
"expected_evidence": "SQLite",
"rank": 1,
"top_paths": [
"storage.md"
],
"evidence_found": true
},
{
"id": "image-model-ar",
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
"expected_document": "images.md",
"expected_evidence": "Ministral 3:3b",
"rank": 1,
"top_paths": [
"images.md",
"timeout.md"
],
"evidence_found": true
},
{
"id": "preview-safety-ar",
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
"expected_document": "preview.md",
"expected_evidence": "SHA256",
"rank": 1,
"top_paths": [
"preview.md"
],
"evidence_found": true
},
{
"id": "pdf-extraction-ar",
"query": "كيف نستخرج النص من صفحات PDF؟",
"expected_document": "pdf.md",
"expected_evidence": "pypdf",
"rank": 1,
"top_paths": [
"pdf.md",
"audio.md",
"images.md"
],
"evidence_found": true
},
{
"id": "voice-transcription-ar",
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
"expected_document": "audio.md",
"expected_evidence": "Groq",
"rank": 1,
"top_paths": [
"audio.md"
],
"evidence_found": true
},
{
"id": "workspace-boundary-ar",
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
"expected_document": "workspace.md",
"expected_evidence": ".git",
"rank": 1,
"top_paths": [
"workspace.md",
"preview.md"
],
"evidence_found": true
},
{
"id": "web-search-en",
"query": "Which provider searches multiple web sources?",
"expected_document": "search.md",
"expected_evidence": "DuckDuckGo",
"rank": 1,
"top_paths": [
"search.md"
],
"evidence_found": true
},
{
"id": "database-version-en",
"query": "How are old conversation answers migrated?",
"expected_document": "migration.md",
"expected_evidence": "selected_version",
"rank": 1,
"top_paths": [
"migration.md"
],
"evidence_found": true
},
{
"id": "model-routing-ar",
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
"expected_document": "routing.md",
"expected_evidence": "auto_routed",
"rank": 1,
"top_paths": [
"routing.md",
"storage.md",
"images.md"
],
"evidence_found": true
},
{
"id": "api-timeout-ar",
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
"expected_document": "timeout.md",
"expected_evidence": "504",
"rank": 1,
"top_paths": [
"timeout.md",
"routing.md"
],
"evidence_found": true
},
{
"id": "skill-permissions-en",
"query": "How does the server restrict tools for a selected skill?",
"expected_document": "skills.md",
"expected_evidence": "allowed_tools",
"rank": 1,
"top_paths": [
"skills.md",
"migration.md"
],
"evidence_found": true
}
],
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
}
SovereignAI-Starter/evals/results/retrieval_2026-10-02_141132.json
+124
View File
@@ -0,0 +1,124 @@
{
"created_at_utc": "2026-10-02T14:11:32.760819+00:00",
"base_url": "http://127.0.0.1:8000",
"dataset": "evals\\knowledge_retrieval_project.jsonl",
"metrics": {
"cases": 9,
"hit_at_1": 0.5555555555555556,
"hit_at_3": 0.7777777777777778,
"mean_reciprocal_rank": 0.6666666666666666,
"evidence_rate": 0.6666666666666666
},
"results": [
{
"id": "project-path-confinement",
"query": "Does resolving a requested source path guarantee it stays under the selected root?",
"expected_document": "fixture_workspace.py",
"expected_evidence": "relative_to(root)",
"rank": 1,
"top_paths": [
"fixture_workspace.py",
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-preview-stale-file",
"query": "How does applying an approved preview detect that the original file changed after review?",
"expected_document": "fixture_workspace.py",
"expected_evidence": "expected_hash",
"rank": 1,
"top_paths": [
"fixture_workspace.py",
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-index-stale-source",
"query": "How does local search avoid returning indexed content after its source file changes?",
"expected_document": "fixture_knowledge.py",
"expected_evidence": "actual_hash == row[\"content_hash\"]",
"rank": null,
"top_paths": [
"fixture_workspace.py",
"fixture_main.py"
],
"evidence_found": false
},
{
"id": "project-chunk-overlap",
"query": "How much surrounding text is preserved between adjacent retrieval segments?",
"expected_document": "fixture_knowledge.py",
"expected_evidence": "CHUNK_OVERLAP = 120",
"rank": null,
"top_paths": [
"fixture_main.py",
"fixture_pdf.py"
],
"evidence_found": false
},
{
"id": "project-arabic-audio-variants",
"query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
"expected_document": "fixture_knowledge.py",
"expected_evidence": "QUERY_SYNONYMS",
"rank": 1,
"top_paths": [
"fixture_knowledge.py",
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-pdf-page-bound",
"query": "What is the maximum number of PDF pages accepted during local extraction?",
"expected_document": "fixture_pdf.py",
"expected_evidence": "max_pages: int = 30",
"rank": 1,
"top_paths": [
"fixture_pdf.py",
"fixture_main.py",
"fixture_workspace.py"
],
"evidence_found": true
},
{
"id": "project-pdf-empty-scan",
"query": "What happens when a PDF has no extractable text, such as a scanned page?",
"expected_document": "fixture_pdf.py",
"expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
"rank": 2,
"top_paths": [
"fixture_main.py",
"fixture_pdf.py"
],
"evidence_found": false
},
{
"id": "project-skill-tool-filter",
"query": "How is a tool call blocked when it is outside the selected skill permissions?",
"expected_document": "fixture_main.py",
"expected_evidence": "tool_name not in selected_skill.allowed_tools",
"rank": 1,
"top_paths": [
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-agent-index-limits",
"query": "What file count and total size limits apply when indexing local knowledge?",
"expected_document": "fixture_knowledge.py",
"expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
"rank": 2,
"top_paths": [
"fixture_main.py",
"fixture_knowledge.py",
"fixture_workspace.py"
],
"evidence_found": true
}
],
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
}
SovereignAI-Starter/evals/results/retrieval_2026-10-02_141337.json
+124
View File
@@ -0,0 +1,124 @@
{
"created_at_utc": "2026-10-02T14:13:37.698395+00:00",
"base_url": "http://127.0.0.1:8000",
"dataset": "evals\\knowledge_retrieval_project.jsonl",
"metrics": {
"cases": 9,
"hit_at_1": 0.6666666666666666,
"hit_at_3": 1.0,
"mean_reciprocal_rank": 0.8333333333333334,
"evidence_rate": 1.0
},
"results": [
{
"id": "project-path-confinement",
"query": "Does resolving a requested source path guarantee it stays under the selected root?",
"expected_document": "fixture_workspace.py",
"expected_evidence": "relative_to(root)",
"rank": 1,
"top_paths": [
"fixture_workspace.py",
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-preview-stale-file",
"query": "How does applying an approved preview detect that the original file changed after review?",
"expected_document": "fixture_workspace.py",
"expected_evidence": "expected_hash",
"rank": 1,
"top_paths": [
"fixture_workspace.py",
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-index-stale-source",
"query": "Which file hash is compared with the stored content hash before returning a search result?",
"expected_document": "fixture_knowledge.py",
"expected_evidence": "actual_hash == row[\"content_hash\"]",
"rank": 2,
"top_paths": [
"fixture_workspace.py",
"fixture_knowledge.py"
],
"evidence_found": true
},
{
"id": "project-chunk-overlap",
"query": "What are the configured chunk size and overlap for local knowledge indexing?",
"expected_document": "fixture_knowledge.py",
"expected_evidence": "CHUNK_OVERLAP = 120",
"rank": 1,
"top_paths": [
"fixture_knowledge.py",
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-arabic-audio-variants",
"query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
"expected_document": "fixture_knowledge.py",
"expected_evidence": "QUERY_SYNONYMS",
"rank": 1,
"top_paths": [
"fixture_knowledge.py",
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-pdf-page-bound",
"query": "What is the maximum number of PDF pages accepted during local extraction?",
"expected_document": "fixture_pdf.py",
"expected_evidence": "max_pages: int = 30",
"rank": 1,
"top_paths": [
"fixture_pdf.py",
"fixture_main.py",
"fixture_workspace.py"
],
"evidence_found": true
},
{
"id": "project-pdf-empty-scan",
"query": "ما الذي يحدث إذا رفع المستخدم ملف PDF ممسوحًا ولا يحتوي نصًا قابلًا للاستخراج؟",
"expected_document": "fixture_pdf.py",
"expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
"rank": 2,
"top_paths": [
"fixture_main.py",
"fixture_pdf.py"
],
"evidence_found": true
},
{
"id": "project-skill-tool-filter",
"query": "How is a tool call blocked when it is outside the selected skill permissions?",
"expected_document": "fixture_main.py",
"expected_evidence": "tool_name not in selected_skill.allowed_tools",
"rank": 1,
"top_paths": [
"fixture_main.py"
],
"evidence_found": true
},
{
"id": "project-agent-index-limits",
"query": "What file count and total size limits apply when indexing local knowledge?",
"expected_document": "fixture_knowledge.py",
"expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
"rank": 2,
"top_paths": [
"fixture_main.py",
"fixture_knowledge.py",
"fixture_workspace.py"
],
"evidence_found": true
}
],
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
}