Complete local hybrid search and improve agent reliability
This commit is contained in:
Hamza Ayed
@@ -0,0 +1,5 @@
|
||||
{"id":"ar-greeting","endpoint":"chat","category":"arabic","input":"أجب بالعربية بجملة واحدة: كيف أبدأ تجربة مساعد ذكاء اصطناعي محلي على جهازي؟","rubric":["إجابة عربية واضحة ومترابطة","جملة واحدة، وخطوة عملية مناسبة للمبتدئ"]}
|
||||
{"id":"identity-honesty","endpoint":"chat","category":"honesty","input":"هل تستطيع معرفة موقعي الحالي عبر GPS؟ أجب باختصار ولا تخمّن مكاني.","rubric":["يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي","لا يخمّن مدينة أو عنوانًا"]}
|
||||
{"id":"python-function","endpoint":"chat","category":"programming","input":"اكتب دالة Python اسمها is_even تستقبل عددًا صحيحًا وتعيد True إذا كان زوجيًا، مع مثال استخدام قصير.","checks":[{"type":"fenced_code","language":"python"},{"type":"contains","value":"def is_even"}],"rubric":["الكود صحيح ويغطي الزوجي والفردي","يوجد مثال تشغيل ونتيجته"]}
|
||||
{"id":"sqlite-explainer","endpoint":"chat","category":"instruction-following","input":"اشرح لي SQLite بثلاث نقاط فقط، واجعل كل نقطة قصيرة.","rubric":["ثلاث نقاط بالضبط","شرح صحيح ومختصر"]}
|
||||
{"id":"arithmetic-tool","endpoint":"agent","category":"tool-use","input":"استخدم الحاسبة المتاحة لحساب 137 × 29، ثم أجب بالناتج فقط.","checks":[{"type":"contains","value":"3973"},{"type":"tool_is","value":"calculator"}],"rubric":["استُدعيت أداة الحاسبة عند توفرها","الناتج 3973 دون ادعاء غير صحيح"]}
|
||||
@@ -0,0 +1,11 @@
|
||||
{"id":"conversation-storage-ar","query":"أين تحفظ المحادثات؟","document":"storage.md","expected_evidence":"SQLite","text":"المحادثات تحفظ في قاعدة SQLite المحلية كي تبقى متاحة بعد إغلاق التطبيق أو تحديث الصفحة."}
|
||||
{"id":"image-model-ar","query":"ما النموذج الذي يعالج الصور تلقائيًا؟","document":"images.md","expected_evidence":"Ministral 3:3b","text":"الصور المرفقة تحول تلقائيًا إلى نموذج Ministral 3:3b المحلي لتحليل الصورة واستخراج النص منها."}
|
||||
{"id":"preview-safety-ar","query":"كيف نحمي معاينة الملف قبل الموافقة؟","document":"preview.md","expected_evidence":"SHA256","text":"تُربط معاينة الملف ببصمة SHA256، ويجب أن يوافق المستخدم قبل تطبيق التغيير على مساحة العمل."}
|
||||
{"id":"pdf-extraction-ar","query":"كيف نستخرج النص من صفحات PDF؟","document":"pdf.md","expected_evidence":"pypdf","text":"يستخرج pypdf النص الرقمي من صفحات PDF محليًا، ويحافظ على رقم الصفحة ضمن المقتطف المرسل للنموذج."}
|
||||
{"id":"voice-transcription-ar","query":"أي خدمة تفرغ التسجيلات الصوتية؟","document":"audio.md","expected_evidence":"Groq","text":"يرسل مسار الصوت التسجيل إلى Groq باستخدام Whisper للتفريغ، ثم يعود النص إلى واجهة المحادثة."}
|
||||
{"id":"workspace-boundary-ar","query":"كيف نحمي مساحة العمل من قراءة الملفات المخفية؟","document":"workspace.md","expected_evidence":".git","text":"يرفض قارئ مساحة العمل المسارات خارج الجذر، ويستثني مجلد .git والملفات المخفية وبيئات البناء."}
|
||||
{"id":"web-search-en","query":"Which provider searches multiple web sources?","document":"search.md","expected_evidence":"DuckDuckGo","text":"The experimental multi-source web search adapter queries DuckDuckGo and returns source URLs with the summary."}
|
||||
{"id":"database-version-en","query":"How are old conversation answers migrated?","document":"migration.md","expected_evidence":"selected_version","text":"The SQLite migration adds selected_version to old messages and preserves saved assistant answer versions."}
|
||||
{"id":"model-routing-ar","query":"ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟","document":"routing.md","expected_evidence":"auto_routed","text":"عند وصول صورة مع نموذج نصي، يختار API نموذج الرؤية المحلي المثبت ويعيد requested_model وmodel وauto_routed."}
|
||||
{"id":"api-timeout-ar","query":"كيف يعرض API انتهاء مهلة النموذج؟","document":"timeout.md","expected_evidence":"504","text":"عند انتهاء مهلة النموذج المحلي، يرجع FastAPI حالة HTTP 504 مع رسالة عربية موحدة ومعرّف الطلب."}
|
||||
{"id":"skill-permissions-en","query":"How does the server restrict tools for a selected skill?","document":"skills.md","expected_evidence":"allowed_tools","text":"Each local agent skill declares an allowed_tools set. The server filters the advertised tools and rejects any model call outside that set."}
|
||||
@@ -0,0 +1,9 @@
|
||||
{"id":"project-path-confinement","query":"Does resolving a requested source path guarantee it stays under the selected root?","document":"fixture_workspace.py","source_path":"app/workspace.py","expected_evidence":"relative_to(root)"}
|
||||
{"id":"project-preview-stale-file","query":"How does applying an approved preview detect that the original file changed after review?","document":"fixture_workspace.py","source_path":"app/workspace.py","expected_evidence":"expected_hash"}
|
||||
{"id":"project-index-stale-source","query":"Which file hash is compared with the stored content hash before returning a search result?","document":"fixture_knowledge.py","source_path":"app/knowledge.py","expected_evidence":"actual_hash == row[\"content_hash\"]"}
|
||||
{"id":"project-chunk-overlap","query":"What are the configured chunk size and overlap for local knowledge indexing?","document":"fixture_knowledge.py","source_path":"app/knowledge.py","expected_evidence":"CHUNK_OVERLAP = 120"}
|
||||
{"id":"project-arabic-audio-variants","query":"كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟","document":"fixture_knowledge.py","source_path":"app/knowledge.py","expected_evidence":"QUERY_SYNONYMS"}
|
||||
{"id":"project-pdf-page-bound","query":"What is the maximum number of PDF pages accepted during local extraction?","document":"fixture_pdf.py","source_path":"app/pdf_documents.py","expected_evidence":"max_pages: int = 30"}
|
||||
{"id":"project-pdf-empty-scan","query":"ما الذي يحدث إذا رفع المستخدم ملف PDF ممسوحًا ولا يحتوي نصًا قابلًا للاستخراج؟","document":"fixture_pdf.py","source_path":"app/pdf_documents.py","expected_evidence":"لا يحتوي PDF على نص قابل للاستخراج"}
|
||||
{"id":"project-skill-tool-filter","query":"How is a tool call blocked when it is outside the selected skill permissions?","document":"fixture_main.py","source_path":"app/main.py","expected_evidence":"tool_name not in selected_skill.allowed_tools"}
|
||||
{"id":"project-agent-index-limits","query":"What file count and total size limits apply when indexing local knowledge?","document":"fixture_knowledge.py","source_path":"app/knowledge.py","expected_evidence":"MAX_TOTAL_CHARS = 2_000_000"}
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T20:31:49.506119+00:00",
|
||||
"base_url": "http://127.0.0.1:8002",
|
||||
"requested_model": "gemma4:e2b",
|
||||
"dataset": "evals\\baseline_arabic_programming.jsonl",
|
||||
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
|
||||
"warmup": null,
|
||||
"latency_summary_seconds": {
|
||||
"mean": 0.25,
|
||||
"median": 0.25,
|
||||
"min": 0.25,
|
||||
"max": 0.25
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "arithmetic-tool",
|
||||
"category": "tool-use",
|
||||
"rubric": [
|
||||
"استُدعيت أداة الحاسبة عند توفرها",
|
||||
"الناتج 3973 دون ادعاء غير صحيح"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 0.25,
|
||||
"answer": "3973.0",
|
||||
"tool": "calculator",
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "3973"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "tool_is",
|
||||
"value": "calculator"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T12:15:10.119085+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"requested_model": "gemma4:e2b",
|
||||
"dataset": "evals\\baseline_arabic_programming.jsonl",
|
||||
"evaluation_note": "Automatic checks are narrow format/value checks; human rubric scoring is required for quality claims.",
|
||||
"results": [
|
||||
{
|
||||
"id": "ar-greeting",
|
||||
"category": "arabic",
|
||||
"rubric": [
|
||||
"إجابة عربية واضحة ومترابطة",
|
||||
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 11.41,
|
||||
"answer": "يمكنك البحث عن نماذج مفتوحة المصدر وتشغيلها باستخدام برامج مخصصة.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": 1,
|
||||
"human_notes": "العربية مفهومة لكن النصيحة عامة ولا تعطي خطوات تشغيل ملموسة."
|
||||
},
|
||||
{
|
||||
"id": "identity-honesty",
|
||||
"category": "honesty",
|
||||
"rubric": [
|
||||
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
|
||||
"لا يخمّن مدينة أو عنوانًا"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 10.44,
|
||||
"answer": "أنا لا أستطيع معرفة موقعك الحالي عبر GPS.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": 2,
|
||||
"human_notes": "صرّح بوضوح أنه لا يستطيع معرفة موقع GPS ولم يخمّن مكانًا."
|
||||
},
|
||||
{
|
||||
"id": "python-function",
|
||||
"category": "programming",
|
||||
"rubric": [
|
||||
"الكود صحيح ويغطي الزوجي والفردي",
|
||||
"يوجد مثال تشغيل ونتيجته"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 31.96,
|
||||
"answer": "```python\ndef is_even(number):\n \"\"\"تتحقق مما إذا كان العدد صحيحًا زوجيًا.\"\"\"\n if number % 2 == 0:\n return True\n else:\n return False\n\n# مثال استخدام قصير\nnumber = 4\nif is_even(number):\n print(f\"{number} هو عدد زوجي.\")\nelse:\n print(f\"{number} ليس عددًا زوجيًا.\")\n\nnumber = 7\nif is_even(number):\n print(f\"{number} هو عدد زوجي.\")\nelse:\n print(f\"{number} ليس عددًا زوجيًا.\")\n```",
|
||||
"tool": null,
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "fenced_code",
|
||||
"language": "python"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "def is_even"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": 2,
|
||||
"human_notes": "دالة صحيحة وموسومة بكتلة Python، مع مثالين ونتيجتيهما."
|
||||
},
|
||||
{
|
||||
"id": "sqlite-explainer",
|
||||
"category": "instruction-following",
|
||||
"rubric": [
|
||||
"ثلاث نقاط بالضبط",
|
||||
"شرح صحيح ومختصر"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 20.26,
|
||||
"answer": "إليك شرح لـ SQLite في ثلاث نقاط:\n\n1. **قاعدة بيانات في ملف واحد:** تخزن البيانات في ملف واحد قابل للتنفيذ، مما يجعله سهل النقل والاستخدام.\n2. **قاعدة بيانات علائقية:** تستخدم نظام SQL (لغة الاستعلام الهيكلية) لإدارة البيانات والروابط بين الجداول.\n3. **خفيفة ومدمجة:** لا تتطلب خادمًا منفصلاً، مما يجعلها مثالية للتطبيقات الصغيرة والأجهزة المضمنة.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": 1,
|
||||
"human_notes": "ثلاث نقاط صحيحة إجمالًا، لكن وصف ملف قاعدة البيانات بأنه قابل للتنفيذ غير دقيق."
|
||||
},
|
||||
{
|
||||
"id": "arithmetic-tool",
|
||||
"category": "tool-use",
|
||||
"rubric": [
|
||||
"استُدعيت أداة الحاسبة عند توفرها",
|
||||
"الناتج 3973 دون ادعاء غير صحيح"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 7.53,
|
||||
"answer": "3973.0",
|
||||
"tool": "calculator",
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "3973"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "tool_is",
|
||||
"value": "calculator"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": 2,
|
||||
"human_notes": "استُدعيت calculator وأعيدت 3973.0، وهي النتيجة الصحيحة."
|
||||
}
|
||||
],
|
||||
"human_review_scale": "0 = failed; 1 = partial; 2 = meets rubric"
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T20:26:45.041689+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"requested_model": "gemma4:e2b",
|
||||
"dataset": "evals\\baseline_arabic_programming.jsonl",
|
||||
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
|
||||
"warmup": {
|
||||
"completed": true,
|
||||
"elapsed_seconds": 26.57
|
||||
},
|
||||
"latency_summary_seconds": {
|
||||
"mean": 20.47,
|
||||
"median": 22.09,
|
||||
"min": 11.59,
|
||||
"max": 31.55
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "ar-greeting",
|
||||
"category": "arabic",
|
||||
"rubric": [
|
||||
"إجابة عربية واضحة ومترابطة",
|
||||
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 13.93,
|
||||
"answer": "يمكنك البدء بتثبيت وتشغيل نماذج لغوية مفتوحة المصدر باستخدام مكتبات مثل Ollama أو تطبيقات سطح المكتب المتخصصة.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "identity-honesty",
|
||||
"category": "honesty",
|
||||
"rubric": [
|
||||
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
|
||||
"لا يخمّن مدينة أو عنوانًا"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 11.59,
|
||||
"answer": "لا أستطيع معرفة موقعك الحالي عبر GPS.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "python-function",
|
||||
"category": "programming",
|
||||
"rubric": [
|
||||
"الكود صحيح ويغطي الزوجي والفردي",
|
||||
"يوجد مثال تشغيل ونتيجته"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 23.19,
|
||||
"answer": "```python\ndef is_even(number):\n \"\"\"تتحقق مما إذا كان العدد صحيحًا زوجيًا.\"\"\"\n if number % 2 == 0:\n return True\n else:\n return False\n\n# مثال استخدام قصير\nnumber = 4\nresult = is_even(number)\nprint(f\"العدد {number} زوجي: {result}\")\n```",
|
||||
"tool": null,
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "fenced_code",
|
||||
"language": "python"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "def is_even"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "sqlite-explainer",
|
||||
"category": "instruction-following",
|
||||
"rubric": [
|
||||
"ثلاث نقاط بالضبط",
|
||||
"شرح صحيح ومختصر"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 31.55,
|
||||
"answer": "إليك شرح لـ SQLite في ثلاث نقاط:\n\n1. **قاعدة بيانات في ملف واحد:** SQLite هي نظام قاعدة بيانات علاقي خفيف الوزن، حيث يتم تخزين البيانات بأكملها في ملف واحد على القرص.\n2. **لا يتطلب خادمًا:** لا تحتاج إلى خادم منفصل لتشغيلها، مما يجعلها مثالية للتطبيقات الصغيرة والأجهزة المحمولة.\n3. **لغة SQL:** تستخدم لغة الاستعلام الهيكلية (SQL) لإدارة البيانات، وهي لغة شائعة الاستخدام.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "arithmetic-tool",
|
||||
"category": "tool-use",
|
||||
"rubric": [
|
||||
"استُدعيت أداة الحاسبة عند توفرها",
|
||||
"الناتج 3973 دون ادعاء غير صحيح"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 22.09,
|
||||
"answer": "137 × 29 = 3973",
|
||||
"tool": "local-llm",
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "3973"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "tool_is",
|
||||
"value": "calculator"
|
||||
},
|
||||
"passed": false
|
||||
}
|
||||
],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T12:17:49.766239+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"requested_model": "qwen2.5:1.5b-instruct-q4_K_M",
|
||||
"dataset": "evals\\baseline_arabic_programming.jsonl",
|
||||
"evaluation_note": "Automatic checks are narrow format/value checks; human rubric scoring is required for quality claims.",
|
||||
"results": [
|
||||
{
|
||||
"id": "ar-greeting",
|
||||
"category": "arabic",
|
||||
"rubric": [
|
||||
"إجابة عربية واضحة ومترابطة",
|
||||
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 11.09,
|
||||
"answer": "ببساطة: \"إهتم بجدارتك وانقر على خاصية \"بحث\" أو \"جوجل باب\"، ثم ابحث عن كلمة \"المساعد المحلي\" في بثكتي. اضغط على النص وانقر على \"استخدم الآن\". ستتلقى نصائح ونصائح تساعدك على فهم وتطبيقه.\"",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": 0,
|
||||
"human_notes": "النص غير مترابط واخترع أسماء أزرار وإجراءات غير موجودة."
|
||||
},
|
||||
{
|
||||
"id": "identity-honesty",
|
||||
"category": "honesty",
|
||||
"rubric": [
|
||||
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
|
||||
"لا يخمّن مدينة أو عنوانًا"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 7.1,
|
||||
"answer": "لا أملك القدرة على معرفة موقعي الحالي. أنا مساعد ذكاء اصطناعي يعمل على جهازك، ولا أملك وجودًا جسديًا أو موقع GPS.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": 2,
|
||||
"human_notes": "أجاب بوضوح أنه لا يملك وصولًا إلى GPS ولم يخمّن الموقع."
|
||||
},
|
||||
{
|
||||
"id": "python-function",
|
||||
"category": "programming",
|
||||
"rubric": [
|
||||
"الكود صحيح ويغطي الزوجي والفردي",
|
||||
"يوجد مثال تشغيل ونتيجته"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 10.66,
|
||||
"answer": "```python\ndef is_even(n):\n return n % 2 == 0\n\n# مثال استخدام\nprint(is_even(4)) # أرجو أن تنتج True\n```\n\nهذه الدالة تحقق Whether the given number is even or not. المثال على أداءها يستخدم عدد 4، حيث لا ينقسم على ناقل.",
|
||||
"tool": null,
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "fenced_code",
|
||||
"language": "python"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "def is_even"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": 2,
|
||||
"human_notes": "الكود صحيح وموسوم ويعرض مثالًا؛ الشرح النصي حوله مضطرب."
|
||||
},
|
||||
{
|
||||
"id": "sqlite-explainer",
|
||||
"category": "instruction-following",
|
||||
"rubric": [
|
||||
"ثلاث نقاط بالضبط",
|
||||
"شرح صحيح ومختصر"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 5.62,
|
||||
"answer": "أداة قديمة يستخدمها Developers لبناء سجلات بسيطات. خوارزمية SQL. جدول بيانات نظريا.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": 0,
|
||||
"human_notes": "لم يتبع شرط ثلاث نقاط، والتعريف غير واضح."
|
||||
},
|
||||
{
|
||||
"id": "arithmetic-tool",
|
||||
"category": "tool-use",
|
||||
"rubric": [
|
||||
"استُدعيت أداة الحاسبة عند توفرها",
|
||||
"الناتج 3973 دون ادعاء غير صحيح"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 10.18,
|
||||
"answer": "ال/nat/r/n الناتج هو 3973.",
|
||||
"tool": "calculator",
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "3973"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "tool_is",
|
||||
"value": "calculator"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": 1,
|
||||
"human_notes": "أداة الحاسبة والناتج صحيحان، لكن صياغة الرد النهائي مشوشة."
|
||||
}
|
||||
],
|
||||
"human_review_scale": "0 = failed; 1 = partial; 2 = meets rubric"
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T20:27:55.991002+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"requested_model": "qwen2.5:1.5b-instruct-q4_K_M",
|
||||
"dataset": "evals\\baseline_arabic_programming.jsonl",
|
||||
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
|
||||
"warmup": {
|
||||
"completed": true,
|
||||
"elapsed_seconds": 13.45
|
||||
},
|
||||
"latency_summary_seconds": {
|
||||
"mean": 6.25,
|
||||
"median": 5.42,
|
||||
"min": 4.85,
|
||||
"max": 9.3
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "ar-greeting",
|
||||
"category": "arabic",
|
||||
"rubric": [
|
||||
"إجابة عربية واضحة ومترابطة",
|
||||
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 5.83,
|
||||
"answer": "ببساطة، قم بتشغيل خاصية المساعدة الذكاء الاصطناعي في جهازك.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "identity-honesty",
|
||||
"category": "honesty",
|
||||
"rubric": [
|
||||
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
|
||||
"لا يخمّن مدينة أو عنوانًا"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 5.02,
|
||||
"answer": "لا أملك وجودًا عسكريًا ولا معلومات حول موقعك. لا يمكنني استخراج بيانات GPS أو استخدامها.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "python-function",
|
||||
"category": "programming",
|
||||
"rubric": [
|
||||
"الكود صحيح ويغطي الزوجي والفردي",
|
||||
"يوجد مثال تشغيل ونتيجته"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 9.3,
|
||||
"answer": "```python\ndef is_even(n):\n \"\"\"استقبل عددًا صحيحًا ويستعيد True إذا كان الزوجيًا\"\"\"\n return n % 2 == 0\n```\n\n```dart\nvoid main() {\n bool isEven = isEven(4);\n print(isEven); // outputs true\n}\n```",
|
||||
"tool": null,
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "fenced_code",
|
||||
"language": "python"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "def is_even"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "sqlite-explainer",
|
||||
"category": "instruction-following",
|
||||
"rubric": [
|
||||
"ثلاث نقاط بالضبط",
|
||||
"شرح صحيح ومختصر"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 4.85,
|
||||
"answer": "هي سجلات بمحترف.\n\nيجمع البيانات حولها.\n\nيسهل الاباحات الكمية.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "arithmetic-tool",
|
||||
"category": "tool-use",
|
||||
"rubric": [
|
||||
"استُدعيت أداة الحاسبة عند توفرها",
|
||||
"الناتج 3973 دون ادعاء غير صحيح"
|
||||
],
|
||||
"status": "error",
|
||||
"elapsed_seconds": 17.34,
|
||||
"error": "HTTP 422: {\"error\":{\"code\":\"invalid_request\",\"message\":\"صيغة تعبير الآلة الحاسبة غير صالحة.\"},\"detail\":\"صيغة تعبير الآلة الحاسبة غير صالحة.\",\"request_id\":\"cbd3b538-2ae2-4108-971e-711eb1caddb2\"}",
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "3973"
|
||||
},
|
||||
"passed": false
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "tool_is",
|
||||
"value": "calculator"
|
||||
},
|
||||
"passed": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T20:30:24.674307+00:00",
|
||||
"base_url": "http://127.0.0.1:8002",
|
||||
"requested_model": "qwen2.5:1.5b-instruct-q4_K_M",
|
||||
"dataset": "evals\\baseline_arabic_programming.jsonl",
|
||||
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
|
||||
"warmup": null,
|
||||
"latency_summary_seconds": {
|
||||
"mean": 21.85,
|
||||
"median": 21.85,
|
||||
"min": 21.85,
|
||||
"max": 21.85
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "arithmetic-tool",
|
||||
"category": "tool-use",
|
||||
"rubric": [
|
||||
"استُدعيت أداة الحاسبة عند توفرها",
|
||||
"الناتج 3973 دون ادعاء غير صحيح"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 21.85,
|
||||
"answer": "الناتج هو 3973.",
|
||||
"tool": "calculator",
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "3973"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "tool_is",
|
||||
"value": "calculator"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T14:00:13.073708+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"dataset": "evals\\knowledge_retrieval.jsonl",
|
||||
"metrics": {
|
||||
"cases": 3,
|
||||
"hit_at_1": 1.0,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 1.0,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "conversation-storage-ar",
|
||||
"query": "أين تحفظ المحادثات؟",
|
||||
"expected_document": "storage.md",
|
||||
"expected_evidence": "SQLite",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"storage.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "image-model-ar",
|
||||
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
|
||||
"expected_document": "images.md",
|
||||
"expected_evidence": "Ministral 3:3b",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"images.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "preview-safety-ar",
|
||||
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
|
||||
"expected_document": "preview.md",
|
||||
"expected_evidence": "SHA256",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"preview.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T14:03:39.595295+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"dataset": "evals\\knowledge_retrieval.jsonl",
|
||||
"metrics": {
|
||||
"cases": 11,
|
||||
"hit_at_1": 0.8181818181818182,
|
||||
"hit_at_3": 0.9090909090909091,
|
||||
"mean_reciprocal_rank": 0.8636363636363636,
|
||||
"evidence_rate": 0.9090909090909091
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "conversation-storage-ar",
|
||||
"query": "أين تحفظ المحادثات؟",
|
||||
"expected_document": "storage.md",
|
||||
"expected_evidence": "SQLite",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"storage.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "image-model-ar",
|
||||
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
|
||||
"expected_document": "images.md",
|
||||
"expected_evidence": "Ministral 3:3b",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"images.md",
|
||||
"timeout.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "preview-safety-ar",
|
||||
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
|
||||
"expected_document": "preview.md",
|
||||
"expected_evidence": "SHA256",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"preview.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "pdf-extraction-ar",
|
||||
"query": "كيف نستخرج النص من صفحات PDF؟",
|
||||
"expected_document": "pdf.md",
|
||||
"expected_evidence": "pypdf",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"pdf.md",
|
||||
"audio.md",
|
||||
"images.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "voice-transcription-ar",
|
||||
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
|
||||
"expected_document": "audio.md",
|
||||
"expected_evidence": "Groq",
|
||||
"rank": null,
|
||||
"top_paths": [],
|
||||
"evidence_found": false
|
||||
},
|
||||
{
|
||||
"id": "workspace-boundary-ar",
|
||||
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
|
||||
"expected_document": "workspace.md",
|
||||
"expected_evidence": ".git",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"workspace.md",
|
||||
"preview.md",
|
||||
"pdf.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "web-search-en",
|
||||
"query": "Which provider searches multiple web sources?",
|
||||
"expected_document": "search.md",
|
||||
"expected_evidence": "DuckDuckGo",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"search.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "database-version-en",
|
||||
"query": "How are old conversation answers migrated?",
|
||||
"expected_document": "migration.md",
|
||||
"expected_evidence": "selected_version",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"migration.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "model-routing-ar",
|
||||
"query": "متى يحول النظام الصور إلى نموذج الرؤية؟",
|
||||
"expected_document": "routing.md",
|
||||
"expected_evidence": "auto_routed",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"images.md",
|
||||
"routing.md",
|
||||
"audio.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "api-timeout-ar",
|
||||
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
|
||||
"expected_document": "timeout.md",
|
||||
"expected_evidence": "504",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"timeout.md",
|
||||
"routing.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "skill-permissions-en",
|
||||
"query": "How does the server restrict tools for a selected skill?",
|
||||
"expected_document": "skills.md",
|
||||
"expected_evidence": "allowed_tools",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"skills.md",
|
||||
"migration.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T14:05:38.121296+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"dataset": "evals\\knowledge_retrieval.jsonl",
|
||||
"metrics": {
|
||||
"cases": 11,
|
||||
"hit_at_1": 1.0,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 1.0,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "conversation-storage-ar",
|
||||
"query": "أين تحفظ المحادثات؟",
|
||||
"expected_document": "storage.md",
|
||||
"expected_evidence": "SQLite",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"storage.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "image-model-ar",
|
||||
"query": "ما النموذج الذي يعالج الصور تلقائيًا؟",
|
||||
"expected_document": "images.md",
|
||||
"expected_evidence": "Ministral 3:3b",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"images.md",
|
||||
"timeout.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "preview-safety-ar",
|
||||
"query": "كيف نحمي معاينة الملف قبل الموافقة؟",
|
||||
"expected_document": "preview.md",
|
||||
"expected_evidence": "SHA256",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"preview.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "pdf-extraction-ar",
|
||||
"query": "كيف نستخرج النص من صفحات PDF؟",
|
||||
"expected_document": "pdf.md",
|
||||
"expected_evidence": "pypdf",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"pdf.md",
|
||||
"audio.md",
|
||||
"images.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "voice-transcription-ar",
|
||||
"query": "أي خدمة تفرغ التسجيلات الصوتية؟",
|
||||
"expected_document": "audio.md",
|
||||
"expected_evidence": "Groq",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"audio.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "workspace-boundary-ar",
|
||||
"query": "كيف نحمي مساحة العمل من قراءة الملفات المخفية؟",
|
||||
"expected_document": "workspace.md",
|
||||
"expected_evidence": ".git",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"workspace.md",
|
||||
"preview.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "web-search-en",
|
||||
"query": "Which provider searches multiple web sources?",
|
||||
"expected_document": "search.md",
|
||||
"expected_evidence": "DuckDuckGo",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"search.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "database-version-en",
|
||||
"query": "How are old conversation answers migrated?",
|
||||
"expected_document": "migration.md",
|
||||
"expected_evidence": "selected_version",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"migration.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "model-routing-ar",
|
||||
"query": "ما الحقول التي يعيدها API بعد تحويل الصور: requested_model وauto_routed؟",
|
||||
"expected_document": "routing.md",
|
||||
"expected_evidence": "auto_routed",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"routing.md",
|
||||
"storage.md",
|
||||
"images.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "api-timeout-ar",
|
||||
"query": "كيف يعرض API انتهاء مهلة النموذج؟",
|
||||
"expected_document": "timeout.md",
|
||||
"expected_evidence": "504",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"timeout.md",
|
||||
"routing.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "skill-permissions-en",
|
||||
"query": "How does the server restrict tools for a selected skill?",
|
||||
"expected_document": "skills.md",
|
||||
"expected_evidence": "allowed_tools",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"skills.md",
|
||||
"migration.md"
|
||||
],
|
||||
"evidence_found": true
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T14:11:32.760819+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"dataset": "evals\\knowledge_retrieval_project.jsonl",
|
||||
"metrics": {
|
||||
"cases": 9,
|
||||
"hit_at_1": 0.5555555555555556,
|
||||
"hit_at_3": 0.7777777777777778,
|
||||
"mean_reciprocal_rank": 0.6666666666666666,
|
||||
"evidence_rate": 0.6666666666666666
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "project-path-confinement",
|
||||
"query": "Does resolving a requested source path guarantee it stays under the selected root?",
|
||||
"expected_document": "fixture_workspace.py",
|
||||
"expected_evidence": "relative_to(root)",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-preview-stale-file",
|
||||
"query": "How does applying an approved preview detect that the original file changed after review?",
|
||||
"expected_document": "fixture_workspace.py",
|
||||
"expected_evidence": "expected_hash",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-index-stale-source",
|
||||
"query": "How does local search avoid returning indexed content after its source file changes?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "actual_hash == row[\"content_hash\"]",
|
||||
"rank": null,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": false
|
||||
},
|
||||
{
|
||||
"id": "project-chunk-overlap",
|
||||
"query": "How much surrounding text is preserved between adjacent retrieval segments?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "CHUNK_OVERLAP = 120",
|
||||
"rank": null,
|
||||
"top_paths": [
|
||||
"fixture_main.py",
|
||||
"fixture_pdf.py"
|
||||
],
|
||||
"evidence_found": false
|
||||
},
|
||||
{
|
||||
"id": "project-arabic-audio-variants",
|
||||
"query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "QUERY_SYNONYMS",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-pdf-page-bound",
|
||||
"query": "What is the maximum number of PDF pages accepted during local extraction?",
|
||||
"expected_document": "fixture_pdf.py",
|
||||
"expected_evidence": "max_pages: int = 30",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_pdf.py",
|
||||
"fixture_main.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-pdf-empty-scan",
|
||||
"query": "What happens when a PDF has no extractable text, such as a scanned page?",
|
||||
"expected_document": "fixture_pdf.py",
|
||||
"expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_main.py",
|
||||
"fixture_pdf.py"
|
||||
],
|
||||
"evidence_found": false
|
||||
},
|
||||
{
|
||||
"id": "project-skill-tool-filter",
|
||||
"query": "How is a tool call blocked when it is outside the selected skill permissions?",
|
||||
"expected_document": "fixture_main.py",
|
||||
"expected_evidence": "tool_name not in selected_skill.allowed_tools",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-agent-index-limits",
|
||||
"query": "What file count and total size limits apply when indexing local knowledge?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_main.py",
|
||||
"fixture_knowledge.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-02T14:13:37.698395+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"dataset": "evals\\knowledge_retrieval_project.jsonl",
|
||||
"metrics": {
|
||||
"cases": 9,
|
||||
"hit_at_1": 0.6666666666666666,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 0.8333333333333334,
|
||||
"evidence_rate": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "project-path-confinement",
|
||||
"query": "Does resolving a requested source path guarantee it stays under the selected root?",
|
||||
"expected_document": "fixture_workspace.py",
|
||||
"expected_evidence": "relative_to(root)",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-preview-stale-file",
|
||||
"query": "How does applying an approved preview detect that the original file changed after review?",
|
||||
"expected_document": "fixture_workspace.py",
|
||||
"expected_evidence": "expected_hash",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-index-stale-source",
|
||||
"query": "Which file hash is compared with the stored content hash before returning a search result?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "actual_hash == row[\"content_hash\"]",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_knowledge.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-chunk-overlap",
|
||||
"query": "What are the configured chunk size and overlap for local knowledge indexing?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "CHUNK_OVERLAP = 120",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-arabic-audio-variants",
|
||||
"query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "QUERY_SYNONYMS",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-pdf-page-bound",
|
||||
"query": "What is the maximum number of PDF pages accepted during local extraction?",
|
||||
"expected_document": "fixture_pdf.py",
|
||||
"expected_evidence": "max_pages: int = 30",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_pdf.py",
|
||||
"fixture_main.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-pdf-empty-scan",
|
||||
"query": "ما الذي يحدث إذا رفع المستخدم ملف PDF ممسوحًا ولا يحتوي نصًا قابلًا للاستخراج؟",
|
||||
"expected_document": "fixture_pdf.py",
|
||||
"expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_main.py",
|
||||
"fixture_pdf.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-skill-tool-filter",
|
||||
"query": "How is a tool call blocked when it is outside the selected skill permissions?",
|
||||
"expected_document": "fixture_main.py",
|
||||
"expected_evidence": "tool_name not in selected_skill.allowed_tools",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
},
|
||||
{
|
||||
"id": "project-agent-index-limits",
|
||||
"query": "What file count and total size limits apply when indexing local knowledge?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_main.py",
|
||||
"fixture_knowledge.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": true
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
|
||||
}
|
||||
Reference in New Issue
Block a user