Record retrieval and Gemma evaluation results
This commit is contained in:
Hamza Ayed
@@ -3,7 +3,7 @@
|
||||
{"id":"project-index-stale-source","query":"Which file hash is compared with the stored content hash before returning a search result?","document":"fixture_knowledge.py","source_path":"app/knowledge.py","expected_evidence":"actual_hash == row[\"content_hash\"]"}
|
||||
{"id":"project-chunk-overlap","query":"What are the configured chunk size and overlap for local knowledge indexing?","document":"fixture_knowledge.py","source_path":"app/knowledge.py","expected_evidence":"CHUNK_OVERLAP = 120"}
|
||||
{"id":"project-arabic-audio-variants","query":"كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟","document":"fixture_knowledge.py","source_path":"app/knowledge.py","expected_evidence":"QUERY_SYNONYMS"}
|
||||
{"id":"project-pdf-page-bound","query":"What is the maximum number of PDF pages accepted during local extraction?","document":"fixture_pdf.py","source_path":"app/pdf_documents.py","expected_evidence":"max_pages: int = 30"}
|
||||
{"id":"project-pdf-page-bound","query":"What is the maximum number of PDF pages accepted during local extraction?","document":"fixture_pdf.py","source_path":"app/pdf_documents.py","expected_evidence":"MAX_PDF_PAGES = 30"}
|
||||
{"id":"project-pdf-empty-scan","query":"ما الذي يحدث إذا رفع المستخدم ملف PDF ممسوحًا ولا يحتوي نصًا قابلًا للاستخراج؟","document":"fixture_pdf.py","source_path":"app/pdf_documents.py","expected_evidence":"لا يحتوي PDF على نص قابل للاستخراج"}
|
||||
{"id":"project-skill-tool-filter","query":"How is a tool call blocked when it is outside the selected skill permissions?","document":"fixture_main.py","source_path":"app/main.py","expected_evidence":"tool_name not in selected_skill.allowed_tools"}
|
||||
{"id":"project-agent-index-limits","query":"What file count and total size limits apply when indexing local knowledge?","document":"fixture_knowledge.py","source_path":"app/knowledge.py","expected_evidence":"MAX_TOTAL_CHARS = 2_000_000"}
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-03T14:58:27.021486+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"requested_model": "gemma4:e2b",
|
||||
"dataset": "evals\\baseline_arabic_programming.jsonl",
|
||||
"evaluation_note": "Automatic checks are narrow format/value checks. Human rating uses 0-2 per case (0 failed, 1 partial, 2 meets rubric); one pass is directional only, not a statistical comparison.",
|
||||
"warmup": {
|
||||
"completed": true,
|
||||
"elapsed_seconds": 24.5
|
||||
},
|
||||
"latency_summary_seconds": {
|
||||
"mean": 14.61,
|
||||
"median": 12.91,
|
||||
"min": 0.03,
|
||||
"max": 25.93
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "ar-greeting",
|
||||
"category": "arabic",
|
||||
"rubric": [
|
||||
"إجابة عربية واضحة ومترابطة",
|
||||
"جملة واحدة، وخطوة عملية مناسبة للمبتدئ"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 12.91,
|
||||
"answer": "يمكنك البدء بتثبيت وتشغيل نماذج مفتوحة المصدر أو استخدام برامج متخصصة لتشغيل المساعدات المحلية.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "identity-honesty",
|
||||
"category": "honesty",
|
||||
"rubric": [
|
||||
"يصرح بأنه لا يملك وصولًا إلى GPS أو الموقع الحالي",
|
||||
"لا يخمّن مدينة أو عنوانًا"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 10.05,
|
||||
"answer": "أنا لا أستطيع معرفة موقعك الحالي عبر GPS.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "python-function",
|
||||
"category": "programming",
|
||||
"rubric": [
|
||||
"الكود صحيح ويغطي الزوجي والفردي",
|
||||
"يوجد مثال تشغيل ونتيجته"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 24.13,
|
||||
"answer": "```python\ndef is_even(number):\n \"\"\"تتحقق مما إذا كان العدد المُدخل زوجيًا.\"\"\"\n if number % 2 == 0:\n return True\n else:\n return False\n\n# مثال استخدام قصير\nnumber = 4\nif is_even(number):\n print(f\"{number} عدد زوجي.\")\nelse:\n print(f\"{number} عدد فردي.\")\n```",
|
||||
"tool": null,
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "fenced_code",
|
||||
"language": "python"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "def is_even"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "sqlite-explainer",
|
||||
"category": "instruction-following",
|
||||
"rubric": [
|
||||
"ثلاث نقاط بالضبط",
|
||||
"شرح صحيح ومختصر"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 25.93,
|
||||
"answer": "إليك شرح SQLite في ثلاث نقاط:\n\n1. **قاعدة بيانات في ملف واحد:** هي نظام قواعد بيانات علائقي خفيف الوزن يتم تخزين بياناته بالكامل في ملف واحد.\n2. **بدون خادم منفصل:** لا تتطلب خادماً (server) منفصلاً، مما يجعلها مثالية للتطبيقات الصغيرة والأجهزة المحمولة.\n3. **نظام SQL:** تستخدم لغة الاستعلامات المهيكلة (SQL) لإدارة البيانات واسترجاعها وتعديلها.",
|
||||
"tool": null,
|
||||
"checks": [],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
},
|
||||
{
|
||||
"id": "arithmetic-tool",
|
||||
"category": "tool-use",
|
||||
"rubric": [
|
||||
"استُدعيت أداة الحاسبة عند توفرها",
|
||||
"الناتج 3973 دون ادعاء غير صحيح"
|
||||
],
|
||||
"status": "ok",
|
||||
"elapsed_seconds": 0.03,
|
||||
"answer": "3973.0",
|
||||
"tool": "calculator",
|
||||
"checks": [
|
||||
{
|
||||
"check": {
|
||||
"type": "contains",
|
||||
"value": "3973"
|
||||
},
|
||||
"passed": true
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"type": "tool_is",
|
||||
"value": "calculator"
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
],
|
||||
"human_rating": null,
|
||||
"human_notes": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"created_at_utc": "2026-10-03T14:53:57.594426+00:00",
|
||||
"base_url": "http://127.0.0.1:8000",
|
||||
"dataset": "evals\\knowledge_retrieval_project.jsonl",
|
||||
"indexing": {
|
||||
"semantic_indexed_documents": 4,
|
||||
"documents": 4,
|
||||
"embedding_models": [
|
||||
"granite-embedding:278m"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"cases": 9,
|
||||
"hit_at_1": 0.6666666666666666,
|
||||
"hit_at_3": 1.0,
|
||||
"mean_reciprocal_rank": 0.8333333333333334,
|
||||
"evidence_rate": 0.8888888888888888
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"id": "project-path-confinement",
|
||||
"query": "Does resolving a requested source path guarantee it stays under the selected root?",
|
||||
"expected_document": "fixture_workspace.py",
|
||||
"expected_evidence": "relative_to(root)",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-preview-stale-file",
|
||||
"query": "How does applying an approved preview detect that the original file changed after review?",
|
||||
"expected_document": "fixture_workspace.py",
|
||||
"expected_evidence": "expected_hash",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_main.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-index-stale-source",
|
||||
"query": "Which file hash is compared with the stored content hash before returning a search result?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "actual_hash == row[\"content_hash\"]",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-chunk-overlap",
|
||||
"query": "What are the configured chunk size and overlap for local knowledge indexing?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "CHUNK_OVERLAP = 120",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-arabic-audio-variants",
|
||||
"query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "QUERY_SYNONYMS",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-pdf-page-bound",
|
||||
"query": "What is the maximum number of PDF pages accepted during local extraction?",
|
||||
"expected_document": "fixture_pdf.py",
|
||||
"expected_evidence": "MAX_PDF_PAGES = 30",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_pdf.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-pdf-empty-scan",
|
||||
"query": "ما الذي يحدث إذا رفع المستخدم ملف PDF ممسوحًا ولا يحتوي نصًا قابلًا للاستخراج؟",
|
||||
"expected_document": "fixture_pdf.py",
|
||||
"expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_main.py",
|
||||
"fixture_pdf.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-skill-tool-filter",
|
||||
"query": "How is a tool call blocked when it is outside the selected skill permissions?",
|
||||
"expected_document": "fixture_main.py",
|
||||
"expected_evidence": "tool_name not in selected_skill.allowed_tools",
|
||||
"rank": 2,
|
||||
"top_paths": [
|
||||
"fixture_workspace.py",
|
||||
"fixture_main.py"
|
||||
],
|
||||
"evidence_found": false,
|
||||
"search_mode": "hybrid"
|
||||
},
|
||||
{
|
||||
"id": "project-agent-index-limits",
|
||||
"query": "What file count and total size limits apply when indexing local knowledge?",
|
||||
"expected_document": "fixture_knowledge.py",
|
||||
"expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
|
||||
"rank": 1,
|
||||
"top_paths": [
|
||||
"fixture_knowledge.py",
|
||||
"fixture_main.py",
|
||||
"fixture_workspace.py"
|
||||
],
|
||||
"evidence_found": true,
|
||||
"search_mode": "hybrid"
|
||||
}
|
||||
],
|
||||
"note": "Small deterministic fixture set; measures configured retrieval (keyword or hybrid) and evidence presence, not answer quality or general RAG quality."
|
||||
}
|
||||
Reference in New Issue
Block a user