56 lines
1.7 KiB
JSON
56 lines
1.7 KiB
JSON
{
|
|
"created_at_utc": "2026-10-03T17:16:19.416417+00:00",
|
|
"base_url": "http://127.0.0.1:8131",
|
|
"dataset": "evals\\knowledge_retrieval_project.jsonl",
|
|
"indexing": {
|
|
"semantic_indexed_documents": 1,
|
|
"documents": 1,
|
|
"embedding_models": [
|
|
"granite-embedding:278m"
|
|
]
|
|
},
|
|
"metrics": {
|
|
"cases": 1,
|
|
"hit_at_1": 1.0,
|
|
"hit_at_3": 1.0,
|
|
"mean_reciprocal_rank": 1.0,
|
|
"evidence_rate": 1.0
|
|
},
|
|
"results": [
|
|
{
|
|
"id": "project-skill-tool-filter",
|
|
"query": "How is a tool call blocked when it is outside the selected skill permissions?",
|
|
"expected_document": "fixture_main.py",
|
|
"expected_evidence": "tool_name not in selected_skill.allowed_tools",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"fixture_main.py"
|
|
],
|
|
"expected_document_chunks": [
|
|
102,
|
|
99,
|
|
100,
|
|
79,
|
|
85,
|
|
84,
|
|
98
|
|
],
|
|
"evidence_found": true,
|
|
"search_mode": "hybrid",
|
|
"agent_answer": "لم يتم العثور على معلومات حول كيفية حظر استدعاء الأداة عندما يكون خارج أذونات المهارات المحددة.\n\nالمقاطع المسترجعة لا تتضمن إجابة مباشرة على هذا السؤال.",
|
|
"agent_tool": "search_knowledge",
|
|
"agent_files": [
|
|
"fixture_main.py"
|
|
],
|
|
"agent_steps": [
|
|
{
|
|
"tool": "search_knowledge",
|
|
"status": "completed"
|
|
}
|
|
],
|
|
"human_rating": null
|
|
}
|
|
],
|
|
"note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
|
|
}
|