125 lines
4.0 KiB
JSON
125 lines
4.0 KiB
JSON
{
|
|
"created_at_utc": "2026-10-02T14:11:32.760819+00:00",
|
|
"base_url": "http://127.0.0.1:8000",
|
|
"dataset": "evals\\knowledge_retrieval_project.jsonl",
|
|
"metrics": {
|
|
"cases": 9,
|
|
"hit_at_1": 0.5555555555555556,
|
|
"hit_at_3": 0.7777777777777778,
|
|
"mean_reciprocal_rank": 0.6666666666666666,
|
|
"evidence_rate": 0.6666666666666666
|
|
},
|
|
"results": [
|
|
{
|
|
"id": "project-path-confinement",
|
|
"query": "Does resolving a requested source path guarantee it stays under the selected root?",
|
|
"expected_document": "fixture_workspace.py",
|
|
"expected_evidence": "relative_to(root)",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"fixture_workspace.py",
|
|
"fixture_main.py"
|
|
],
|
|
"evidence_found": true
|
|
},
|
|
{
|
|
"id": "project-preview-stale-file",
|
|
"query": "How does applying an approved preview detect that the original file changed after review?",
|
|
"expected_document": "fixture_workspace.py",
|
|
"expected_evidence": "expected_hash",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"fixture_workspace.py",
|
|
"fixture_main.py"
|
|
],
|
|
"evidence_found": true
|
|
},
|
|
{
|
|
"id": "project-index-stale-source",
|
|
"query": "How does local search avoid returning indexed content after its source file changes?",
|
|
"expected_document": "fixture_knowledge.py",
|
|
"expected_evidence": "actual_hash == row[\"content_hash\"]",
|
|
"rank": null,
|
|
"top_paths": [
|
|
"fixture_workspace.py",
|
|
"fixture_main.py"
|
|
],
|
|
"evidence_found": false
|
|
},
|
|
{
|
|
"id": "project-chunk-overlap",
|
|
"query": "How much surrounding text is preserved between adjacent retrieval segments?",
|
|
"expected_document": "fixture_knowledge.py",
|
|
"expected_evidence": "CHUNK_OVERLAP = 120",
|
|
"rank": null,
|
|
"top_paths": [
|
|
"fixture_main.py",
|
|
"fixture_pdf.py"
|
|
],
|
|
"evidence_found": false
|
|
},
|
|
{
|
|
"id": "project-arabic-audio-variants",
|
|
"query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
|
|
"expected_document": "fixture_knowledge.py",
|
|
"expected_evidence": "QUERY_SYNONYMS",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"fixture_knowledge.py",
|
|
"fixture_main.py"
|
|
],
|
|
"evidence_found": true
|
|
},
|
|
{
|
|
"id": "project-pdf-page-bound",
|
|
"query": "What is the maximum number of PDF pages accepted during local extraction?",
|
|
"expected_document": "fixture_pdf.py",
|
|
"expected_evidence": "max_pages: int = 30",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"fixture_pdf.py",
|
|
"fixture_main.py",
|
|
"fixture_workspace.py"
|
|
],
|
|
"evidence_found": true
|
|
},
|
|
{
|
|
"id": "project-pdf-empty-scan",
|
|
"query": "What happens when a PDF has no extractable text, such as a scanned page?",
|
|
"expected_document": "fixture_pdf.py",
|
|
"expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
|
|
"rank": 2,
|
|
"top_paths": [
|
|
"fixture_main.py",
|
|
"fixture_pdf.py"
|
|
],
|
|
"evidence_found": false
|
|
},
|
|
{
|
|
"id": "project-skill-tool-filter",
|
|
"query": "How is a tool call blocked when it is outside the selected skill permissions?",
|
|
"expected_document": "fixture_main.py",
|
|
"expected_evidence": "tool_name not in selected_skill.allowed_tools",
|
|
"rank": 1,
|
|
"top_paths": [
|
|
"fixture_main.py"
|
|
],
|
|
"evidence_found": true
|
|
},
|
|
{
|
|
"id": "project-agent-index-limits",
|
|
"query": "What file count and total size limits apply when indexing local knowledge?",
|
|
"expected_document": "fixture_knowledge.py",
|
|
"expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
|
|
"rank": 2,
|
|
"top_paths": [
|
|
"fixture_main.py",
|
|
"fixture_knowledge.py",
|
|
"fixture_workspace.py"
|
|
],
|
|
"evidence_found": true
|
|
}
|
|
],
|
|
"note": "Small deterministic fixture set; measures lexical retrieval only, not answer quality or general RAG quality."
|
|
}
|