{
  "created_at_utc": "2026-10-03T17:12:10.145801+00:00",
  "base_url": "http://127.0.0.1:8131",
  "dataset": "evals\\knowledge_retrieval_project.jsonl",
  "indexing": {
    "semantic_indexed_documents": 4,
    "documents": 4,
    "embedding_models": [
      "granite-embedding:278m"
    ]
  },
  "metrics": {
    "cases": 9,
    "hit_at_1": 0.8888888888888888,
    "hit_at_3": 1.0,
    "mean_reciprocal_rank": 0.9444444444444444,
    "evidence_rate": 1.0
  },
  "results": [
    {
      "id": "project-path-confinement",
      "query": "Does resolving a requested source path guarantee it stays under the selected root?",
      "expected_document": "fixture_workspace.py",
      "expected_evidence": "relative_to(root)",
      "rank": 1,
      "top_paths": [
        "fixture_workspace.py",
        "fixture_main.py"
      ],
      "expected_document_chunks": [
        5,
        19,
        1,
        6,
        10
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    },
    {
      "id": "project-preview-stale-file",
      "query": "How does applying an approved preview detect that the original file changed after review?",
      "expected_document": "fixture_workspace.py",
      "expected_evidence": "expected_hash",
      "rank": 1,
      "top_paths": [
        "fixture_workspace.py",
        "fixture_main.py"
      ],
      "expected_document_chunks": [
        13,
        15,
        12,
        11,
        14
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    },
    {
      "id": "project-index-stale-source",
      "query": "Which file hash is compared with the stored content hash before returning a search result?",
      "expected_document": "fixture_knowledge.py",
      "expected_evidence": "actual_hash == row[\"content_hash\"]",
      "rank": 1,
      "top_paths": [
        "fixture_knowledge.py",
        "fixture_workspace.py",
        "fixture_main.py"
      ],
      "expected_document_chunks": [
        17,
        11
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    },
    {
      "id": "project-chunk-overlap",
      "query": "What are the configured chunk size and overlap for local knowledge indexing?",
      "expected_document": "fixture_knowledge.py",
      "expected_evidence": "CHUNK_OVERLAP = 120",
      "rank": 1,
      "top_paths": [
        "fixture_knowledge.py",
        "fixture_main.py",
        "fixture_workspace.py"
      ],
      "expected_document_chunks": [
        3,
        0,
        7,
        16
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    },
    {
      "id": "project-arabic-audio-variants",
      "query": "كيف يربط فهرس المعرفة بين تعابير عربية مختلفة عند البحث عن تفريغ التسجيل؟",
      "expected_document": "fixture_knowledge.py",
      "expected_evidence": "QUERY_SYNONYMS",
      "rank": 1,
      "top_paths": [
        "fixture_knowledge.py",
        "fixture_main.py"
      ],
      "expected_document_chunks": [
        1,
        0
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    },
    {
      "id": "project-pdf-page-bound",
      "query": "What is the maximum number of PDF pages accepted during local extraction?",
      "expected_document": "fixture_pdf.py",
      "expected_evidence": "MAX_PDF_PAGES = 30",
      "rank": 1,
      "top_paths": [
        "fixture_pdf.py",
        "fixture_main.py"
      ],
      "expected_document_chunks": [
        2,
        0,
        1,
        4,
        5,
        6
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    },
    {
      "id": "project-pdf-empty-scan",
      "query": "ما الذي يحدث إذا رفع المستخدم ملف PDF ممسوحًا ولا يحتوي نصًا قابلًا للاستخراج؟",
      "expected_document": "fixture_pdf.py",
      "expected_evidence": "لا يحتوي PDF على نص قابل للاستخراج",
      "rank": 1,
      "top_paths": [
        "fixture_pdf.py",
        "fixture_main.py"
      ],
      "expected_document_chunks": [
        1,
        9,
        0
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    },
    {
      "id": "project-skill-tool-filter",
      "query": "How is a tool call blocked when it is outside the selected skill permissions?",
      "expected_document": "fixture_main.py",
      "expected_evidence": "tool_name not in selected_skill.allowed_tools",
      "rank": 1,
      "top_paths": [
        "fixture_main.py",
        "fixture_workspace.py",
        "fixture_knowledge.py"
      ],
      "expected_document_chunks": [
        102,
        85,
        99,
        100,
        79,
        98
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    },
    {
      "id": "project-agent-index-limits",
      "query": "What file count and total size limits apply when indexing local knowledge?",
      "expected_document": "fixture_knowledge.py",
      "expected_evidence": "MAX_TOTAL_CHARS = 2_000_000",
      "rank": 2,
      "top_paths": [
        "fixture_workspace.py",
        "fixture_knowledge.py",
        "fixture_main.py"
      ],
      "expected_document_chunks": [
        0,
        3
      ],
      "evidence_found": true,
      "search_mode": "hybrid"
    }
  ],
  "note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
}