Continue roadmap: local TLS and agent review preparation

2026-10-04 00:35:05 +03:00
parent 379098f3d6
commit d29275043a
28 changed files with 3226 additions and 47 deletions
@@ -0,0 +1,55 @@
+{
+  "created_at_utc": "2026-10-03T17:16:19.416417+00:00",
+  "base_url": "http://127.0.0.1:8131",
+  "dataset": "evals\\knowledge_retrieval_project.jsonl",
+  "indexing": {
+    "semantic_indexed_documents": 1,
+    "documents": 1,
+    "embedding_models": [
+      "granite-embedding:278m"
+    ]
+  },
+  "metrics": {
+    "cases": 1,
+    "hit_at_1": 1.0,
+    "hit_at_3": 1.0,
+    "mean_reciprocal_rank": 1.0,
+    "evidence_rate": 1.0
+  },
+  "results": [
+    {
+      "id": "project-skill-tool-filter",
+      "query": "How is a tool call blocked when it is outside the selected skill permissions?",
+      "expected_document": "fixture_main.py",
+      "expected_evidence": "tool_name not in selected_skill.allowed_tools",
+      "rank": 1,
+      "top_paths": [
+        "fixture_main.py"
+      ],
+      "expected_document_chunks": [
+        102,
+        99,
+        100,
+        79,
+        85,
+        84,
+        98
+      ],
+      "evidence_found": true,
+      "search_mode": "hybrid",
+      "agent_answer": "لم يتم العثور على معلومات حول كيفية حظر استدعاء الأداة عندما يكون خارج أذونات المهارات المحددة.\n\nالمقاطع المسترجعة لا تتضمن إجابة مباشرة على هذا السؤال.",
+      "agent_tool": "search_knowledge",
+      "agent_files": [
+        "fixture_main.py"
+      ],
+      "agent_steps": [
+        {
+          "tool": "search_knowledge",
+          "status": "completed"
+        }
+      ],
+      "human_rating": null
+    }
+  ],
+  "note": "Small deterministic fixture set. Retrieval metrics cover source ranking and evidence presence, not general RAG quality. Optional agent answers are retained for human review and are not auto-scored."
+}