feat: surface retrieved evidence when agent abstains

SovereignAI-Starter/scripts/eval_knowledge_retrieval.py

@@ -93,6 +93,7 @@ def main() -> int:
     results: list[dict] = []
     index_summary: dict[str, object] = {}
     files = sorted({case["document"] for case in cases})
+    projects_url = args.base_url.rstrip("/") + "/v1/agent/projects"
     index_url = args.base_url.rstrip("/") + "/v1/agent/knowledge/index"
     search_url = args.base_url.rstrip("/") + "/v1/agent/knowledge/search"
     base_url = args.base_url.rstrip("/")
@@ -121,7 +122,21 @@ def main() -> int:
                 destination.write_text(case["text"], encoding="utf-8")
         index_payload = {"workspace_path": str(workspace_path), "files": files}
         index_attempted = False
+        project_registration_attempted = False
+        project_registered = False
         try:
+            # A temporary directory is outside the default workspace grant. Register
+            # it explicitly through the same local-user API used by the desktop app.
+            project_registration_attempted = True
+            registration = post_json(
+                projects_url,
+                {"workspace_path": str(workspace_path)},
+                timeout=args.request_timeout,
+                token=token,
+            )
+            project_registered = True
+            if Path(registration.get("path", "")).resolve() != workspace_path.resolve():
+                raise RuntimeError("The API registered a different fixture workspace.")
             index_attempted = True
             index_result = post_json(
                 index_url, index_payload, timeout=args.request_timeout, token=token
@@ -195,8 +210,9 @@ def main() -> int:
                     result["human_rating"] = None
                 results.append(result)
         finally:
+            original_error = sys.exc_info()[0] is not None
+            cleanup_errors: list[Exception] = []
             if index_attempted:
-                original_error = sys.exc_info()[0] is not None
                 try:
                     deletion = delete_json(
                         index_url,
@@ -209,10 +225,29 @@ def main() -> int:
                             "The API did not confirm cleanup for every fixture document."
                         )
                 except Exception as cleanup_error:
-                    if original_error:
+                    cleanup_errors.append(cleanup_error)
+            if project_registration_attempted:
+                try:
+                    revocation = delete_json(
+                        projects_url,
+                        {"workspace_path": str(workspace_path)},
+                        timeout=args.request_timeout,
+                        token=token,
+                    )
+                    if project_registered and not revocation.get("removed"):
+                        raise RuntimeError(
+                            "The API did not confirm removal of the temporary project registration."
+                        )
+                except Exception as cleanup_error:
+                    cleanup_errors.append(cleanup_error)
+            if cleanup_errors:
+                if original_error:
+                    for cleanup_error in cleanup_errors:
                         print(f"Cleanup also failed: {cleanup_error}", file=sys.stderr)
-                    else:
-                        raise
+                else:
+                    raise RuntimeError(
+                        "; ".join(str(error) for error in cleanup_errors)
+                    ) from cleanup_errors[0]
 
     logout()