Complete local hybrid search and improve agent reliability

SovereignAI-Starter/tests/test_agent_skills.py

@@ -0,0 +1,202 @@
+import asyncio
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+from fastapi.testclient import TestClient
+from fastapi import HTTPException
+
+_TEST_DATA_DIR = None
+if "SOVEREIGNAI_DATA_DIR" not in os.environ:
+    _TEST_DATA_DIR = tempfile.TemporaryDirectory(prefix="sovereignai-skills-tests-")
+    os.environ["SOVEREIGNAI_DATA_DIR"] = _TEST_DATA_DIR.name
+
+from app.main import AgentRequest, _execute_agent, app, safe_arithmetic
+
+
+class AgentSkillTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.client = TestClient(app)
+        cls.workspace = str(Path(__file__).resolve().parents[1])
+
+    def test_skill_catalog_discloses_scope_and_permissions(self) -> None:
+        response = self.client.get("/v1/agent/skills")
+
+        self.assertEqual(response.status_code, 200)
+        skills = {item["id"]: item for item in response.json()["skills"]}
+        self.assertEqual(set(skills), {"code_explain", "code_review", "test_plan"})
+        self.assertNotIn("propose_file_change", skills["code_explain"]["allowed_tools"])
+        self.assertIn("propose_file_change", skills["code_review"]["allowed_tools"])
+        self.assertEqual(response.json()["default"], None)
+
+    def test_safe_arithmetic_accepts_common_unicode_operator_symbols(self) -> None:
+        self.assertEqual(safe_arithmetic("137 × 29"), 3973.0)
+        self.assertEqual(safe_arithmetic("12 ÷ 3 − 1"), 3.0)
+
+    def test_explicit_calculator_request_uses_bounded_local_calculator(self) -> None:
+        with patch("app.main.get_completion", new=AsyncMock()) as model:
+            result = asyncio.run(
+                _execute_agent(
+                    AgentRequest(
+                        task="استخدم الحاسبة المتاحة لحساب 137 × 29، ثم أجب بالناتج فقط.",
+                        model="qwen2.5:1.5b-instruct-q4_K_M",
+                    )
+                )
+            )
+
+        model.assert_not_awaited()
+        self.assertEqual(result["tool"], "calculator")
+        self.assertEqual(result["result"], 3973.0)
+
+    def test_explain_skill_is_sent_to_model_and_hides_file_write_tool(self) -> None:
+        completion = {"choices": [{"message": {"content": "شرح مختصر."}}]}
+        with patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model:
+            result = asyncio.run(
+                _execute_agent(
+                    AgentRequest(
+                        task="اشرح بنية المشروع باختصار",
+                        workspace_path=self.workspace,
+                        skill_id="code_explain",
+                    )
+                )
+            )
+
+        payload = model.await_args.args[0]
+        tool_names = [tool["function"]["name"] for tool in payload["tools"]]
+        system = payload["messages"][0]["content"]
+        self.assertIn("شرح الكود", system)
+        self.assertIn("ميّز بين ما قرأته وما استنتجته", system)
+        self.assertEqual(tool_names, ["calculator", "search_workspace", "search_knowledge"])
+        self.assertEqual(result["skill"], "code_explain")
+
+    def test_review_skill_exposes_preview_tool_but_never_applies_it(self) -> None:
+        completion = {"choices": [{"message": {"content": "سأعرض النتائج."}}]}
+        with patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model:
+            result = asyncio.run(
+                _execute_agent(
+                    AgentRequest(
+                        task="راجع الملفات دون تعديل",
+                        workspace_path=self.workspace,
+                        skill_id="code_review",
+                    )
+                )
+            )
+
+        payload = model.await_args.args[0]
+        tool_names = [tool["function"]["name"] for tool in payload["tools"]]
+        self.assertIn("propose_file_change", tool_names)
+        self.assertIn("لا تطبق الكتابة", payload["messages"][0]["content"])
+        self.assertEqual(result["skill"], "code_review")
+        self.assertNotIn("proposal", result)
+
+    def test_preselected_file_is_read_once_without_redundant_search_call(self) -> None:
+        completion = {"choices": [{"message": {"content": "المهارات مسجلة في قاموس محلي."}}]}
+        with patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model:
+            result = asyncio.run(
+                _execute_agent(
+                    AgentRequest(
+                        task="اشرح الملف المحدد",
+                        workspace_path=self.workspace,
+                        workspace_files=["app/skills.py"],
+                        skill_id="code_explain",
+                    )
+                )
+            )
+
+        self.assertEqual(model.await_count, 1)
+        self.assertEqual(result["files"], ["app/skills.py"])
+        self.assertIn("Curated, local agent skills", model.await_args.args[0]["messages"][1]["content"])
+
+    def test_explicit_knowledge_search_is_prefetched_before_model_answer(self) -> None:
+        completion = {"choices": [{"message": {"content": "المرحلة 5 تضيف الفهرسة المحلية."}}]}
+        match = {"path": "ROADMAP.md", "chunk": 2, "text": "SQLite FTS5 local index", "excerpt": "SQLite FTS5"}
+        with (
+            patch("app.main.knowledge.search", return_value=[match]) as retrieve,
+            patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model,
+        ):
+            result = asyncio.run(
+                _execute_agent(
+                    AgentRequest(
+                        task="ابحث في فهرس المعرفة عن المرحلة 5",
+                        workspace_path=self.workspace,
+                        skill_id="test_plan",
+                    )
+                )
+            )
+
+        retrieve.assert_called_once()
+        model_payload = model.await_args.args[0]
+        self.assertEqual(model_payload["max_tokens"], 384)
+        self.assertIn("SQLite FTS5 local index", model_payload["messages"][1]["content"])
+        self.assertNotIn(
+            "search_knowledge",
+            [tool["function"]["name"] for tool in model_payload["tools"]],
+        )
+        self.assertEqual(result["tool"], "search_knowledge")
+        self.assertEqual(result["files"], ["ROADMAP.md"])
+
+    def test_test_plan_skill_only_advertises_workspace_search(self) -> None:
+        completion = {"choices": [{"message": {"content": "ثلاث حالات اختبار مقترحة."}}]}
+        with patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model:
+            asyncio.run(
+                _execute_agent(
+                    AgentRequest(
+                        task="أنشئ خطة اختبار",
+                        workspace_path=self.workspace,
+                        skill_id="test_plan",
+                    )
+                )
+            )
+
+        tools = model.await_args.args[0]["tools"]
+        self.assertEqual(
+            [tool["function"]["name"] for tool in tools],
+            ["search_workspace", "search_knowledge"],
+        )
+
+    def test_server_rejects_tool_call_outside_active_skill_permissions(self) -> None:
+        completion = {
+            "choices": [
+                {
+                    "message": {
+                        "tool_calls": [
+                            {
+                                "id": "call-forbidden",
+                                "function": {
+                                    "name": "propose_file_change",
+                                    "arguments": '{"path":"new.py","operation":"create","content":"print(1)"}',
+                                },
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+        with patch("app.main.get_completion", new=AsyncMock(return_value=completion)):
+            with self.assertRaises(HTTPException) as error:
+                asyncio.run(
+                    _execute_agent(
+                        AgentRequest(
+                            task="اشرح المشروع",
+                            workspace_path=self.workspace,
+                            skill_id="code_explain",
+                        )
+                    )
+                )
+
+        self.assertEqual(error.exception.status_code, 422)
+
+    def test_unknown_skill_is_rejected_by_request_contract(self) -> None:
+        response = self.client.post(
+            "/v1/agent/run",
+            json={"task": "سؤال", "skill_id": "execute_shell"},
+        )
+
+        self.assertEqual(response.status_code, 422)
+
+
+if __name__ == "__main__":
+    unittest.main()