203 lines
8.6 KiB
Python
203 lines
8.6 KiB
Python
import asyncio
|
||
import os
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from unittest.mock import AsyncMock, patch
|
||
|
||
from fastapi.testclient import TestClient
|
||
from fastapi import HTTPException
|
||
|
||
_TEST_DATA_DIR = None
|
||
if "SOVEREIGNAI_DATA_DIR" not in os.environ:
|
||
_TEST_DATA_DIR = tempfile.TemporaryDirectory(prefix="sovereignai-skills-tests-")
|
||
os.environ["SOVEREIGNAI_DATA_DIR"] = _TEST_DATA_DIR.name
|
||
|
||
from app.main import AgentRequest, _execute_agent, app, safe_arithmetic
|
||
|
||
|
||
class AgentSkillTests(unittest.TestCase):
|
||
@classmethod
|
||
def setUpClass(cls) -> None:
|
||
cls.client = TestClient(app)
|
||
cls.workspace = str(Path(__file__).resolve().parents[1])
|
||
|
||
def test_skill_catalog_discloses_scope_and_permissions(self) -> None:
|
||
response = self.client.get("/v1/agent/skills")
|
||
|
||
self.assertEqual(response.status_code, 200)
|
||
skills = {item["id"]: item for item in response.json()["skills"]}
|
||
self.assertEqual(set(skills), {"code_explain", "code_review", "test_plan"})
|
||
self.assertNotIn("propose_file_change", skills["code_explain"]["allowed_tools"])
|
||
self.assertIn("propose_file_change", skills["code_review"]["allowed_tools"])
|
||
self.assertEqual(response.json()["default"], None)
|
||
|
||
def test_safe_arithmetic_accepts_common_unicode_operator_symbols(self) -> None:
|
||
self.assertEqual(safe_arithmetic("137 × 29"), 3973.0)
|
||
self.assertEqual(safe_arithmetic("12 ÷ 3 − 1"), 3.0)
|
||
|
||
def test_explicit_calculator_request_uses_bounded_local_calculator(self) -> None:
|
||
with patch("app.main.get_completion", new=AsyncMock()) as model:
|
||
result = asyncio.run(
|
||
_execute_agent(
|
||
AgentRequest(
|
||
task="استخدم الحاسبة المتاحة لحساب 137 × 29، ثم أجب بالناتج فقط.",
|
||
model="qwen2.5:1.5b-instruct-q4_K_M",
|
||
)
|
||
)
|
||
)
|
||
|
||
model.assert_not_awaited()
|
||
self.assertEqual(result["tool"], "calculator")
|
||
self.assertEqual(result["result"], 3973.0)
|
||
|
||
def test_explain_skill_is_sent_to_model_and_hides_file_write_tool(self) -> None:
|
||
completion = {"choices": [{"message": {"content": "شرح مختصر."}}]}
|
||
with patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model:
|
||
result = asyncio.run(
|
||
_execute_agent(
|
||
AgentRequest(
|
||
task="اشرح بنية المشروع باختصار",
|
||
workspace_path=self.workspace,
|
||
skill_id="code_explain",
|
||
)
|
||
)
|
||
)
|
||
|
||
payload = model.await_args.args[0]
|
||
tool_names = [tool["function"]["name"] for tool in payload["tools"]]
|
||
system = payload["messages"][0]["content"]
|
||
self.assertIn("شرح الكود", system)
|
||
self.assertIn("ميّز بين ما قرأته وما استنتجته", system)
|
||
self.assertEqual(tool_names, ["calculator", "search_workspace", "search_knowledge"])
|
||
self.assertEqual(result["skill"], "code_explain")
|
||
|
||
def test_review_skill_exposes_preview_tool_but_never_applies_it(self) -> None:
|
||
completion = {"choices": [{"message": {"content": "سأعرض النتائج."}}]}
|
||
with patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model:
|
||
result = asyncio.run(
|
||
_execute_agent(
|
||
AgentRequest(
|
||
task="راجع الملفات دون تعديل",
|
||
workspace_path=self.workspace,
|
||
skill_id="code_review",
|
||
)
|
||
)
|
||
)
|
||
|
||
payload = model.await_args.args[0]
|
||
tool_names = [tool["function"]["name"] for tool in payload["tools"]]
|
||
self.assertIn("propose_file_change", tool_names)
|
||
self.assertIn("لا تطبق الكتابة", payload["messages"][0]["content"])
|
||
self.assertEqual(result["skill"], "code_review")
|
||
self.assertNotIn("proposal", result)
|
||
|
||
def test_preselected_file_is_read_once_without_redundant_search_call(self) -> None:
|
||
completion = {"choices": [{"message": {"content": "المهارات مسجلة في قاموس محلي."}}]}
|
||
with patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model:
|
||
result = asyncio.run(
|
||
_execute_agent(
|
||
AgentRequest(
|
||
task="اشرح الملف المحدد",
|
||
workspace_path=self.workspace,
|
||
workspace_files=["app/skills.py"],
|
||
skill_id="code_explain",
|
||
)
|
||
)
|
||
)
|
||
|
||
self.assertEqual(model.await_count, 1)
|
||
self.assertEqual(result["files"], ["app/skills.py"])
|
||
self.assertIn("Curated, local agent skills", model.await_args.args[0]["messages"][1]["content"])
|
||
|
||
def test_explicit_knowledge_search_is_prefetched_before_model_answer(self) -> None:
|
||
completion = {"choices": [{"message": {"content": "المرحلة 5 تضيف الفهرسة المحلية."}}]}
|
||
match = {"path": "ROADMAP.md", "chunk": 2, "text": "SQLite FTS5 local index", "excerpt": "SQLite FTS5"}
|
||
with (
|
||
patch("app.main.knowledge.search", return_value=[match]) as retrieve,
|
||
patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model,
|
||
):
|
||
result = asyncio.run(
|
||
_execute_agent(
|
||
AgentRequest(
|
||
task="ابحث في فهرس المعرفة عن المرحلة 5",
|
||
workspace_path=self.workspace,
|
||
skill_id="test_plan",
|
||
)
|
||
)
|
||
)
|
||
|
||
retrieve.assert_called_once()
|
||
model_payload = model.await_args.args[0]
|
||
self.assertEqual(model_payload["max_tokens"], 384)
|
||
self.assertIn("SQLite FTS5 local index", model_payload["messages"][1]["content"])
|
||
self.assertNotIn(
|
||
"search_knowledge",
|
||
[tool["function"]["name"] for tool in model_payload["tools"]],
|
||
)
|
||
self.assertEqual(result["tool"], "search_knowledge")
|
||
self.assertEqual(result["files"], ["ROADMAP.md"])
|
||
|
||
def test_test_plan_skill_only_advertises_workspace_search(self) -> None:
|
||
completion = {"choices": [{"message": {"content": "ثلاث حالات اختبار مقترحة."}}]}
|
||
with patch("app.main.get_completion", new=AsyncMock(return_value=completion)) as model:
|
||
asyncio.run(
|
||
_execute_agent(
|
||
AgentRequest(
|
||
task="أنشئ خطة اختبار",
|
||
workspace_path=self.workspace,
|
||
skill_id="test_plan",
|
||
)
|
||
)
|
||
)
|
||
|
||
tools = model.await_args.args[0]["tools"]
|
||
self.assertEqual(
|
||
[tool["function"]["name"] for tool in tools],
|
||
["search_workspace", "search_knowledge"],
|
||
)
|
||
|
||
def test_server_rejects_tool_call_outside_active_skill_permissions(self) -> None:
|
||
completion = {
|
||
"choices": [
|
||
{
|
||
"message": {
|
||
"tool_calls": [
|
||
{
|
||
"id": "call-forbidden",
|
||
"function": {
|
||
"name": "propose_file_change",
|
||
"arguments": '{"path":"new.py","operation":"create","content":"print(1)"}',
|
||
},
|
||
}
|
||
]
|
||
}
|
||
}
|
||
]
|
||
}
|
||
with patch("app.main.get_completion", new=AsyncMock(return_value=completion)):
|
||
with self.assertRaises(HTTPException) as error:
|
||
asyncio.run(
|
||
_execute_agent(
|
||
AgentRequest(
|
||
task="اشرح المشروع",
|
||
workspace_path=self.workspace,
|
||
skill_id="code_explain",
|
||
)
|
||
)
|
||
)
|
||
|
||
self.assertEqual(error.exception.status_code, 422)
|
||
|
||
def test_unknown_skill_is_rejected_by_request_contract(self) -> None:
|
||
response = self.client.post(
|
||
"/v1/agent/run",
|
||
json={"task": "سؤال", "skill_id": "execute_shell"},
|
||
)
|
||
|
||
self.assertEqual(response.status_code, 422)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|