Complete local hybrid search and improve agent reliability

SovereignAI-Starter/app/database.py

@@ -103,6 +103,17 @@ def initialize_database() -> None:
             );
             CREATE INDEX IF NOT EXISTS idx_answer_feedback_user
                 ON answer_feedback(user_id, updated_at DESC);
+
+            CREATE TABLE IF NOT EXISTS agent_audit_events (
+                id TEXT PRIMARY KEY,
+                tool TEXT NOT NULL,
+                method TEXT NOT NULL,
+                status_code INTEGER NOT NULL,
+                duration_ms INTEGER NOT NULL,
+                created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+            );
+            CREATE INDEX IF NOT EXISTS idx_agent_audit_created
+                ON agent_audit_events(created_at DESC);
             """
         )
         message_columns = {
@@ -319,4 +330,37 @@ def save_answer_feedback(
         )
 
 
+def record_agent_audit_event(
+    event_id: str,
+    tool: str,
+    method: str,
+    status_code: int,
+    duration_ms: int,
+) -> None:
+    """Record agent route metadata only; never persist prompts or file contents."""
+    with _connect() as connection:
+        connection.execute(
+            """
+            INSERT INTO agent_audit_events(id, tool, method, status_code, duration_ms)
+            VALUES (?, ?, ?, ?, ?)
+            """,
+            (event_id, tool, method, status_code, duration_ms),
+        )
+
+
+def list_agent_audit_events(limit: int = 50) -> list[dict[str, Any]]:
+    bounded_limit = max(1, min(limit, 200))
+    with _connect() as connection:
+        rows = connection.execute(
+            """
+            SELECT id, tool, method, status_code, duration_ms, created_at
+            FROM agent_audit_events
+            ORDER BY created_at DESC, rowid DESC
+            LIMIT ?
+            """,
+            (bounded_limit,),
+        ).fetchall()
+    return [dict(row) for row in rows]
+
+
 initialize_database()

SovereignAI-Starter/app/embeddings.py

@@ -0,0 +1,67 @@
+"""Optional local Ollama embeddings for semantic knowledge retrieval."""
+
+from __future__ import annotations
+
+import math
+import os
+from typing import Sequence
+import httpx
+
+
+class EmbeddingUnavailable(RuntimeError):
+    """The configured local embedding model is missing or cannot embed text."""
+
+
+def embedding_model_name() -> str | None:
+    return os.getenv("KNOWLEDGE_EMBEDDING_MODEL", "granite-embedding:278m").strip() or None
+
+
+def _ollama_base_url() -> str:
+    configured = os.getenv("LOCAL_LLM_BASE_URL", "http://127.0.0.1:11434/v1").rstrip("/")
+    if configured.endswith("/v1"):
+        configured = configured[:-3]
+    return configured
+
+
+async def embed_texts(texts: Sequence[str], *, model: str | None = None) -> list[list[float]]:
+    """Generate bounded-size embedding batches through the local Ollama API."""
+    if not texts:
+        return []
+    configured_model = model if model is not None else embedding_model_name()
+    model_name = configured_model.strip() if configured_model else ""
+    if not model_name:
+        raise EmbeddingUnavailable("لم يُضبط نموذج التضمين المحلي.")
+    endpoint = f"{_ollama_base_url()}/api/embed"
+    results: list[list[float]] = []
+    try:
+        async with httpx.AsyncClient(timeout=httpx.Timeout(180.0, connect=5.0)) as client:
+            for offset in range(0, len(texts), 32):
+                batch = [str(text)[:4_000] for text in texts[offset : offset + 32]]
+                response = await client.post(
+                    endpoint,
+                    json={"model": model_name, "input": batch, "keep_alive": "5m"},
+                )
+                response.raise_for_status()
+                payload = response.json()
+                vectors = payload.get("embeddings") if isinstance(payload, dict) else None
+                if not isinstance(vectors, list) or len(vectors) != len(batch):
+                    raise EmbeddingUnavailable("أعاد نموذج التضمين عدد متجهات غير متوقع.")
+                dimension = None
+                for vector in vectors:
+                    if not isinstance(vector, list) or not vector:
+                        raise EmbeddingUnavailable("أعاد نموذج التضمين متجهًا فارغًا.")
+                    if dimension is None:
+                        dimension = len(vector)
+                    if len(vector) != dimension:
+                        raise EmbeddingUnavailable("أبعاد متجهات التضمين غير متطابقة.")
+                    numeric = [float(value) for value in vector]
+                    if not all(math.isfinite(value) for value in numeric):
+                        raise EmbeddingUnavailable("أعاد نموذج التضمين قيمًا غير صالحة.")
+                    if not any(value != 0 for value in numeric):
+                        raise EmbeddingUnavailable("أعاد نموذج التضمين متجهًا صفريًا.")
+                    results.append(numeric)
+    except EmbeddingUnavailable:
+        raise
+    except (httpx.HTTPError, ValueError, TypeError, KeyError) as exc:
+        raise EmbeddingUnavailable("نموذج التضمين المحلي غير متاح؛ سيبقى البحث النصي مستخدمًا.") from exc
+    return results

SovereignAI-Starter/app/knowledge.py

@@ -0,0 +1,372 @@
+"""Bounded, local SQLite FTS index for user-selected workspace documents."""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+import hashlib
+import json
+import math
+import struct
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Iterator
+
+from app import database
+from app import workspace
+
+CHUNK_SIZE = 1000
+CHUNK_OVERLAP = 120
+MAX_CHUNKS_PER_DOCUMENT = 2
+MAX_FILES_PER_INDEX = 20
+MAX_TOTAL_CHARS = 2_000_000
+SEARCH_STOP_WORDS = {
+    "ابحث", "البحث", "بحث", "في", "عن", "فهرس", "الفهرس", "المعرفة", "المحلي",
+    "المحلية", "قاعدة", "المرحلة", "مرحلة", "ثم", "لخص", "استشهد", "مسار", "الملف",
+    "أي", "ما", "كيف", "أين", "وين", "خدمة", "الذي", "التي", "إلى", "من", "على",
+    "the", "and", "for", "from", "with", "search", "knowledge", "index", "indexed",
+}
+QUERY_SYNONYMS = {
+    "تفرغ": ("تفريغ", "التفريغ", "للتفريغ", "transcription"),
+    "يفرغ": ("تفريغ", "التفريغ", "للتفريغ", "transcription"),
+    "التسجيلات": ("التسجيل", "التسجيلات"),
+    "الصوتية": ("الصوت", "الصوتية", "audio"),
+}
+
+
+@contextmanager
+def _connect() -> Iterator[sqlite3.Connection]:
+    connection = sqlite3.connect(database.DATABASE_PATH, timeout=10)
+    try:
+        with connection:
+            yield connection
+    finally:
+        connection.close()
+
+
+def initialize() -> None:
+    with _connect() as connection:
+        connection.execute("PRAGMA foreign_keys = ON")
+        connection.executescript(
+            """
+            CREATE TABLE IF NOT EXISTS knowledge_documents (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                user_id TEXT NOT NULL,
+                workspace_path TEXT NOT NULL,
+                relative_path TEXT NOT NULL,
+                content_hash TEXT NOT NULL,
+                updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                UNIQUE(user_id, workspace_path, relative_path)
+            );
+            CREATE TABLE IF NOT EXISTS knowledge_chunks (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                document_id INTEGER NOT NULL REFERENCES knowledge_documents(id) ON DELETE CASCADE,
+                chunk_index INTEGER NOT NULL,
+                body TEXT NOT NULL,
+                UNIQUE(document_id, chunk_index)
+            );
+            CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_fts USING fts5(
+                body, tokenize = 'unicode61'
+            );
+            CREATE TABLE IF NOT EXISTS knowledge_embeddings (
+                chunk_id INTEGER PRIMARY KEY REFERENCES knowledge_chunks(id) ON DELETE CASCADE,
+                model TEXT NOT NULL,
+                dimensions INTEGER NOT NULL,
+                vector BLOB NOT NULL
+            );
+            """
+        )
+
+
+def _chunks(text: str) -> list[str]:
+    cleaned = text.strip()
+    if not cleaned:
+        return []
+    step = CHUNK_SIZE - CHUNK_OVERLAP
+    result = [cleaned[start : start + CHUNK_SIZE] for start in range(0, len(cleaned), step)]
+    return [chunk for chunk in result if chunk.strip()]
+
+
+def index_document(
+    *, user_id: str, workspace_path: Path, relative_path: str, text: str, content_hash: str
+) -> dict[str, Any]:
+    chunks = _chunks(text)
+    if not chunks:
+        raise ValueError("الملف لا يحتوي على نص قابل للفهرسة.")
+    if sum(map(len, chunks)) > MAX_TOTAL_CHARS + CHUNK_SIZE:
+        raise ValueError("نص الملف يتجاوز حد الفهرسة.")
+
+    root = str(workspace_path.resolve(strict=True))
+    with _connect() as connection:
+        connection.execute("PRAGMA foreign_keys = ON")
+        existing = connection.execute(
+            "SELECT id FROM knowledge_documents WHERE user_id=? AND workspace_path=? AND relative_path=?",
+            (user_id, root, relative_path),
+        ).fetchone()
+        if existing:
+            document_id = int(existing[0])
+            rows = connection.execute(
+                "SELECT id FROM knowledge_chunks WHERE document_id=?", (document_id,)
+            ).fetchall()
+            connection.executemany("DELETE FROM knowledge_fts WHERE rowid=?", rows)
+            connection.executemany("DELETE FROM knowledge_embeddings WHERE chunk_id=?", rows)
+            connection.execute("DELETE FROM knowledge_chunks WHERE document_id=?", (document_id,))
+            connection.execute(
+                "UPDATE knowledge_documents SET content_hash=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
+                (content_hash, document_id),
+            )
+        else:
+            cursor = connection.execute(
+                "INSERT INTO knowledge_documents(user_id,workspace_path,relative_path,content_hash) VALUES(?,?,?,?)",
+                (user_id, root, relative_path, content_hash),
+            )
+            document_id = int(cursor.lastrowid)
+
+        for index, body in enumerate(chunks):
+            cursor = connection.execute(
+                "INSERT INTO knowledge_chunks(document_id,chunk_index,body) VALUES(?,?,?)",
+                (document_id, index, body),
+            )
+            connection.execute(
+                "INSERT INTO knowledge_fts(rowid,body) VALUES(?,?)",
+                (int(cursor.lastrowid), body),
+            )
+    return {"path": relative_path, "chunks": len(chunks), "characters": len(text)}
+
+
+def store_embeddings(
+    *, user_id: str, workspace_path: Path, relative_path: str, model: str, vectors: list[list[float]]
+) -> int:
+    """Replace a document's vectors, requiring one finite vector per indexed chunk."""
+    with _connect() as connection:
+        connection.row_factory = sqlite3.Row
+        document = connection.execute(
+            "SELECT id FROM knowledge_documents WHERE user_id=? AND workspace_path=? AND relative_path=?",
+            (user_id, str(workspace_path.resolve()), relative_path),
+        ).fetchone()
+        if document is None:
+            return 0
+        chunks = connection.execute(
+            "SELECT id FROM knowledge_chunks WHERE document_id=? ORDER BY chunk_index",
+            (document["id"],),
+        ).fetchall()
+        if len(chunks) != len(vectors):
+            raise ValueError("يجب أن يقابل كل مقطع مفهرس متجه تضمين واحد.")
+        prepared: list[tuple[int, str, int, bytes]] = []
+        dimensions: int | None = None
+        for chunk, vector in zip(chunks, vectors, strict=True):
+            if not vector or (dimensions is not None and len(vector) != dimensions):
+                raise ValueError("أبعاد متجهات التضمين غير صالحة أو غير متطابقة.")
+            dimensions = len(vector)
+            numeric = [float(value) for value in vector]
+            if not all(math.isfinite(value) for value in numeric) or not any(numeric):
+                raise ValueError("متجه التضمين يحتوي قيمة غير صالحة.")
+            packed = struct.pack(f"<{dimensions}f", *numeric)
+            prepared.append((int(chunk["id"]), model, dimensions, packed))
+        connection.execute(
+            "DELETE FROM knowledge_embeddings WHERE chunk_id IN "
+            "(SELECT id FROM knowledge_chunks WHERE document_id=?)",
+            (document["id"],),
+        )
+        connection.executemany(
+            "INSERT INTO knowledge_embeddings(chunk_id,model,dimensions,vector) VALUES(?,?,?,?)",
+            prepared,
+        )
+    return len(prepared)
+
+
+def has_embeddings(*, user_id: str, workspace_path: Path, model: str) -> bool:
+    with _connect() as connection:
+        row = connection.execute(
+            """SELECT 1 FROM knowledge_embeddings AS e
+               JOIN knowledge_chunks AS c ON c.id=e.chunk_id
+               JOIN knowledge_documents AS d ON d.id=c.document_id
+               WHERE d.user_id=? AND d.workspace_path=? AND e.model=? LIMIT 1""",
+            (user_id, str(workspace_path.resolve()), model),
+        ).fetchone()
+    return row is not None
+
+
+def search_by_embedding(
+    vector: list[float], *, user_id: str, workspace_path: Path, model: str, limit: int = 5
+) -> list[dict[str, Any]]:
+    if not vector or not all(math.isfinite(float(value)) for value in vector):
+        return []
+    query_norm = math.sqrt(sum(float(value) ** 2 for value in vector))
+    if query_norm == 0:
+        return []
+    with _connect() as connection:
+        connection.row_factory = sqlite3.Row
+        rows = connection.execute(
+            """SELECT d.id AS document_id, d.relative_path, d.content_hash,
+                      c.chunk_index, c.body, e.dimensions, e.vector
+               FROM knowledge_embeddings AS e
+               JOIN knowledge_chunks AS c ON c.id=e.chunk_id
+               JOIN knowledge_documents AS d ON d.id=c.document_id
+               WHERE d.user_id=? AND d.workspace_path=? AND e.model=?""",
+            (user_id, str(workspace_path.resolve()), model),
+        ).fetchall()
+    ranked: list[tuple[float, sqlite3.Row]] = []
+    for row in rows:
+        dimensions = int(row["dimensions"])
+        if dimensions != len(vector) or len(row["vector"]) != dimensions * 4:
+            continue
+        values = struct.unpack(f"<{dimensions}f", row["vector"])
+        norm = math.sqrt(sum(value * value for value in values))
+        if norm == 0:
+            continue
+        similarity = sum(float(left) * right for left, right in zip(vector, values, strict=True)) / (query_norm * norm)
+        ranked.append((similarity, row))
+    ranked.sort(key=lambda item: (-item[0], item[1]["relative_path"], item[1]["chunk_index"]))
+    bounded_limit = max(1, min(limit, 5))
+    selected: list[tuple[float, sqlite3.Row]] = []
+    per_document: dict[int, int] = {}
+    for score, row in ranked:
+        document_id = int(row["document_id"])
+        if per_document.get(document_id, 0) >= MAX_CHUNKS_PER_DOCUMENT:
+            continue
+        per_document[document_id] = per_document.get(document_id, 0) + 1
+        selected.append((score, row))
+        if len(selected) >= bounded_limit:
+            break
+    valid: dict[int, bool] = {}
+    results: list[dict[str, Any]] = []
+    for score, row in selected:
+        document_id = int(row["document_id"])
+        if document_id not in valid:
+            try:
+                path = workspace.relative_knowledge_file(workspace_path, str(row["relative_path"]))
+                valid[document_id] = hashlib.sha256(path.read_bytes()).hexdigest() == row["content_hash"]
+            except (OSError, ValueError):
+                valid[document_id] = False
+            if not valid[document_id]:
+                delete_document(
+                    user_id=user_id,
+                    workspace_path=workspace_path,
+                    relative_path=str(row["relative_path"]),
+                )
+        if valid[document_id]:
+            body = str(row["body"])
+            results.append(
+                {
+                    "path": row["relative_path"],
+                    "chunk": row["chunk_index"],
+                    "text": body,
+                    "excerpt": body[:220],
+                    "similarity": round(score, 4),
+                }
+            )
+    return results
+
+
+def merge_search_results(
+    lexical: list[dict[str, Any]], semantic: list[dict[str, Any]], *, limit: int = 5
+) -> list[dict[str, Any]]:
+    """Combine lexical and semantic ranks with reciprocal-rank fusion."""
+    combined: dict[tuple[str, int], dict[str, Any]] = {}
+    scores: dict[tuple[str, int], float] = {}
+    for rank, result in enumerate(lexical, 1):
+        key = (str(result["path"]), int(result["chunk"]))
+        combined[key] = dict(result)
+        scores[key] = scores.get(key, 0.0) + 1 / (60 + rank)
+    for rank, result in enumerate(semantic, 1):
+        key = (str(result["path"]), int(result["chunk"]))
+        if key not in combined:
+            combined[key] = dict(result)
+        else:
+            combined[key]["similarity"] = result.get("similarity")
+        scores[key] = scores.get(key, 0.0) + 1 / (60 + rank)
+    ordered = sorted(combined, key=lambda key: (-scores[key], key[0], key[1]))
+    return [combined[key] for key in ordered[: max(1, min(limit, 5))]]
+
+
+def delete_document(*, user_id: str, workspace_path: Path, relative_path: str) -> bool:
+    root = str(workspace_path.resolve())
+    with _connect() as connection:
+        connection.execute("PRAGMA foreign_keys = ON")
+        row = connection.execute(
+            "SELECT id FROM knowledge_documents WHERE user_id=? AND workspace_path=? AND relative_path=?",
+            (user_id, root, relative_path),
+        ).fetchone()
+        if row is None:
+            return False
+        document_id = int(row[0])
+        chunks = connection.execute(
+            "SELECT id FROM knowledge_chunks WHERE document_id=?", (document_id,)
+        ).fetchall()
+        connection.executemany("DELETE FROM knowledge_fts WHERE rowid=?", chunks)
+        connection.executemany("DELETE FROM knowledge_embeddings WHERE chunk_id=?", chunks)
+        connection.execute("DELETE FROM knowledge_documents WHERE id=?", (document_id,))
+    return True
+
+
+def search(query: str, *, user_id: str, workspace_path: Path, limit: int = 5) -> list[dict[str, Any]]:
+    base_terms = [
+        term
+        for term in dict.fromkeys(re.findall(r"[\w\u0600-\u06ff]{2,}", query.casefold()))
+        if term not in SEARCH_STOP_WORDS
+    ]
+    terms = list(
+        dict.fromkeys(
+            term
+            for base in base_terms
+            for term in (base, *QUERY_SYNONYMS.get(base, ()))
+        )
+    )[:20]
+    if not terms:
+        return []
+    match_query = " OR ".join('"' + term.replace('"', '""') + '"' for term in terms)
+    bounded_limit = max(1, min(limit, 5))
+    with _connect() as connection:
+        connection.row_factory = sqlite3.Row
+        rows = connection.execute(
+            """
+            SELECT d.id AS document_id, d.relative_path, d.content_hash,
+                   c.chunk_index, c.body,
+                   snippet(knowledge_fts, 0, '[', ']', ' … ', 24) AS excerpt,
+                   bm25(knowledge_fts) AS rank
+            FROM knowledge_fts
+            JOIN knowledge_chunks AS c ON c.id=knowledge_fts.rowid
+            JOIN knowledge_documents AS d ON d.id=c.document_id
+            WHERE knowledge_fts MATCH ? AND d.user_id=? AND d.workspace_path=?
+            ORDER BY rank, d.relative_path, c.chunk_index
+            LIMIT ?
+            """,
+            (match_query, user_id, str(workspace_path.resolve()), bounded_limit * 8),
+        ).fetchall()
+    selected_rows: list[sqlite3.Row] = []
+    chunks_per_document: dict[int, int] = {}
+    for row in rows:
+        document_id = int(row["document_id"])
+        if chunks_per_document.get(document_id, 0) >= MAX_CHUNKS_PER_DOCUMENT:
+            continue
+        chunks_per_document[document_id] = chunks_per_document.get(document_id, 0) + 1
+        selected_rows.append(row)
+        if len(selected_rows) >= bounded_limit:
+            break
+    current_documents: dict[int, bool] = {}
+    for row in selected_rows:
+        document_id = int(row["document_id"])
+        if document_id in current_documents:
+            continue
+        relative_path = str(row["relative_path"])
+        try:
+            source = workspace.relative_knowledge_file(workspace_path, relative_path)
+            actual_hash = hashlib.sha256(source.read_bytes()).hexdigest()
+            current_documents[document_id] = actual_hash == row["content_hash"]
+        except (OSError, ValueError):
+            current_documents[document_id] = False
+        if not current_documents[document_id]:
+            delete_document(
+                user_id=user_id,
+                workspace_path=workspace_path,
+                relative_path=relative_path,
+            )
+    return [
+        {"path": row["relative_path"], "chunk": row["chunk_index"], "text": row["body"], "excerpt": row["excerpt"]}
+        for row in selected_rows
+        if current_documents.get(int(row["document_id"]), False)
+    ]
+
+
+initialize()

SovereignAI-Starter/app/local_ocr.py

@@ -0,0 +1,167 @@
+"""Optional, local-only Arabic/English OCR for images and scanned documents."""
+
+from __future__ import annotations
+
+import os
+import re
+import threading
+import time
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+
+from PIL import Image, UnidentifiedImageError
+
+
+MAX_OCR_PIXELS = 2_000_000
+MAX_OCR_RESULTS = 120
+MAX_OCR_TEXT_CHARS = 12_000
+_OCR_LOCK = threading.Lock()
+_READER: Any | None = None
+_INITIALIZATION_FAILED = False
+_ARABIC_TEXT = re.compile(r"[\u0600-\u06ff]")
+
+
+class LocalOCRError(RuntimeError):
+    """OCR is not installed, its weights are unavailable, or the image is invalid."""
+
+
+def _sort_reading_order(detections: list[Any]) -> list[Any]:
+    """Group detected text into rows, then sort Arabic rows right-to-left."""
+    located = []
+    heights = []
+    for detection in detections:
+        try:
+            box, text, confidence = detection
+            points = [(float(point[0]), float(point[1])) for point in box]
+            xs = [point[0] for point in points]
+            ys = [point[1] for point in points]
+            center_x = (min(xs) + max(xs)) / 2
+            center_y = (min(ys) + max(ys)) / 2
+            box_height = max(1.0, max(ys) - min(ys))
+            heights.append(box_height)
+            located.append((center_y, center_x, box, str(text), float(confidence)))
+        except (TypeError, ValueError, IndexError, KeyError):
+            continue
+    if not located:
+        return []
+
+    heights.sort()
+    band = max(12.0, heights[len(heights) // 2] * 0.7)
+    rows: list[list[tuple[float, float, Any, str, float]]] = []
+    for item in sorted(located, key=lambda value: value[0]):
+        if not rows:
+            rows.append([item])
+            continue
+        current_y = sum(part[0] for part in rows[-1]) / len(rows[-1])
+        if item[0] - current_y <= band:
+            rows[-1].append(item)
+        else:
+            rows.append([item])
+
+    ordered = []
+    for row in rows:
+        rtl = any(_ARABIC_TEXT.search(item[3]) for item in row)
+        ordered.extend(sorted(row, key=lambda item: item[1], reverse=rtl))
+    return [(box, text, confidence) for _, _, box, text, confidence in ordered]
+
+
+def _get_reader() -> Any:
+    global _READER, _INITIALIZATION_FAILED
+    if _INITIALIZATION_FAILED:
+        raise LocalOCRError("محرك OCR المحلي غير متاح.")
+    if _READER is not None:
+        return _READER
+
+    with _OCR_LOCK:
+        if _INITIALIZATION_FAILED:
+            raise LocalOCRError("محرك OCR المحلي غير متاح.")
+        if _READER is not None:
+            return _READER
+        try:
+            import easyocr
+
+            configured_dir = os.getenv("LOCAL_OCR_MODEL_DIR", "").strip()
+            if configured_dir:
+                model_dir = Path(configured_dir).expanduser()
+            else:
+                local_app_data = os.getenv("LOCALAPPDATA")
+                base_dir = Path(local_app_data) if local_app_data else Path.home() / ".local" / "share"
+                model_dir = base_dir / "SovereignAI" / "models" / "easyocr"
+            model_dir.mkdir(parents=True, exist_ok=True)
+            user_network_dir = model_dir / "user_network"
+            user_network_dir.mkdir(parents=True, exist_ok=True)
+            allow_download = os.getenv("LOCAL_OCR_ALLOW_DOWNLOAD", "true").strip().lower() in {
+                "1", "true", "yes", "on"
+            }
+            _READER = easyocr.Reader(
+                ["ar", "en"],
+                gpu=False,
+                model_storage_directory=str(model_dir),
+                user_network_directory=str(user_network_dir),
+                download_enabled=allow_download,
+                verbose=False,
+            )
+            return _READER
+        except Exception as exc:
+            _INITIALIZATION_FAILED = True
+            raise LocalOCRError("تعذر تهيئة OCR المحلي؛ تحقق من تثبيت المتطلبات ووجود أوزان النموذج.") from exc
+
+
+def recognize_image_text(raw: bytes, *, label: str = "الصورة") -> dict[str, Any]:
+    """Read bounded Arabic/English text in memory without writing the upload to disk."""
+    if not raw:
+        raise LocalOCRError("ملف الصورة فارغ.")
+    try:
+        with Image.open(BytesIO(raw)) as source:
+            if source.width <= 0 or source.height <= 0:
+                raise LocalOCRError("أبعاد الصورة غير صالحة.")
+            if source.width * source.height > MAX_OCR_PIXELS:
+                raise LocalOCRError(f"تجاوزت {label} حد OCR البالغ {MAX_OCR_PIXELS} بكسل.")
+            source.load()
+            rgb_image = source.convert("RGB")
+    except LocalOCRError:
+        raise
+    except (UnidentifiedImageError, OSError, ValueError) as exc:
+        raise LocalOCRError("تعذر فك الصورة لإجراء OCR المحلي.") from exc
+
+    try:
+        import numpy as np
+
+        reader = _get_reader()
+        started = time.perf_counter()
+        # EasyOCR/PyTorch inference is serialized: one image at a time on this CPU.
+        with _OCR_LOCK:
+            detections = reader.readtext(
+                np.asarray(rgb_image),
+                detail=1,
+                paragraph=False,
+                batch_size=1,
+                workers=0,
+            )
+        detections = _sort_reading_order(detections)
+        lines = [
+            {
+                "text": str(text)[:1000],
+                "confidence": round(float(confidence), 3),
+                "box": [[round(float(point[0]), 1), round(float(point[1]), 1)] for point in box],
+            }
+            for box, text, confidence in detections[:MAX_OCR_RESULTS]
+            if str(text).strip()
+        ]
+        combined_text = "\n".join(item["text"] for item in lines)[:MAX_OCR_TEXT_CHARS]
+        return {
+            "engine": "easyocr-local-ar-en",
+            "text": combined_text,
+            "lines": lines,
+            "average_confidence": round(
+                sum(item["confidence"] for item in lines) / len(lines), 3
+            ) if lines else None,
+            "elapsed_seconds": round(time.perf_counter() - started, 2),
+        }
+    except LocalOCRError:
+        raise
+    except Exception as exc:
+        raise LocalOCRError("فشل OCR المحلي أثناء تحليل الصورة.") from exc
+    finally:
+        rgb_image.close()

SovereignAI-Starter/app/model_provider.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import os
 from collections.abc import AsyncIterator
@@ -28,6 +29,10 @@ class ModelProvider(Protocol):
 
     async def list_models(self) -> list[str]: ...
 
+    async def describe_models(
+        self, model_names: list[str]
+    ) -> dict[str, dict[str, Any]]: ...
+
 
 class OllamaProvider:
     name = "ollama"
@@ -62,6 +67,11 @@ class OllamaProvider:
                 status_code=502,
                 detail=f"خطأ من خادم النموذج المحلي: {detail}",
             ) from exc
+        except httpx.TimeoutException as exc:
+            raise HTTPException(
+                status_code=504,
+                detail="انتهت مهلة انتظار خادم النموذج المحلي.",
+            ) from exc
         except httpx.RequestError as exc:
             raise HTTPException(
                 status_code=503,
@@ -125,6 +135,63 @@ class OllamaProvider:
             if isinstance(item, dict) and item.get("name")
         ]
 
+    async def describe_models(
+        self, model_names: list[str]
+    ) -> dict[str, dict[str, Any]]:
+        """Read capability metadata advertised by the installed Ollama models."""
+        ollama_base = (
+            self.base_url[:-3] if self.base_url.endswith("/v1") else self.base_url
+        )
+        capability_order = [
+            "completion",
+            "vision",
+            "audio",
+            "tools",
+            "thinking",
+            "embedding",
+        ]
+        semaphore = asyncio.Semaphore(4)
+
+        async def describe(client: httpx.AsyncClient, model_name: str):
+            async with semaphore:
+                try:
+                    response = await client.post(
+                        f"{ollama_base}/api/show",
+                        json={"model": model_name, "verbose": False},
+                    )
+                    response.raise_for_status()
+                    payload = response.json()
+                    raw = (
+                        payload.get("capabilities")
+                        if isinstance(payload, dict)
+                        else None
+                    )
+                    verified = isinstance(raw, list)
+                    return model_name, {
+                        "verified": verified,
+                        "capabilities": [
+                            capability
+                            for capability in capability_order
+                            if capability in raw
+                        ]
+                        if verified
+                        else [],
+                    }
+                except (httpx.HTTPError, ValueError, TypeError):
+                    return model_name, {"verified": False, "capabilities": []}
+
+        try:
+            async with httpx.AsyncClient(timeout=10.0) as client:
+                described = await asyncio.gather(
+                    *(describe(client, name) for name in model_names)
+                )
+                return dict(described)
+        except httpx.HTTPError:
+            return {
+                name: {"verified": False, "capabilities": []}
+                for name in model_names
+            }
+
 
 def get_model_provider() -> ModelProvider:
     provider_name = os.getenv("MODEL_PROVIDER", "ollama").strip().lower()

SovereignAI-Starter/app/pdf_documents.py

@@ -0,0 +1,203 @@
+"""Safe extraction of embedded text from a bounded, user-selected PDF."""
+
+from __future__ import annotations
+
+import base64
+import math
+import threading
+from io import BytesIO
+from typing import Any
+
+import pypdfium2 as pdfium
+from pypdf import PdfReader, apply_configuration
+from pypdf.errors import LimitReachedError, PdfReadError
+from PIL import Image
+
+
+MAX_PDF_PAGES = 30
+MAX_SCANNED_PAGES = 3
+MAX_RENDERED_PAGE_PIXELS = 2_000_000
+MAX_RENDERED_TOTAL_BYTES = 8_000_000
+PDF_RENDER_DPI = 144
+_PDFIUM_LOCK = threading.Lock()
+
+
+class PdfDocumentError(ValueError):
+    def __init__(self, status_code: int, message: str) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+
+
+def extract_pdf_text(
+    raw: bytes,
+    name: str,
+    *,
+    max_pages: int = MAX_PDF_PAGES,
+    max_chars: int = 24_000,
+    allow_empty: bool = False,
+) -> str:
+    parsed = extract_pdf_pages_text(raw, name, max_pages=max_pages, max_chars=max_chars)
+    sections = [
+        f"--- صفحة {page['page']} ---\n{page['text']}"
+        for page in parsed["pages"]
+        if page["text"]
+    ]
+    if parsed["truncated"]:
+        sections.append(f"[اقتُصر النص المستخرج على {max_chars} محرف.]")
+    if not sections and not allow_empty:
+        raise PdfDocumentError(
+            415,
+            f"لا يحتوي PDF على نص قابل للاستخراج؛ قد تكون صفحاته صورًا ممسوحة: {name}.",
+        )
+    return "\n\n".join(sections)
+
+
+def extract_pdf_pages_text(
+    raw: bytes,
+    name: str,
+    *,
+    max_pages: int = MAX_PDF_PAGES,
+    max_chars: int = 24_000,
+) -> dict[str, Any]:
+    """Extract bounded text per page while preserving which pages are image-only."""
+    if not raw.startswith(b"%PDF-"):
+        raise PdfDocumentError(415, f"ترويسة ملف PDF غير صالحة: {name}.")
+    try:
+        with apply_configuration(
+            maximum_declared_stream_length=8_000_000,
+            array_based_stream_maximum_output_length=8_000_000,
+            zlib_maximum_output_length=8_000_000,
+            lzw_maximum_output_length=8_000_000,
+            run_length_maximum_output_length=8_000_000,
+            jbig2_maximum_output_length=8_000_000,
+            image_maximum_buffer_size=8_000_000,
+            page_tree_maximum_entries=1_000,
+            page_tree_maximum_depth=30,
+        ):
+            reader = PdfReader(BytesIO(raw), strict=True)
+            if reader.is_encrypted:
+                raise PdfDocumentError(415, f"ملف PDF محمي بكلمة مرور وغير مدعوم: {name}.")
+            if len(reader.pages) > max_pages:
+                raise PdfDocumentError(413, f"الحد الأقصى {max_pages} صفحة لكل PDF: {name}.")
+            pages: list[dict[str, Any]] = []
+            extracted_length = 0
+            truncated = False
+            for page_number, page in enumerate(reader.pages, 1):
+                text = (page.extract_text() or "").strip()
+                if not text:
+                    pages.append({"page": page_number, "text": "", "has_text": False})
+                    continue
+                section = f"--- صفحة {page_number} ---\n{text}"
+                remaining = max_chars - extracted_length
+                if remaining <= 0:
+                    truncated = True
+                    pages.append({"page": page_number, "text": "", "has_text": True})
+                    continue
+                if len(section) > remaining:
+                    section = section[:remaining]
+                    truncated = True
+                page_text = section.split("\n", 1)[1] if "\n" in section else ""
+                pages.append({"page": page_number, "text": page_text, "has_text": True})
+                extracted_length += len(section)
+                if extracted_length >= max_chars:
+                    truncated = truncated or page_number < len(reader.pages)
+                    if truncated:
+                        for later_page_number in range(page_number + 1, len(reader.pages) + 1):
+                            pages.append(
+                                {"page": later_page_number, "text": "", "has_text": True}
+                            )
+                    break
+    except PdfDocumentError:
+        raise
+    except (PdfReadError, LimitReachedError, ValueError, KeyError) as exc:
+        raise PdfDocumentError(415, f"تعذر قراءة بنية PDF: {name}.") from exc
+    return {
+        "pages": pages,
+        "total_pages": len(pages),
+        "truncated": truncated,
+    }
+
+
+def render_scanned_pdf_pages(
+    raw: bytes,
+    name: str,
+    *,
+    max_pages: int = MAX_SCANNED_PAGES,
+    page_numbers: list[int] | None = None,
+) -> dict[str, Any]:
+    """Render selected PDF pages to bounded JPEG data for local vision/OCR."""
+    try:
+        # pypdfium2/PDFium is not thread-safe; serialize all native PDFium work.
+        with _PDFIUM_LOCK:
+            document = pdfium.PdfDocument(raw)
+            try:
+                total_pages = len(document)
+                if total_pages > MAX_PDF_PAGES:
+                    raise PdfDocumentError(413, f"الحد الأقصى {MAX_PDF_PAGES} صفحة لكل PDF: {name}.")
+                page_limit = max(0, min(max_pages, MAX_SCANNED_PAGES))
+                if page_numbers is None:
+                    selected_pages = list(range(1, min(total_pages, page_limit) + 1))
+                    truncated = total_pages > len(selected_pages)
+                else:
+                    if len(page_numbers) > page_limit:
+                        raise PdfDocumentError(
+                            413,
+                            f"الحد الأقصى {MAX_SCANNED_PAGES} صفحة ممسوحة للتحليل في الطلب الواحد.",
+                        )
+                    if (
+                        len(set(page_numbers)) != len(page_numbers)
+                        or any(number < 1 or number > total_pages for number in page_numbers)
+                    ):
+                        raise PdfDocumentError(422, f"أرقام صفحات PDF المحددة غير صالحة: {name}.")
+                    selected_pages = page_numbers
+                    truncated = False
+                pages: list[dict[str, Any]] = []
+                total_rendered_bytes = 0
+                for page_number in selected_pages:
+                    index = page_number - 1
+                    page = document[index]
+                    bitmap = None
+                    try:
+                        width, height = page.get_size()
+                        if width <= 0 or height <= 0:
+                            continue
+                        target_scale = PDF_RENDER_DPI / 72
+                        pixel_scale = math.sqrt(MAX_RENDERED_PAGE_PIXELS / (width * height))
+                        scale = min(target_scale, pixel_scale)
+                        bitmap = page.render(scale=scale, rev_byteorder=True, limit_image_cache=True)
+                        image = bitmap.to_pil().convert("RGB")
+                        output = BytesIO()
+                        try:
+                            image.save(output, format="JPEG", quality=82, optimize=True)
+                        finally:
+                            image.close()
+                        jpeg = output.getvalue()
+                        if len(jpeg) > 4_000_000:
+                            raise PdfDocumentError(413, f"حجم الصفحة الممسوحة كبير بعد التحويل: {name}.")
+                        total_rendered_bytes += len(jpeg)
+                        if total_rendered_bytes > MAX_RENDERED_TOTAL_BYTES:
+                            raise PdfDocumentError(413, "تجاوز مجموع الصور المحولة من PDF حد 8 ميغابايت.")
+                        pages.append(
+                            {
+                                "page": page_number,
+                                "mime_type": "image/jpeg",
+                                "data": base64.b64encode(jpeg).decode("ascii"),
+                            }
+                        )
+                    finally:
+                        if bitmap is not None:
+                            bitmap.close()
+                        page.close()
+                if not pages:
+                    raise PdfDocumentError(415, f"لا توجد صفحات PDF قابلة للتحويل إلى صورة: {name}.")
+                return {
+                    "pages": pages,
+                    "total_pages": total_pages,
+                    "truncated": truncated,
+                }
+            finally:
+                document.close()
+    except PdfDocumentError:
+        raise
+    except Exception as exc:
+        raise PdfDocumentError(415, f"تعذر تحويل صفحات PDF الممسوح إلى صور: {name}.") from exc

SovereignAI-Starter/app/skills.py

@@ -0,0 +1,62 @@
+"""Curated, local agent skills with explicit tool permissions."""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class AgentSkill:
+    id: str
+    name: str
+    description: str
+    instructions: str
+    allowed_tools: frozenset[str]
+
+    def public_metadata(self) -> dict[str, object]:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "description": self.description,
+            "allowed_tools": sorted(self.allowed_tools),
+        }
+
+
+SKILLS: dict[str, AgentSkill] = {
+    "code_explain": AgentSkill(
+        id="code_explain",
+        name="شرح الكود",
+        description="يفسر الغرض وتدفق البيانات والدوال المهمة، مع الاستشهاد بالمسار والسطر عند توفرهما.",
+        instructions=(
+            "اشرح الملفات المحددة على مراحل: الغرض العام، تدفق البيانات، ثم أهم الدوال. "
+            "استشهد بمسار الملف ورقم السطر عندما يظهر في المقتطف، وميّز بين ما قرأته وما استنتجته. "
+            "استعمل البحث المعرفي للوثائق المفهرسة عند الحاجة. لا تقترح تعديلًا ولا تطلب أداة كتابة في هذه المهارة."
+        ),
+        allowed_tools=frozenset({"calculator", "search_workspace", "search_knowledge"}),
+    ),
+    "code_review": AgentSkill(
+        id="code_review",
+        name="مراجعة الكود",
+        description="يركز على العيوب القابلة للتحقق والأمان والأخطاء الحدية، ويصنفها حسب الخطورة.",
+        instructions=(
+            "راجع الكود بحثًا عن عيوب قابلة لإعادة الإنتاج في الصحة والأمان ومعالجة الأخطاء والحدود. "
+            "اعرض النتائج أولًا مرتبة حسب الخطورة، مع المسار والسطر والدليل والأثر. "
+            "لا تختلق مشاكل أسلوبية. لا تطلب propose_file_change إلا إذا طلب المستخدم صراحةً إصلاحًا أو تعديلًا؛ "
+            "وحتى عندئذٍ لا تطبق الكتابة، بل حضّر معاينة diff فقط."
+        ),
+        allowed_tools=frozenset({"calculator", "search_workspace", "search_knowledge", "propose_file_change"}),
+    ),
+    "test_plan": AgentSkill(
+        id="test_plan",
+        name="خطة اختبارات",
+        description="يستخرج حالات النجاح والحواف والفشل من الكود ويكتب خطوات ونتائج متوقعة دون تشغيلها.",
+        instructions=(
+            "أنشئ خطة اختبارات قابلة للتنفيذ يدويًا: المسار الطبيعي، المدخلات الحدية، وحالات الفشل. "
+            "لكل حالة اذكر الإعداد والمدخل والنتيجة المتوقعة. لا تدّع أنك شغّلت اختبارًا ولا تشغّل أوامر. "
+            "اربط الحالات بالدوال أو الملفات التي قرأتها فقط."
+        ),
+        allowed_tools=frozenset({"search_workspace", "search_knowledge"}),
+    ),
+}
+
+
+def get_skill(skill_id: str | None) -> AgentSkill | None:
+    return SKILLS.get(skill_id) if skill_id is not None else None

SovereignAI-Starter/app/workspace.py

@@ -4,7 +4,13 @@ from __future__ import annotations
 
 import os
 import re
+import difflib
+import hashlib
+import tempfile
+import time
 from pathlib import Path
+from pathlib import PurePosixPath
+from uuid import uuid4
 
 ALLOWED_SUFFIXES = {
     ".py", ".dart", ".md", ".txt", ".json", ".yaml", ".yml", ".toml",
@@ -16,6 +22,11 @@ IGNORED_PARTS = {
 }
 MAX_FILE_BYTES = 256 * 1024
 MAX_SCAN_FILES = 500
+PROPOSAL_TTL_SECONDS = 10 * 60
+MAX_PENDING_PROPOSALS = 32
+
+
+_pending_changes: dict[str, dict[str, object]] = {}
 
 
 def configured_root() -> Path | None:
@@ -26,6 +37,23 @@ def configured_root() -> Path | None:
     return root if root.is_dir() else None
 
 
+def selected_root(value: str | None) -> Path | None:
+    """Resolve an explicitly selected directory, falling back to server config."""
+    if value is None or not value.strip():
+        return configured_root()
+    try:
+        root = Path(value).expanduser().resolve(strict=True)
+    except (OSError, RuntimeError) as exc:
+        raise ValueError("مجلد مساحة العمل المحدد غير موجود أو غير متاح.") from exc
+    if not root.is_dir():
+        raise ValueError("يجب اختيار مجلد صالح لمساحة العمل.")
+    if root == Path(root.anchor):
+        raise ValueError("اختر مجلد مشروع محددًا، وليس جذر القرص.")
+    if root.name.startswith(".") or root.name in IGNORED_PARTS:
+        raise ValueError("لا يمكن استخدام مجلد مخفي أو مستثنى كمساحة عمل.")
+    return root
+
+
 def relative_file(root: Path, relative_path: str) -> Path:
     candidate = (root / relative_path).resolve(strict=True)
     try:
@@ -44,6 +72,193 @@ def relative_file(root: Path, relative_path: str) -> Path:
     return candidate
 
 
+def relative_knowledge_file(root: Path, relative_path: str) -> Path:
+    """Resolve a bounded UTF-8 document or PDF selected for local knowledge use."""
+    candidate = (root / relative_path).resolve(strict=True)
+    try:
+        candidate.relative_to(root)
+    except ValueError as exc:
+        raise ValueError("المسار المطلوب خارج مساحة العمل.") from exc
+    if not candidate.is_file() or candidate.suffix.lower() not in (ALLOWED_SUFFIXES | {".pdf"}):
+        raise ValueError("هذا النوع من الملفات غير مسموح بفهرسته.")
+    if any(
+        part in IGNORED_PARTS or part.startswith(".")
+        for part in candidate.relative_to(root).parts
+    ):
+        raise ValueError("قراءة الملفات المخفية أو المستثناة غير مسموحة.")
+    if candidate.stat().st_size > MAX_FILE_BYTES:
+        raise ValueError("الملف أكبر من الحد المسموح للقراءة (256 كيلوبايت).")
+    return candidate
+
+
+def _write_target(root: Path, relative_path: str, operation: str) -> tuple[Path, bytes]:
+    if (
+        len(relative_path) > 1024
+        or "\x00" in relative_path
+        or any(char in relative_path for char in '<>:"|?*')
+    ):
+        raise ValueError("مسار الملف غير صالح.")
+    normalized = relative_path.replace("\\", "/")
+    relative = PurePosixPath(normalized)
+    if (
+        relative.is_absolute()
+        or not relative.parts
+        or any(part in {"", ".", ".."} for part in relative.parts)
+        or any(part.startswith(".") or part in IGNORED_PARTS for part in relative.parts)
+    ):
+        raise ValueError("يسمح بالكتابة داخل مسارات نسبية غير مخفية في مساحة العمل فقط.")
+    reserved_names = {"CON", "PRN", "AUX", "NUL"} | {
+        f"{prefix}{number}"
+        for prefix in ("COM", "LPT")
+        for number in range(1, 10)
+    }
+    if any(
+        part.endswith((".", " ")) or part.split(".", 1)[0].upper() in reserved_names
+        for part in relative.parts
+    ):
+        raise ValueError("اسم الملف غير صالح على Windows.")
+    if relative.suffix.lower() not in ALLOWED_SUFFIXES:
+        raise ValueError("امتداد الملف غير مسموح للوكيل.")
+
+    root = root.resolve(strict=True)
+    target = root.joinpath(*relative.parts)
+    current = root
+    for part in relative.parts[:-1]:
+        current = current / part
+        if current.is_symlink():
+            raise ValueError("لا يسمح بالكتابة عبر مجلدات الروابط الرمزية.")
+    if not target.parent.is_dir():
+        raise ValueError("يجب أن يكون المجلد الأب موجودًا؛ لا ينشئ الوكيل مجلدات تلقائيًا.")
+    resolved_parent = target.parent.resolve(strict=True)
+    try:
+        resolved_parent.relative_to(root)
+    except ValueError as exc:
+        raise ValueError("مجلد الملف خارج مساحة العمل المحددة.") from exc
+
+    if target.is_symlink():
+        raise ValueError("لا يسمح باستبدال ملف رابط رمزي.")
+    exists = target.exists()
+    if operation == "create" and exists:
+        raise ValueError("الملف موجود بالفعل؛ اطلب تحديثه بدل إنشائه.")
+    if operation == "update" and not exists:
+        raise ValueError("الملف المراد تحديثه غير موجود.")
+    if operation not in {"create", "update"}:
+        raise ValueError("نوع التغيير غير مسموح.")
+    if not exists:
+        return target, b""
+
+    safe_target = relative_file(root, relative.as_posix())
+    raw = safe_target.read_bytes()
+    return safe_target, raw
+
+
+def create_change_preview(
+    root: Path,
+    relative_path: str,
+    operation: str,
+    content: str,
+) -> dict[str, object]:
+    """Build and retain a short-lived diff; this function never writes the file."""
+    raw_content = content.encode("utf-8")
+    if len(raw_content) > MAX_FILE_BYTES:
+        raise ValueError("المحتوى المقترح يتجاوز حد 256 كيلوبايت.")
+    target, original = _write_target(root, relative_path, operation)
+    try:
+        old_text = original.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise ValueError("لا يمكن تحديث ملف غير محفوظ بترميز UTF-8.") from exc
+
+    newline = "\r\n" if b"\r\n" in original else "\n"
+    proposed_text = content.replace("\r\n", "\n").replace("\n", newline)
+    proposed_bytes = proposed_text.encode("utf-8")
+    if operation == "update" and original == proposed_bytes:
+        raise ValueError("المحتوى المقترح مطابق للملف الحالي ولا يحتاج إلى تعديل.")
+
+    relative = relative_path.replace("\\", "/")
+    before = old_text.splitlines(keepends=True)
+    after = proposed_text.splitlines(keepends=True)
+    diff = "".join(
+        difflib.unified_diff(
+            before,
+            after,
+            fromfile=f"a/{relative}" if operation == "update" else "/dev/null",
+            tofile=f"b/{relative}",
+            lineterm="",
+        )
+    )
+    expired = [
+        key for key, item in _pending_changes.items()
+        if float(item["expires_at"]) <= time.time()
+    ]
+    for key in expired:
+        _pending_changes.pop(key, None)
+    if len(_pending_changes) >= MAX_PENDING_PROPOSALS:
+        raise ValueError("هناك عدد كبير من معاينات التغيير المعلقة؛ ألغِ بعضها قبل إنشاء معاينة أخرى.")
+    token = str(uuid4())
+    _pending_changes[token] = {
+        "root": str(root.resolve(strict=True)),
+        "path": relative,
+        "operation": operation,
+        "content": proposed_bytes,
+        "expected_hash": hashlib.sha256(original).hexdigest(),
+        "expires_at": time.time() + PROPOSAL_TTL_SECONDS,
+    }
+    return {
+        "token": token,
+        "path": relative,
+        "operation": operation,
+        "diff": diff,
+        "expires_in_seconds": PROPOSAL_TTL_SECONDS,
+    }
+
+
+def apply_change_preview(token: str) -> dict[str, str]:
+    """Apply a reviewed proposal once, only if its target is still unchanged."""
+    proposal = _pending_changes.pop(token, None)
+    if proposal is None or float(proposal["expires_at"]) <= time.time():
+        raise ValueError("انتهت صلاحية معاينة التغيير أو استُخدمت مسبقًا؛ أنشئ معاينة جديدة.")
+
+    root = Path(str(proposal["root"])).resolve(strict=True)
+    relative_path = str(proposal["path"])
+    operation = str(proposal["operation"])
+    target, current = _write_target(root, relative_path, operation)
+    current_hash = hashlib.sha256(current).hexdigest()
+    if current_hash != proposal["expected_hash"]:
+        raise ValueError("تغير الملف منذ عرض المعاينة؛ أنشئ diff جديدًا قبل التطبيق.")
+
+    content = proposal["content"]
+    if not isinstance(content, bytes):
+        raise ValueError("بيانات المعاينة غير صالحة.")
+    temporary_path: str | None = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode="wb",
+            prefix=".sovereignai-review-",
+            suffix=".tmp",
+            dir=target.parent,
+            delete=False,
+        ) as temporary_file:
+            temporary_path = temporary_file.name
+            temporary_file.write(content)
+            temporary_file.flush()
+            os.fsync(temporary_file.fileno())
+        if operation == "update":
+            os.chmod(temporary_path, target.stat().st_mode)
+        # Recheck to avoid clobbering edits made while the temporary file was written.
+        _, latest = _write_target(root, relative_path, operation)
+        if hashlib.sha256(latest).hexdigest() != proposal["expected_hash"]:
+            raise ValueError("تغير الملف أثناء التطبيق؛ لم تُحفظ المعاينة.")
+        os.replace(temporary_path, target)
+        temporary_path = None
+    finally:
+        if temporary_path is not None:
+            try:
+                os.unlink(temporary_path)
+            except OSError:
+                pass
+    return {"path": relative_path, "operation": operation, "status": "applied"}
+
+
 def list_text_files(root: Path) -> list[Path]:
     files: list[Path] = []
     for current, directories, filenames in os.walk(root, followlinks=False):
@@ -65,6 +280,27 @@ def list_text_files(root: Path) -> list[Path]:
     return files
 
 
+def list_knowledge_files(root: Path) -> list[Path]:
+    files: list[Path] = []
+    for current, directories, filenames in os.walk(root, followlinks=False):
+        directories[:] = [
+            name for name in directories
+            if name not in IGNORED_PARTS and not name.startswith(".")
+        ]
+        for filename in filenames:
+            path = Path(current) / filename
+            if path.suffix.lower() not in (ALLOWED_SUFFIXES | {".pdf"}):
+                continue
+            try:
+                relative_knowledge_file(root, path.relative_to(root).as_posix())
+            except (OSError, ValueError):
+                continue
+            files.append(path)
+            if len(files) >= MAX_SCAN_FILES:
+                return files
+    return files
+
+
 def retrieve(task: str, root: Path, limit: int = 3) -> list[tuple[str, str]]:
     terms = {
         term.casefold()