feat: sync scripts, scraper, and sql tools from server

2026-08-16 13:19:10 +03:00
parent 32865e78df
commit 3136a2b7aa
14 changed files with 28844 additions and 0 deletions
@@ -0,0 +1,8 @@
+nohup: ignoring input
+🏙️ [PROD] Starting full Damascus scan on Mac (Low-Resource)...
+
+🚀 PASS: COMMERCIAL_PUBLIC (Low-Resource Mode)
+📋 [DEBUG] Grid: 44x66 blocks. Starting...
+🔎 [TRACE] Scanning Block 0,0 | 33.45000, 36.17000
+      🛡️  Bypassing Google Consent (ID: #L2AGLb)...
+      🛡️  Bypassing Google Consent (ID: #L2AGLb)...
@@ -0,0 +1,227 @@
+import asyncio
+import os
+import requests
+import json
+import re
+import math
+import time
+from urllib.parse import unquote
+from playwright.async_api import async_playwright
+from dotenv import load_dotenv
+
+# Environment Setup
+load_dotenv()
+API_URL = os.getenv("API_URL", "http://localhost:3200/api")
+API_KEY = os.getenv("API_KEY", "intaleq_secret_2026")
+
+# PRODUCTION SCAN CALIBRATION (Damascus Pro - High Performance Mac Edition)
+DAMASCUS_BOUNDS = {
+    "min_lat": 33.4500,
+    "max_lat": 33.5800,
+    "min_lng": 36.1700,
+    "max_lng": 36.4000
+}
+
+PASSES = {
+    "RESIDENTIAL": {
+        "categories": ["منزل", "بيت", "بناء سكني", "دخلة", "حارة"],
+        "lat_step": 0.0015,
+        "lng_step": 0.0018,
+        "zoom": 20,
+        "scrolls": 4
+    },
+    "COMMERCIAL_PUBLIC": {
+        "categories": ["سوق", "محل", "مطعم", "كافيه", "صيدلية", "عيادة", "جامع", "وزارة", "مدرسة"],
+        "lat_step": 0.0030,
+        "lng_step": 0.0035,
+        "zoom": 18,
+        "scrolls": 3
+    }
+}
+
+class DamascusProScanner:
+    def __init__(self, bounds: dict):
+        self.bounds = bounds
+        self.global_seen = set()
+        self.coord_pattern = re.compile(r"!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)|/@(-?\d+\.\d+),(-?\d+\.\d+)")
+
+    async def route_interceptor(self, route):
+        """السماح بكافة الموارد على الـ Mac لضمان دقة النتائج"""
+        return await route.continue_()
+
+    def calculate_distance(self, lat1, lon1, lat2, lon2):
+        R = 6371
+        d_lat = math.radians(lat2 - lat1)
+        d_lon = math.radians(lon2 - lon1)
+        a = math.sin(d_lat / 2)**2 + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * math.sin(d_lon / 2)**2
+        return R * (2 * math.asin(math.sqrt(a)))
+
+    async def bypass_consent(self, page):
+        """تخطي صفحة موافقة جوجل بمختلف أشكالها (للعمل بسلاسة في السيرفر)"""
+        try:
+            selectors = [
+                "#L2AGLb", 
+                "button[aria-label*='Accept']", 
+                "button:has-text('أوافق')", 
+                "button:has-text('I agree')",
+                "form[action*='consent'] button"
+            ]
+            for selector in selectors:
+                if await page.locator(selector).count() > 0:
+                    print(f"      🛡️  Bypassing Google Consent (Selector: {selector})...", flush=True)
+                    await page.click(selector)
+                    await page.wait_for_timeout(2000)
+                    return True
+            return False
+        except:
+            return False
+
+    async def wait_for_results(self, page):
+        """الانتظار الذكي لنتائج البحث أو صفحة الخريطة"""
+        try:
+            await page.wait_for_selector('a[href*="/maps/place/"]', timeout=15000)
+            return True
+        except:
+            return False
+
+    async def extract_places(self, page, center_lat, center_lng, pass_name) -> list[dict]:
+        await self.bypass_consent(page)
+        
+        # التمرير لأسفل لتحميل المزيد من النتائج
+        for _ in range(PASSES[pass_name].get('scrolls', 3)):
+            await page.mouse.wheel(0, 4000)
+            await page.wait_for_timeout(1000)
+
+        # الانتظار حتى تظهر النتائج فعلياً
+        await self.wait_for_results(page)
+        
+        places = []
+        elements = await page.query_selector_all('a[href*="/maps/place/"]')
+        if len(elements) > 0:
+            print(f"      ✨ Found {len(elements)} potential results on page", flush=True)
+
+        for el in elements:
+            try:
+                href = await el.get_attribute('href')
+                if not href: continue
+                name_match = re.search(r"/place/([^/]+)", href)
+                if not name_match: continue
+                name = unquote(name_match.group(1)).replace('+', ' ').strip()
+                lat, lng = None, None
+                matches = self.coord_pattern.findall(href)
+                for m in matches:
+                    valid = [val for val in m if val]
+                    if len(valid) == 2:
+                        lat, lng = float(valid[0]), float(valid[1])
+                        break
+                if name and lat and lng:
+                    dist = self.calculate_distance(center_lat, center_lng, lat, lng)
+                    if dist > 3.0: continue # زيادة المدى قليلاً
+                    FORBIDDEN = ["google", "تكبير", "تصغير", "بحث", "مساحة", "إبلاغ", "عن"]
+                    if any(key in name.lower() for key in FORBIDDEN): continue
+                    dedup_key = f"{name.lower()}|{round(lat, 5)}|{round(lng, 5)}"
+                    if dedup_key not in self.global_seen:
+                        self.global_seen.add(dedup_key)
+                        places.append({
+                            "name": name, "name_ar": name, "latitude": lat, "longitude": lng,
+                            "category": pass_name.title(), "city": "Damascus", "source": "scraper_admin"
+                        })
+            except: continue
+        return places
+
+    def send_batch(self, places: list[dict]):
+        if not places: return
+        try:
+            res = requests.post(
+                f"{API_URL}/geocoding/upsert-batch", 
+                headers={"X-API-Key": API_KEY}, 
+                json={"places": places}, 
+                timeout=20
+            )
+            if res.status_code in [200, 201]:
+                print(f"   ✅ [SYNC SUCCESS] {len(places)} points added. Response: {res.text[:50]}", flush=True)
+            else:
+                print(f"   ❌ [SYNC ERROR] Status: {res.status_code} | Body: {res.text}", flush=True)
+        except Exception as e:
+            print(f"   ⚠️ [SYNC EXCEPTION] {str(e)}", flush=True)
+
+    async def run_pass(self, pass_name: str, test_mode=False):
+        cfg = PASSES[pass_name]
+        print(f"\n🚀 PASS: {pass_name} (Low-Resource Mode)", flush=True)
+        async with async_playwright() as p:
+            user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+            
+            # استخدام مجلد دائم لحفظ الـ Cookies والحالة لضمان تجاوز الحجب
+            user_data_dir = os.path.join(os.getcwd(), "browser_state")
+            context = await p.chromium.launch_persistent_context(
+                user_data_dir,
+                headless=not test_mode,
+                viewport={"width": 1024, "height": 768}, 
+                locale="ar-SA",
+                user_agent=user_agent
+            )
+            
+            # حدود البحث (تصغير المنطقة جداً في وضع الاختبار)
+            bounds = {
+                "min_lat": 33.5100, "max_lat": 33.5110,
+                "min_lng": 36.2900, "max_lng": 36.2915
+            } if test_mode else self.bounds
+            
+            rows = math.ceil((bounds['max_lat'] - bounds['min_lat']) / cfg['lat_step'])
+            cols = math.ceil((bounds['max_lng'] - bounds['min_lng']) / cfg['lng_step'])
+            
+            print(f"📋 [DEBUG] Grid: {rows}x{cols} blocks. Starting...", flush=True)
+
+            for r in range(rows):
+                for c in range(cols):
+                    lat = bounds['min_lat'] + (r * cfg['lat_step'])
+                    lng = bounds['min_lng'] + (c * cfg['lng_step'])
+                    
+                    print(f"🔎 [TRACE] Scanning Block {r},{c} | {lat:.5f}, {lng:.5f}", flush=True)
+                    
+                    page = await context.new_page()
+                    await page.route("**/*", self.route_interceptor)
+                    
+                    for cat in cfg['categories']:
+                        url = f"https://www.google.com/maps/search/{cat}/@{lat},{lng},{cfg['zoom']}z?hl=ar"
+                        try:
+                            await page.goto(url, wait_until="domcontentloaded", timeout=40000)
+                            
+                            # Trace & Handle Consent via CSS ID (More reliable)
+                            title = await page.title()
+                            if "قبل المتابعة" in title or "Before you continue" in title or "Consent" in title:
+                                print(f"      🛡️  Bypassing Google Consent (ID: #L2AGLb)...", flush=True)
+                                try:
+                                    await page.click("button#L2AGLb")
+                                    await page.wait_for_load_state("networkidle", timeout=10000)
+                                except: pass
+
+                            # الانتظار حتى تظهر أول نتيجة بحث أو انتهاء الوقت
+                            try:
+                                await page.wait_for_selector('a[href*="/maps/place/"]', timeout=8000)
+                            except: pass
+
+                            for _ in range(cfg['scrolls']):
+                                await page.mouse.wheel(0, 1000)
+                                await asyncio.sleep(0.5 if test_mode else 0.4)
+                            
+                            found = await self.extract_places(page, lat, lng, pass_name)
+                            if found:
+                                self.send_batch(found)
+                                print(f" ✅ [{pass_name}] Block {r},{c} | Found {len(found)} results", flush=True)
+                        except Exception as e:
+                            print(f" ⚠️ [ERROR] {str(e)[:40]}", flush=True)
+                    
+                    await page.close()
+                    await asyncio.sleep(1.0 if test_mode else 0.6)
+            await browser.close()
+
+if __name__ == "__main__":
+    scanner = DamascusProScanner(DAMASCUS_BOUNDS)
+    async def main():
+        print("🏙️ [PROD] Starting full Damascus scan on Mac (Low-Resource)...", flush=True)
+        # تشغيل المسح التجاري أولاً
+        await scanner.run_pass("COMMERCIAL_PUBLIC", test_mode=False)
+        # ثم المسح السكني الكثيف
+        await scanner.run_pass("RESIDENTIAL", test_mode=False)
+    asyncio.run(main())
@@ -0,0 +1,173 @@
+import asyncio
+import os
+import requests
+import json
+import re
+import math
+import urllib.parse
+from urllib.parse import unquote
+from playwright.async_api import async_playwright
+from dotenv import load_dotenv
+
+# Environment Setup
+load_dotenv()
+API_URL = os.getenv("API_URL", "http://localhost:3200/api") # القيمة الافتراضية للسيرفر
+API_KEY = os.getenv("API_KEY", "intaleq_secret_2026")
+
+# PRODUCTION SCAN CALIBRATION (Damascus Pro - High Performance Server Edition)
+DAMASCUS_BOUNDS = {
+    "min_lat": 33.4500,
+    "max_lat": 33.5800,
+    "min_lng": 36.1700,
+    "max_lng": 36.4000
+}
+
+PASSES = {
+    "COMMERCIAL_PUBLIC": {
+        "categories": ["سوق", "محل", "مطعم", "كافيه", "صيدلية", "عيادة", "جامع", "وزارة", "مدرسة"],
+        "lat_step": 0.0030,
+        "lng_step": 0.0035,
+        "zoom": 18,
+        "scrolls": 3
+    },
+    "RESIDENTIAL": {
+        "categories": ["منزل", "بيت", "بناء سكني", "دخلة", "حارة"],
+        "lat_step": 0.0015,
+        "lng_step": 0.0018,
+        "zoom": 20,
+        "scrolls": 4
+    }
+}
+
+class DamascusProScanner:
+    def __init__(self, bounds: dict):
+        self.bounds = bounds
+        self.global_seen = set()
+        self.coord_pattern = re.compile(r"!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)|/@(-?\d+\.\d+),(-?\d+\.\d+)")
+
+    def calculate_distance(self, lat1, lon1, lat2, lon2):
+        R = 6371
+        d_lat = math.radians(lat2 - lat1)
+        d_lon = math.radians(lon2 - lon1)
+        a = math.sin(d_lat / 2)**2 + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * math.sin(d_lon / 2)**2
+        return R * (2 * math.asin(math.sqrt(a)))
+
+    async def bypass_consent(self, page):
+        """تخطي صفحة موافقة جوجل بمختلف أشكالها"""
+        try:
+            selectors = ["#L2AGLb", "button[aria-label*='Accept']", "button:has-text('أوافق')", "button:has-text('I agree')"]
+            for selector in selectors:
+                if await page.locator(selector).count() > 0:
+                    await page.click(selector)
+                    await asyncio.sleep(1)
+                    return True
+            return False
+        except: return False
+
+    async def extract_places(self, page, center_lat, center_lng, pass_name) -> list[dict]:
+        await self.bypass_consent(page)
+        places = []
+        try:
+            elements = await page.query_selector_all('a[href*="/maps/place/"]')
+            for el in elements:
+                try:
+                    href = await el.get_attribute('href')
+                    if not href: continue
+                    name_match = re.search(r"/place/([^/]+)", href)
+                    if not name_match: continue
+                    name = urllib.parse.unquote(name_match.group(1)).replace('+', ' ').strip()
+                    lat, lng = None, None
+                    matches = self.coord_pattern.findall(href)
+                    for m in matches:
+                        valid = [val for val in m if val]
+                        if len(valid) == 2:
+                            lat, lng = float(valid[0]), float(valid[1])
+                            break
+                    if name and lat and lng:
+                        dist = self.calculate_distance(center_lat, center_lng, lat, lng)
+                        if dist > 3.5: continue
+                        if any(key in name.lower() for key in ["google", "بحث", "إبلاغ"]): continue
+                        dedup_key = f"{name.lower()}|{round(lat, 5)}|{round(lng, 5)}"
+                        if dedup_key not in self.global_seen:
+                            self.global_seen.add(dedup_key)
+                            places.append({
+                                "name": name, "name_ar": name, "latitude": lat, "longitude": lng,
+                                "category": pass_name.title(), "city": "Damascus", "source": "scraper_admin"
+                            })
+                except: continue
+        except: pass
+        return places
+
+    def send_batch(self, places: list[dict]):
+        if not places: return
+        try:
+            res = requests.post(f"{API_URL}/geocoding/upsert-batch", headers={"X-API-Key": API_KEY}, json={"places": places}, timeout=15)
+            if res.status_code in [200, 201]:
+                print(f"   ✅ [SYNC SUCCESS] {len(places)} points added.", flush=True)
+            else:
+                print(f"   ❌ [SYNC ERROR] {res.status_code}", flush=True)
+        except Exception as e:
+            print(f"   ⚠️ [SYNC EXCEPTION] {str(e)[:50]}", flush=True)
+
+    async def run_pass(self, pass_name: str, test_mode=False):
+        cfg = PASSES[pass_name]
+        print(f"\n🚀 PASS: {pass_name} (Nuclear Mode Active)", flush=True)
+        
+        async with async_playwright() as p:
+            user_data_dir = os.path.join(os.getcwd(), "browser_state_pro")
+            context = await p.chromium.launch_persistent_context(
+                user_data_dir,
+                headless=True,
+                viewport={"width": 1280, "height": 720},
+                user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+            )
+            
+            # حقن الكوكيز فوراً
+            await context.add_cookies([
+                {"name": "CONSENT", "value": "YES+cb.20230531-17-p0.en+FX+908", "domain": ".google.com", "path": "/"},
+                {"name": "SOCS", "value": "CAISHAgBEhJnd3NfMjAyMzA4MzAtMF9SQzIaAnByIAE", "domain": ".google.com", "path": "/"}
+            ])
+            
+            page = await context.new_page()
+            await page.set_extra_http_headers({"Accept-Language": "ar-SY,ar;q=0.9,en-US;q=0.8"})
+            
+            rows = math.ceil((self.bounds['max_lat'] - self.bounds['min_lat']) / cfg['lat_step'])
+            cols = math.ceil((self.bounds['max_lng'] - self.bounds['min_lng']) / cfg['lng_step'])
+            
+            for r in range(rows):
+                for c in range(cols):
+                    lat = self.bounds['min_lat'] + (r * cfg['lat_step'])
+                    lng = self.bounds['min_lng'] + (c * cfg['lng_step'])
+                    
+                    try:
+                        async with asyncio.timeout(45): # مهلة 45 ثانية لكل كتلة
+                            print(f"🔎 [BLOCK {r},{c}] {lat:.4f}, {lng:.4f} | ", end="", flush=True)
+                            cat = cfg['categories'][0] # نكتفي بصنف واحد حالياً للسرعة الكلية
+                            url = f"https://www.google.com/maps/search/{cat}/@{lat},{lng},{cfg['zoom']}z?hl=ar"
+                            
+                            await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+                            await self.bypass_consent(page)
+                            
+                            for _ in range(cfg['scrolls']):
+                                await page.mouse.wheel(0, 1500)
+                                await asyncio.sleep(0.3)
+                            
+                            found = await self.extract_places(page, lat, lng, pass_name)
+                            if found:
+                                self.send_batch(found)
+                                print(f"✅ Found {len(found)}", flush=True)
+                            else:
+                                print(f"⚪ Empty", flush=True)
+                    except Exception as e:
+                        print(f"⚠️ Skip: {str(e)[:20]}", flush=True)
+                        continue
+            
+            await context.close()
+
+if __name__ == "__main__":
+    scanner = DamascusProScanner(DAMASCUS_BOUNDS)
+    async def main():
+        print("🏙️ [PROD] Damascus Scanner PRO - Server Edition", flush=True)
+        await scanner.run_pass("COMMERCIAL_PUBLIC")
+        await scanner.run_pass("RESIDENTIAL")
+    asyncio.run(main())
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Download Existing Syria Data from Server to Mac
+# To be run on the Mac terminal
+
+SERVER_IP="188.68.36.205"
+SERVER_USER="hamzadoctor"
+REMOTE_PATH="/home/hamzadoctor/app"
+KEY_PATH="/Users/hamzaaleghwairyeen/.ssh/doctory-key"
+
+echo "🌍 Connecting to server to dump existing Syria landmarks..."
+
+# 1. Generate dump on server
+ssh -i "$KEY_PATH" "$SERVER_USER@$SERVER_IP" "docker exec -t map-db pg_dump -U mapuser -d mapdb -t places_syria --data-only --inserts > $REMOTE_PATH/existing_syria_data.sql"
+
+# 2. Download the dump to local Mac
+echo "📥 Downloading dump to Mac..."
+rsync -avz -e "ssh -i $KEY_PATH" "$SERVER_USER@$SERVER_IP:$REMOTE_PATH/existing_syria_data.sql" infrastructure/docker/postgis/
+
+echo "✅ Download complete: infrastructure/docker/postgis/existing_syria_data.sql"
+echo "Now run ./infrastructure/scripts/local_db_prep.sh to merge."
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Script to execute Syria data import on the server
+# To be run from /home/hamzadoctor/app
+
+APP_DIR="/home/hamzadoctor/app"
+CSV_FILE="syria_final_complete.csv"
+SQL_FILE="infrastructure/docker/postgis/import_syria_csv.sql"
+
+echo "📦 Transferring CSV to PostGIS container..."
+docker cp "$APP_DIR/$CSV_FILE" map-db:/tmp/syria_data.csv
+
+echo "💾 Running SQL import script..."
+# We use docker compose exec db psql to run the logic
+# First, update the SQL logic to use the /tmp path for COPY
+# We'll create a temporary SQL wrapper to handle the COPY command with the correct path
+
+docker exec -i map-db psql -U mapuser -d mapdb <<EOF
+-- Load schema
+\i /home/hamzadoctor/app/$SQL_FILE
+
+-- Perform COPY (must be done in the container pointing to /tmp/syria_data.csv)
+CREATE TEMP TABLE staging_syria_temp (
+    name TEXT,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    category TEXT,
+    address TEXT,
+    description TEXT,
+    created_at TIMESTAMP
+);
+
+COPY staging_syria_temp(name, latitude, longitude, category, address, description, created_at) 
+FROM '/tmp/syria_data.csv' 
+WITH (FORMAT csv, HEADER false, QUOTE '"', ENCODING 'UTF8');
+
+INSERT INTO staging_syria SELECT * FROM staging_syria_temp;
+DROP TABLE staging_syria_temp;
+
+-- Final merge logic is already in the SQL file loaded via \i
+EOF
+
+echo "✅ Import completed successfully!"
@@ -0,0 +1,52 @@
+#!/bin/bash
+# Script to import Egypt and Jordan OSM data into the PostGIS database
+# نص برمجي لاستيراد بيانات مصر والأردن إلى قاعدة البيانات
+
+set -e
+
+DATA_DIR="./infrastructure/osm-data"
+DB_USER=${POSTGRES_USER:-mapuser}
+DB_NAME=${POSTGRES_DB:-mapdb}
+
+echo "🌍 Starting MENA Regional Data Import..."
+echo "🌍 البدء في استيراد البيانات الإقليمية..."
+
+# 1. Check for files
+if [ ! -f "$DATA_DIR/jordan-latest.osm.pbf" ]; then
+    echo "❌ Jordan PBF missing. Please run setup-osm.sh first."
+    exit 1
+fi
+
+if [ ! -f "$DATA_DIR/egypt-latest.osm.pbf" ]; then
+    echo "📥 Downloading Egypt OSM PBF..."
+    curl -L https://download.geofabrik.de/africa/egypt-latest.osm.pbf -o "$DATA_DIR/egypt-latest.osm.pbf"
+fi
+
+# 2. Detect Docker Compose Command
+if command -v docker-compose &> /dev/null; then
+    DOKCER_COMPOSE="docker-compose"
+else
+    DOKCER_COMPOSE="docker compose"
+fi
+
+echo "🐳 Using $DOKCER_COMPOSE..."
+
+# 3. Import Jordan (Append)
+echo "🇯🇴 Importing Jordan data (Append mode)..."
+$DOKCER_COMPOSE --profile import run --rm osm-import osm2pgsql \
+    --append --slim --cache 1000 \
+    --database "$DB_NAME" --host db --user "$DB_USER" \
+    /data/jordan-latest.osm.pbf
+
+# 4. Import Egypt (Append)
+echo "🇪🇬 Importing Egypt data (Append mode)..."
+$DOKCER_COMPOSE --profile import run --rm osm-import osm2pgsql \
+    --append --slim --cache 1000 \
+    --database "$DB_NAME" --host db --user "$DB_USER" \
+    /data/egypt-latest.osm.pbf
+
+echo "✅ Import complete. Restarting tile and routing services..."
+$DOKCER_COMPOSE restart martin routing
+
+echo "🚀 MENA Regional mapping is now live!"
+echo "🚀 تم تفعيل خرائط المنطقة بنجاح!"
@@ -0,0 +1,110 @@
+#!/bin/bash
+# Local Database Preparation & Merge Script
+# To be run on the Mac terminal after download_server_data.sh
+
+set -e
+
+echo "🚀 Starting local database merge and preparation..."
+
+# 1. Start the DB container
+echo "📦 Starting PostGIS container..."
+docker compose up -d db
+
+# 2. Wait for DB to be ready
+echo "⏳ Waiting for database to be ready..."
+until docker compose exec -T db pg_isready -U mapuser -d mapdb; do
+  sleep 2
+done
+
+# 3. Initialize Schema & Import Existing Server Data
+echo "🛠️ Initializing schema and loading server data..."
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+DROP TABLE IF EXISTS places_syria;
+CREATE TABLE places_syria (
+    id SERIAL PRIMARY KEY,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    name TEXT,
+    name_ar TEXT,
+    name_en TEXT,
+    address TEXT,
+    category TEXT,
+    neighbourhood TEXT,
+    city TEXT,
+    description TEXT,
+    created_at TIMESTAMP DEFAULT NOW(),
+    source TEXT,
+    location GEOMETRY(Point, 4326)
+);
+
+CREATE OR REPLACE FUNCTION sync_place_location() RETURNS trigger AS \$\$
+BEGIN
+  IF NEW.latitude IS NOT NULL AND NEW.longitude IS NOT NULL THEN
+    NEW.location := ST_SetSRID(ST_MakePoint(CAST(NEW.longitude AS FLOAT), CAST(NEW.latitude AS FLOAT)), 4326);
+  END IF;
+  RETURN NEW;
+END;
+\$\$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_sync_place_location ON places_syria;
+CREATE TRIGGER trg_sync_place_location
+BEFORE INSERT OR UPDATE ON places_syria
+FOR EACH ROW EXECUTE FUNCTION sync_place_location();
+"
+
+# Load existing data if available
+if [ -f "infrastructure/docker/postgis/existing_syria_data.sql" ]; then
+    echo "📥 Loading existing server data..."
+    docker compose exec -T db psql -U mapuser -d mapdb < infrastructure/docker/postgis/existing_syria_data.sql
+fi
+
+# 4. Import New CSV Data
+echo "📥 Importing new CSV landmark data..."
+docker cp infrastructure/docker/postgis/syria_final_complete.csv map-db:/tmp/syria_data.csv
+
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+DROP TABLE IF EXISTS staging_syria;
+CREATE TABLE staging_syria (
+    name TEXT,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    category TEXT,
+    address TEXT,
+    description TEXT,
+    created_at TIMESTAMP
+);
+
+-- HEADER true fixes the 'invalid input syntax' error
+COPY staging_syria(name, latitude, longitude, category, address, description, created_at) 
+FROM '/tmp/syria_data.csv' 
+WITH (FORMAT csv, HEADER true, QUOTE '\"', ENCODING 'UTF8');
+
+-- Merge into places_syria while avoiding duplicates
+-- We check for name + spatial proximity (approx 50m)
+INSERT INTO places_syria (name, name_ar, latitude, longitude, category, address, description, source, created_at)
+SELECT 
+    name, 
+    name, 
+    latitude, 
+    longitude, 
+    category, 
+    address, 
+    description, 
+    'csv_import_2026_04',
+    COALESCE(created_at, NOW())
+FROM staging_syria s
+WHERE NOT EXISTS (
+    SELECT 1 FROM places_syria p 
+    WHERE (p.name = s.name OR p.name_ar = s.name)
+    AND ST_DWithin(
+        ST_SetSRID(ST_MakePoint(CAST(p.longitude AS FLOAT), CAST(p.latitude AS FLOAT)), 4326)::geography, 
+        ST_SetSRID(ST_MakePoint(CAST(s.longitude AS FLOAT), CAST(s.latitude AS FLOAT)), 4326)::geography, 
+        50
+    )
+);
+
+DROP TABLE staging_syria;
+"
+
+echo "✅ Local database merge complete."
+echo "Run ./infrastructure/scripts/local_verify.sh to review the results."
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Local Verification Script for Syria Landmarks
+# To be run on the Mac terminal
+
+set -e
+
+echo "📊 --- Local Syria Data Audit ---"
+
+# 1. Total Count
+TOTAL=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "SELECT count(*) FROM places_syria;")
+echo "📍 Total Landmarks: $TOTAL"
+
+# 2. Category Distribution
+echo "🗂️  Category Distribution:"
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+SELECT category, count(*) as count 
+FROM places_syria 
+GROUP BY category 
+ORDER BY count DESC 
+LIMIT 10;
+"
+
+# 3. Spatial Bounds Check (Damascus region)
+echo "🌍 Spatial Check (Damascus):"
+DM_COUNT=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "
+SELECT count(*) FROM places_syria 
+WHERE latitude BETWEEN 33.4 AND 33.6 AND longitude BETWEEN 36.2 AND 36.4;
+")
+echo "🏙️  Landmarks in Damascus area: $DM_COUNT"
+
+# 4. Generate Export if user is satisfied
+echo "💾 Generating SQL Export..."
+docker compose exec -T db pg_dump -U mapuser -d mapdb -t places_syria --data-only --inserts > infrastructure/docker/postgis/syria_export.sql
+
+echo "--------------------------------"
+echo "✅ Verification complete. Export saved to: infrastructure/docker/postgis/syria_export.sql"
+echo "If you are happy with the results, run sync_to_server.sh to push the data."
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Server Restoration Script for Syria Landmarks
+# To be run on the server terminal
+
+set -e
+
+APP_DIR="/home/hamzadoctor/app"
+EXPORT_FILE="infrastructure/docker/postgis/syria_export.sql"
+
+echo "🔄 Restoring Syria landmarks to production database..."
+
+# 1. Clean existing manual imports to avoid PK conflicts
+echo "🧹 Clearing existing manual entries..."
+docker exec -i map-db psql -U mapuser -d mapdb -c "DELETE FROM places_syria WHERE source = 'manual_import_2026_04';"
+
+# 2. Inject the SQL dump
+echo "📥 Injecting SQL dump..."
+docker exec -i map-db psql -U mapuser -d mapdb < "$APP_DIR/$EXPORT_FILE"
+
+# 3. Refresh API cache
+echo "🧹 Flushing Redis..."
+docker exec -i map-redis redis-cli flushall
+
+echo "✅ Restoration complete! Syria landmarks are live."
@@ -0,0 +1,33 @@
+#!/bin/bash
+# script to download Jordan OSM data
+# نص برمجبي لتحميل بيانات الخرائط للأردن
+
+set -e
+
+DATA_DIR="./infrastructure/osm-data"
+PBF_URL="https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
+
+echo "📍 Starting Jordan Map Setup..."
+echo "📍 البدء في إعداد خرائط الأردن..."
+
+mkdir -p $DATA_DIR
+
+if [ ! -f "$DATA_DIR/jordan-latest.osm.pbf" ]; then
+    echo "📥 Downloading Jordan OSM PBF from Geofabrik..."
+    echo "📥 جاري تحميل بيانات الخرائط من Geofabrik..."
+    curl -L $PBF_URL -o "$DATA_DIR/jordan-latest.osm.pbf"
+else
+    echo "✅ Jordan PBF already exists."
+    echo "✅ ملف البيانات موجود بالفعل."
+fi
+
+# Note on MBTiles:
+# Tileserver-GL requires an .mbtiles file to serve tiles locally.
+# You can generate one from the .pbf using 'tilemaker' or download a free extract 
+# from MapTiler (https://maptiler.com/data) and place it in $DATA_DIR/jordan.mbtiles
+
+echo "⚠️  Important: To serve tiles locally, you need 'jordan.mbtiles' in $DATA_DIR"
+echo "⚠️  تنبيه: لتشغيل الخرائط محلياً، يجب توفير ملف 'jordan.mbtiles' في المجلد المذكور"
+
+echo "🚀 Setup complete. Please ensure jordan.mbtiles is present before running 'docker-compose up'."
+echo "🚀 اكتمل الإعداد. تأكد من وجود ملف mbtiles قبل تشغيل النظام."
@@ -0,0 +1,60 @@
+#!/bin/bash
+# --------------------------------------------------------------------------
+# Intaleq Map Platform - 10-Day Update Script
+# سكربت تحديث خرائط انطلاقة - التحديث الدوري (كل 10 أيام)
+# --------------------------------------------------------------------------
+
+set -e # Exit on error
+
+echo "🚀 Starting 10-day map update..."
+
+# 1. Configuration (From .env or defaults)
+PBF_FILE="/data/jordan-latest.osm.pbf"
+SOURCE_URL="https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
+APP_DIR="/home/hamzadoctor/app"
+
+cd "$APP_DIR"
+
+# 2. Download latest OSM data (Running on Host)
+# تحميل أحدث البيانات للأردن وسوريا
+echo "🌍 Downloading latest OpenStreetMap data for Jordan & Syria..."
+wget -O "infrastructure/osm-data/jordan-latest.osm.pbf.new" "https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
+wget -O "infrastructure/osm-data/syria-latest.osm.pbf.new" "https://download.geofabrik.de/asia/syria-latest.osm.pbf"
+
+mv "infrastructure/osm-data/jordan-latest.osm.pbf.new" "infrastructure/osm-data/jordan-latest.osm.pbf"
+mv "infrastructure/osm-data/syria-latest.osm.pbf.new" "infrastructure/osm-data/syria-latest.osm.pbf"
+
+# 3. Import new data to PostGIS
+# استيراد البيانات إلى قاعدة البيانات
+echo "💾 Importing Jordan (Create)..."
+docker compose --profile import run --rm osm-import osm2pgsql --create --slim --cache 1000 --database mapdb --host db --user mapuser /data/jordan-latest.osm.pbf
+
+echo "💾 Importing Syria (Append)..."
+docker compose --profile import run --rm osm-import osm2pgsql --append --slim --cache 1000 --database mapdb --host db --user mapuser /data/syria-latest.osm.pbf
+
+# 4. Merge Data (Jordan + Syria)
+OSM_FILE="infrastructure/osm-data/region.osm.pbf"
+echo "🗺️ Merging Jordan and Syria data into $OSM_FILE..."
+osmium merge infrastructure/osm-data/jordan-latest.osm.pbf infrastructure/osm-data/syria-latest.osm.pbf -o $OSM_FILE --overwrite
+
+# 5. Spatial Integrity check for Geocoding (Landmarks)
+# Purge any legacy landmarks outside the expanded region
+echo "📍 Syncing user-submitted landmarks and purging invalid coordinates..."
+DELETED_COUNT=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "DELETE FROM places_syria WHERE longitude < 34 OR longitude > 43 OR latitude < 29 OR latitude > 38;")
+echo "   ✅ Purged ${DELETED_COUNT//[[:space:]]/} invalid landmarks outside the expanded region."
+
+# Force Spatial Geometry Update
+docker compose exec -T db psql -U mapuser -d mapdb -c "UPDATE places_syria SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326) WHERE location IS NULL OR latitude IS NOT NULL;"
+
+# 6. Rebuild Routing Index (GraphHopper)
+echo "🚗 Rebuilding GraphHopper routing index (This may take ~5-8 minutes)..."
+docker compose stop routing
+# Delete old cache - VERY IMPORTANT to force full re-index
+rm -rf infrastructure/osm-data/graph-cache infrastructure/osm-data/default-gh
+docker compose up -d routing
+
+# 7. Final Cleanup & Cache Flush
+echo "🧹 Clearing Redis traffic cache..."
+docker compose exec -T redis redis-cli flushall
+
+echo "Done! Map platform is now fully synchronized with Jordan & Syria roads (including Damascus)."