feat: add place gate population scripts, map style assets, automated cron sync tasks, and expand dashboard and landing applications

scripts/backup_db.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# ==============================================================================
+# SIRO Maps — Daily Database Backup Script
+# ==============================================================================
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/home/hamzadoctor/app}"
+BACKUP_DIR="${APP_DIR}/backups/db"
+TIMESTAMP=$(date '+%Y%m%d_%H%M%S')
+BACKUP_FILE="${BACKUP_DIR}/mapdb_custom_tables_${TIMESTAMP}.sql.gz"
+RETENTION_DAYS=7
+
+mkdir -p "$BACKUP_DIR"
+
+echo "💾 [$(date '+%Y-%m-%d %H:%M:%S')] Starting SIRO Maps custom tables backup..."
+
+# Dump custom tables: places_*, place_gates, billing, users, telemetry
+docker compose -f "${APP_DIR}/docker-compose.yml" exec -T db pg_dump -U mapuser -d mapdb \
+  -t 'places_*' \
+  -t 'place_gates' \
+  -t 'users' \
+  -t 'api_keys' \
+  -t 'billing_*' \
+  -t 'invoices' \
+  -t 'subscriptions' \
+  -t 'pricing_plans' \
+  -t 'telemetry_*' \
+  --no-owner --no-privileges | gzip > "$BACKUP_FILE"
+
+echo "✅ Backup created successfully: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"
+
+# Prune backups older than RETENTION_DAYS
+echo "🧹 Pruning backups older than ${RETENTION_DAYS} days..."
+find "$BACKUP_DIR" -type f -name "*.sql.gz" -mtime +"$RETENTION_DAYS" -delete
+
+echo "🏁 [$(date '+%Y-%m-%d %H:%M:%S')] Backup process finished."

scripts/cleanup_logs.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# ==============================================================================
+# SIRO Maps — Weekly Maintenance & Log Cleanup Script
+# ==============================================================================
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/home/hamzadoctor/app}"
+LOG_DIR="${APP_DIR}/logs"
+RETENTION_DAYS=14
+
+echo "🧹 [$(date '+%Y-%m-%d %H:%M:%S')] Starting weekly log and docker maintenance..."
+
+# 1. Clean old log files
+if [ -d "$LOG_DIR" ]; then
+  find "$LOG_DIR" -type f -name "*.log" -mtime +"$RETENTION_DAYS" -exec rm -f {} +
+  echo "✅ Removed log files older than ${RETENTION_DAYS} days."
+fi
+
+# 2. Truncate current active logs if they exceed 50MB
+for log in "$LOG_DIR"/*.log; do
+  if [ -f "$log" ]; then
+    size=$(stat -c%s "$log" 2>/dev/null || stat -f%z "$log" 2>/dev/null || echo 0)
+    if [ "$size" -gt 52428800 ]; then # 50MB
+      tail -n 10000 "$log" > "${log}.tmp" && mv "${log}.tmp" "$log"
+      echo "✂️ Truncated large log file: $log"
+    fi
+  fi
+done
+
+# 3. Clean dangling docker images and build cache
+docker image prune -f --filter "until=168h" > /dev/null 2>&1 || true
+docker builder prune -f --keep-storage 2GB > /dev/null 2>&1 || true
+
+echo "🏁 [$(date '+%Y-%m-%d %H:%M:%S')] Cleanup completed successfully."

scripts/enrich_egypt_places.py

@@ -0,0 +1,464 @@
+#!/usr/bin/env python3
+"""
+Enrich Egypt Places Dataset (`places_egypt`) from:
+1. OpenStreetMap Named Points (`planet_osm_point`)
+2. OpenStreetMap Named Polygons (`planet_osm_polygon`)
+3. Overture Maps Places (`overture_place`)
+4. Deduplicate using spatial grid (<60m) against existing places_egypt
+5. Refresh `unified_search_index`
+"""
+
+import os
+import sys
+import math
+import time
+import argparse
+from collections import defaultdict
+
+def haversine(lat1, lon1, lat2, lon2):
+    R = 6371000  # meters
+    phi1 = math.radians(lat1)
+    phi2 = math.radians(lat2)
+    delta_phi = math.radians(lat2 - lat1)
+    delta_lambda = math.radians(lon2 - lon1)
+    a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
+    return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+def normalize_text(t):
+    if not t:
+        return ''
+    s = t.strip()
+    s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
+    return ' '.join(s.split()).lower()
+
+def clean_sql_str(val):
+    if val is None:
+        return 'NULL'
+    s = str(val).strip().replace("'", "''")
+    return f"'{s}'"
+
+OSM_CAT_MAP = {
+    'village': 'قرية / تجمع سكاني',
+    'town': 'بلدة / مركز',
+    'city': 'مدينة',
+    'suburb': 'حي / ضاحية',
+    'neighbourhood': 'منطقة سكنية',
+    'hamlet': 'نجع / كفر',
+    'locality': 'موضع / منطقة',
+    'school': 'مدرسة / تعليم',
+    'college': 'كلية',
+    'university': 'جامعة',
+    'kindergarten': 'روضة أطفال',
+    'supermarket': 'سوبرماركت وهايبرماركت',
+    'convenience': 'محل بقالة',
+    'bakery': 'مخبز وفرن',
+    'butcher': 'جزارة وقصابة',
+    'mall': 'مركز تسوق ومول',
+    'place_of_worship': 'مسجد / دار عبادة',
+    'mosque': 'جامع / مسجد',
+    'clinic': 'مركز طبي وعيادة',
+    'pharmacy': 'صيدلية',
+    'hospital': 'مستشفى',
+    'doctors': 'عيادات تخصصية',
+    'dentist': 'عيادة أسنان',
+    'restaurant': 'مطعم',
+    'cafe': 'مقهى وكافيه',
+    'fast_food': 'مأكولات سريعة',
+    'fuel': 'محطة وقود وبنزين',
+    'bank': 'مصرف / بنك',
+    'atm': 'ماكينة صراف آلي ATM',
+    'bureau_de_change': 'مكتب صرافة',
+    'police': 'قسم شرطة ونقطة أمنية',
+    'fire_station': 'وحدة مطافئ / دفاع مدني',
+    'post_office': 'مكتب بريد',
+    'attraction': 'معلم سياحي وتاريخي',
+    'hotel': 'فندق وإقامة',
+    'guest_house': 'نُزل واستضافة',
+    'museum': 'متحف وآثار',
+    'park': 'حديقة ومنتزه عام',
+    'stadium': 'استاد وملعب رياضي',
+    'sports_centre': 'مركز ونادي رياضي',
+    'car_repair': 'ورشة صيانة سيارات',
+    'car_wash': 'مغسلة سيارات',
+    'company': 'شركة ومقر أعمال',
+    'government': 'جهة حكومية ورسمية',
+    'mobile_phone': 'متجر هواتف ومحمول',
+    'clothes': 'متجر ملابس وأزياء',
+    'shoes': 'متجر أحذية',
+    'hairdresser': 'صالون حلاقة وتجميل',
+    'optician': 'بصريات ونظارات',
+    'hardware': 'أدوات بناء وتجهيزات',
+}
+
+OVERTURE_CAT_MAP = {
+    'restaurant': 'مطعم',
+    'cafe': 'مقهى وكافيه',
+    'coffee_shop': 'مقهى وكوفي شوب',
+    'casual_eatery': 'مأكولات سريعة وخفيفة',
+    'bakery': 'مخبز ومعجنات',
+    'food_and_beverage_store': 'بقالة ومواد غذائية',
+    'supermarket': 'سوبرماركت وهايبرماركت',
+    'fashion_and_apparel_store': 'متجر ملابس وأزياء',
+    'shoe_store': 'متجر أحذية',
+    'jewelry_store': 'مجوهرات وحلي',
+    'electronics_store': 'متجر إلكترونيات وأجهزة',
+    'mobile_phone_store': 'متجر هواتف ذكية',
+    'hardware_home_and_garden_store': 'مواد بناء ومستلزمات منزلية',
+    'home_service': 'خدمات منزلية وصيانة',
+    'personal_or_beauty_service': 'صالون ومركز تجميل',
+    'hospital': 'مستشفى / مركز طبي',
+    'dental_clinic': 'عيادة أسنان',
+    'specialized_health_care': 'عيادة تخصصية',
+    'pharmacy': 'صيدلية',
+    'hotel': 'فندق وإقامة',
+    'place_of_learning': 'منشأة تعليمية',
+    'school': 'مدرسة',
+    'college_university': 'جامعة / كلية',
+    'historic_site': 'موقع تاريخي وأثري',
+    'religious_organization': 'مؤسسة دينية',
+    'mosque': 'مسجد / جامع',
+    'real_estate_service': 'مكتب عقاري',
+    'professional_service': 'خدمات مهنية واستشارية',
+    'travel_service': 'مكتب سياحة وسفر',
+    'corporate_or_business_office': 'شركة ومؤسسة أعمال',
+    'car_repair': 'صيانة وتصليح سيارات',
+    'fuel_station': 'محطة وقود',
+    'bank': 'مصرف / بنك',
+    'government_office': 'مصلحة وهيئة حكومية',
+}
+
+def map_category(cat_str, source_type='osm'):
+    if not cat_str:
+        return 'مكان ونقطة اهتمام'
+    clean = str(cat_str).strip().lower()
+    if source_type == 'overture':
+        return OVERTURE_CAT_MAP.get(clean, 'مكان عام وتجاري')
+    return OSM_CAT_MAP.get(clean, 'مكان عام')
+
+def get_grid_cell(lat, lon, cell_size=0.01):
+    return (int(lat / cell_size), int(lon / cell_size))
+
+def is_duplicate(name_norm, lat, lon, grid, max_dist_m=60.0):
+    d_lat_thresh = 0.0006
+    d_lon_thresh = 0.0007
+    c_x, c_y = get_grid_cell(lat, lon)
+    for dx in (-1, 0, 1):
+        for dy in (-1, 0, 1):
+            cell = (c_x + dx, c_y + dy)
+            if cell in grid:
+                for existing_name, ex_lat, ex_lon in grid[cell]:
+                    if abs(lat - ex_lat) > d_lat_thresh or abs(lon - ex_lon) > d_lon_thresh:
+                        continue
+                    dist = haversine(lat, lon, ex_lat, ex_lon)
+                    if dist <= max_dist_m:
+                        if (name_norm == existing_name or 
+                            name_norm in existing_name or 
+                            existing_name in name_norm):
+                            return True
+    return False
+
+def add_to_grid(name_norm, lat, lon, grid):
+    cell = get_grid_cell(lat, lon)
+    grid[cell].append((name_norm, lat, lon))
+
+def main():
+    parser = argparse.ArgumentParser(description="Enrich Egypt places from OSM & Overture.")
+    parser.add_argument("--db-host", default="127.0.0.1")
+    parser.add_argument("--db-port", type=int, default=5432)
+    parser.add_argument("--db-user", default="mapuser")
+    parser.add_argument("--db-pass", default="mappass")
+    parser.add_argument("--db-name", default="mapdb")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    import pg8000.native
+
+    print("=" * 70)
+    print("🇪🇬 SIRO Maps — Comprehensive Egypt Places Enrichment & Deduplication")
+    print("=" * 70)
+
+    print(f"🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
+    con = pg8000.native.Connection(
+        user=args.db_user,
+        password=args.db_pass,
+        host=args.db_host,
+        port=args.db_port,
+        database=args.db_name
+    )
+
+    t0 = time.time()
+    grid = defaultdict(list)
+
+    # 1. Load existing places_egypt
+    print("\n📦 Step 1: Loading existing places_egypt into spatial index...")
+    existing = con.run("""
+        SELECT id, name, latitude, longitude 
+        FROM places_egypt 
+        WHERE latitude IS NOT NULL AND longitude IS NOT NULL;
+    """)
+    print(f"   -> Loaded {len(existing):,} existing places.")
+    for row in existing:
+        pid, name, lat, lon = row
+        lat = float(lat)
+        lon = float(lon)
+        norm_name = normalize_text(name)
+        add_to_grid(norm_name, lat, lon, grid)
+
+    print(f"   ✓ Spatial grid built with {sum(len(v) for v in grid.values()):,} items.")
+
+    # Egypt Bounding Box: 21.8 <= lat <= 31.8, 24.7 <= lon <= 36.9
+    EG_MIN_LON, EG_MIN_LAT = 24.7, 21.8
+    EG_MAX_LON, EG_MAX_LAT = 36.9, 31.8
+
+    # 2. Harvest from planet_osm_point in Egypt
+    print("\n📍 Step 2: Harvesting from OpenStreetMap Named Points (planet_osm_point)...")
+    osm_pts = con.run(f"""
+        SELECT 
+            name,
+            COALESCE(amenity, shop, tourism, historic, place, leisure, office, building, highway, "natural") as cat,
+            ST_Y(ST_Transform(way, 4326)) as lat,
+            ST_X(ST_Transform(way, 4326)) as lon
+        FROM planet_osm_point
+        WHERE name IS NOT NULL 
+          AND TRIM(name) != ''
+          AND way && ST_Transform(ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326), 3857);
+    """)
+    print(f"   -> Found {len(osm_pts):,} candidates in planet_osm_point for Egypt.")
+
+    osm_pts_to_insert = []
+    osm_pts_dups = 0
+
+    for r in osm_pts:
+        name = str(r[0]).strip()
+        cat_raw = r[1]
+        lat = float(r[2])
+        lon = float(r[3])
+
+        if not (EG_MIN_LAT <= lat <= EG_MAX_LAT and EG_MIN_LON <= lon <= EG_MAX_LON):
+            continue
+
+        norm_name = normalize_text(name)
+        if len(norm_name) < 2:
+            continue
+
+        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
+            osm_pts_dups += 1
+            continue
+
+        category = map_category(cat_raw, 'osm')
+        pop_score = 65 if cat_raw in ('village', 'town', 'city', 'hospital', 'university', 'mosque') else 40
+
+        item = {
+            'name': name,
+            'name_ar': name,
+            'name_en': name,
+            'category': category,
+            'city': 'مصر',
+            'address': f"{category}, مصر",
+            'lat': lat,
+            'lon': lon,
+            'source': 'osm_point',
+            'popularity_score': pop_score
+        }
+        osm_pts_to_insert.append(item)
+        add_to_grid(norm_name, lat, lon, grid)
+
+    print(f"   ✓ OSM Points Processing Complete:")
+    print(f"      - Duplicates Filtered: {osm_pts_dups:,}")
+    print(f"      - Net New Places: {len(osm_pts_to_insert):,}")
+
+    # 3. Harvest from planet_osm_polygon in Egypt
+    print("\n🏛️ Step 3: Harvesting from OpenStreetMap Named Polygons (planet_osm_polygon)...")
+    osm_polys = con.run(f"""
+        SELECT 
+            name,
+            COALESCE(amenity, shop, tourism, historic, place, leisure, building, landuse) as cat,
+            ST_Y(ST_Centroid(ST_Transform(way, 4326))) as lat,
+            ST_X(ST_Centroid(ST_Transform(way, 4326))) as lon
+        FROM planet_osm_polygon
+        WHERE name IS NOT NULL 
+          AND TRIM(name) != ''
+          AND (amenity IS NOT NULL OR shop IS NOT NULL OR tourism IS NOT NULL OR historic IS NOT NULL 
+               OR place IS NOT NULL OR leisure IS NOT NULL OR landuse IN ('commercial', 'retail', 'industrial', 'cemetery', 'religious'))
+          AND way && ST_Transform(ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326), 3857);
+    """)
+    print(f"   -> Found {len(osm_polys):,} candidates in planet_osm_polygon for Egypt.")
+
+    osm_polys_to_insert = []
+    osm_polys_dups = 0
+
+    for r in osm_polys:
+        name = str(r[0]).strip()
+        cat_raw = r[1]
+        lat = float(r[2])
+        lon = float(r[3])
+
+        if not (EG_MIN_LAT <= lat <= EG_MAX_LAT and EG_MIN_LON <= lon <= EG_MAX_LON):
+            continue
+
+        norm_name = normalize_text(name)
+        if len(norm_name) < 2:
+            continue
+
+        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
+            osm_polys_dups += 1
+            continue
+
+        category = map_category(cat_raw, 'osm')
+        pop_score = 70 if cat_raw in ('hospital', 'university', 'mall', 'stadium', 'attraction') else 45
+
+        item = {
+            'name': name,
+            'name_ar': name,
+            'name_en': name,
+            'category': category,
+            'city': 'مصر',
+            'address': f"{category}, مصر",
+            'lat': lat,
+            'lon': lon,
+            'source': 'osm_polygon',
+            'popularity_score': pop_score
+        }
+        osm_polys_to_insert.append(item)
+        add_to_grid(norm_name, lat, lon, grid)
+
+    print(f"   ✓ OSM Polygons Processing Complete:")
+    print(f"      - Duplicates Filtered: {osm_polys_dups:,}")
+    print(f"      - Net New Places: {len(osm_polys_to_insert):,}")
+
+    # 4. Harvest from overture_place in Egypt
+    print("\n🗺️ Step 4: Harvesting from Overture Maps Places (overture_place)...")
+    ovt_rows = con.run(f"""
+        SELECT 
+            name_primary,
+            basic_category,
+            addresses,
+            confidence,
+            ST_Y(ST_Centroid(location::geometry)) as lat,
+            ST_X(ST_Centroid(location::geometry)) as lon
+        FROM overture_place
+        WHERE name_primary IS NOT NULL 
+          AND TRIM(name_primary) != ''
+          AND ST_Within(location::geometry, ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326));
+    """)
+    print(f"   -> Found {len(ovt_rows):,} candidates in overture_place for Egypt.")
+
+    ovt_to_insert = []
+    ovt_dups = 0
+
+    for r in ovt_rows:
+        name = str(r[0]).strip()
+        cat_str = r[1]
+        addr_list = r[2]
+        confidence = float(r[3] or 0.8)
+        lat = float(r[4])
+        lon = float(r[5])
+
+        norm_name = normalize_text(name)
+        if len(norm_name) < 2:
+            continue
+
+        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
+            ovt_dups += 1
+            continue
+
+        city = 'مصر'
+        addr_text = None
+        if addr_list and isinstance(addr_list, list) and len(addr_list) > 0:
+            a = addr_list[0]
+            if isinstance(a, dict):
+                city = a.get('locality') or a.get('region') or 'مصر'
+                addr_text = a.get('freeform')
+
+        category = map_category(cat_str, 'overture')
+        pop_score = min(100, int(confidence * 60 + 20))
+
+        item = {
+            'name': name,
+            'name_ar': name,
+            'name_en': name,
+            'category': category,
+            'city': city,
+            'address': addr_text or f"{city}, مصر",
+            'lat': lat,
+            'lon': lon,
+            'source': 'overture_maps',
+            'popularity_score': pop_score
+        }
+        ovt_to_insert.append(item)
+        add_to_grid(norm_name, lat, lon, grid)
+
+    print(f"   ✓ Overture Processing Complete:")
+    print(f"      - Duplicates Filtered: {ovt_dups:,}")
+    print(f"      - Net New Places: {len(ovt_to_insert):,}")
+
+    all_new = osm_pts_to_insert + osm_polys_to_insert + ovt_to_insert
+    print(f"\n🌟 Total Net New Places to Insert into places_egypt: {len(all_new):,}")
+
+    if args.dry_run:
+        print("💡 Dry run complete. No database changes made.")
+        return
+
+    # 5. Insert into places_egypt
+    print(f"\n🚀 Step 5: Inserting {len(all_new):,} places into places_egypt...")
+    t_insert = time.time()
+    batch_size = 2500
+    total_inserted = 0
+
+    con.run("BEGIN;")
+    for i in range(0, len(all_new), batch_size):
+        batch = all_new[i:i + batch_size]
+        values = []
+        for r in batch:
+            c_name = clean_sql_str(r['name'])
+            c_name_ar = clean_sql_str(r['name_ar'])
+            c_name_en = clean_sql_str(r['name_en'])
+            c_cat = clean_sql_str(r['category'])
+            c_city = clean_sql_str(r['city'])
+            c_addr = clean_sql_str(r['address'])
+            c_src = clean_sql_str(r['source'])
+            lat = r['lat']
+            lon = r['lon']
+            pop = r['popularity_score']
+
+            line = (
+                f"({c_name}, {c_name_ar}, {c_name_en}, {lat:.7f}, {lon:.7f}, {c_cat}, "
+                f"{c_city}, {c_addr}, {c_src}, {pop}, "
+                f"ST_SetSRID(ST_MakePoint({lon:.7f}, {lat:.7f}), 4326))"
+            )
+            values.append(line)
+
+        sql = (
+            "INSERT INTO places_egypt ("
+            "    name, name_ar, name_en, latitude, longitude, category,"
+            "    city, address, source, popularity_score, location"
+            ") VALUES " + ",\n".join(values) + ";"
+        )
+        con.run(sql)
+        total_inserted += len(batch)
+        print(f"   -> Progress: {total_inserted:,} / {len(all_new):,} places inserted...")
+
+    con.run("COMMIT;")
+    print(f"✅ Ingestion committed successfully in {time.time() - t_insert:.2f}s!")
+
+    # 6. Refresh Materialized View
+    print("\n🔄 Step 6: Refreshing materialized view unified_search_index concurrently...")
+    t_mv = time.time()
+    try:
+        con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
+        print(f"   ✓ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")
+    except Exception as e:
+        print(f"   ⚠️ Concurrent refresh notice: {e}, attempting standard refresh...")
+        con.run("REFRESH MATERIALIZED VIEW unified_search_index;")
+        print(f"   ✓ unified_search_index refreshed!")
+
+    con.run("ANALYZE places_egypt;")
+    con.run("ANALYZE unified_search_index;")
+
+    total_count = con.run("SELECT count(*) FROM places_egypt;")[0][0]
+    print(f"\n🎯 Total Places in places_egypt: {total_count:,}")
+    con.close()
+    print(f"🎉 Egypt places enrichment completed in {time.time() - t0:.2f}s!")
+
+if __name__ == '__main__':
+    main()

scripts/enrich_iraq_comprehensive.py

@@ -0,0 +1,523 @@
+#!/usr/bin/env python3
+"""
+Comprehensive Iraq Places Enrichment & Deduplication Script.
+Enriches `places_iraq` table by harvesting and deduplicating:
+1. Overture Maps Places (`overture_place`)
+2. OpenStreetMap Named Points (`planet_osm_point`)
+3. OpenStreetMap Named Polygons (`planet_osm_polygon`)
+4. Administrative boundaries & Governorates linking
+5. Concurrently refreshes `unified_search_index`
+"""
+
+import os
+import sys
+import math
+import time
+import argparse
+from collections import defaultdict
+
+def haversine(lat1, lon1, lat2, lon2):
+    R = 6371000  # meters
+    phi1 = math.radians(lat1)
+    phi2 = math.radians(lat2)
+    delta_phi = math.radians(lat2 - lat1)
+    delta_lambda = math.radians(lon2 - lon1)
+    a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
+    return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+def normalize_text(t):
+    if not t:
+        return ''
+    s = t.strip()
+    s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
+    return ' '.join(s.split()).lower()
+
+def clean_sql_str(val):
+    if val is None:
+        return 'NULL'
+    s = str(val).strip().replace("'", "''")
+    return f"'{s}'"
+
+# Category Translations
+OVERTURE_CAT_MAP = {
+    'restaurant': 'مطعم',
+    'cafe': 'مقهى وكافيه',
+    'coffee_shop': 'مقهى وكوفي شوب',
+    'casual_eatery': 'وجبات سريعة ومأكولات خفيفة',
+    'bakery': 'مخبز ومعجنات',
+    'food_and_beverage_store': 'بقالة ومواد غذائية',
+    'supermarket': 'سوبرماركت وهايبرماركت',
+    'fashion_and_apparel_store': 'متجر ألبسة وأزياء',
+    'shoe_store': 'متجر أحذية',
+    'jewelry_store': 'مجوهرات وحلي',
+    'electronics_store': 'متجر إلكترونيات وأجهزة',
+    'mobile_phone_store': 'متجر هواتف ذكية',
+    'hardware_home_and_garden_store': 'متجر مواد بناء ومستلزمات منزلية',
+    'home_service': 'خدمات منزلية وصيانة',
+    'personal_or_beauty_service': 'صالون ومركز تجميل',
+    'hair_salon': 'صالون حلاقة وتزيين',
+    'hospital': 'مستشفى / مركز طبي',
+    'dental_clinic': 'عيادة طب أسنان',
+    'specialized_health_care': 'عيادة استشارية متخصصة',
+    'pharmacy': 'صيدلية',
+    'hotel': 'فندق وإقامة',
+    'place_of_learning': 'مؤسسة تعليمية',
+    'school': 'مدرسة',
+    'college_university': 'كلية / جامعة',
+    'historic_site': 'موقع تاريخي وأثري',
+    'religious_organization': 'دار عبادة ومؤسسة دينية',
+    'mosque': 'مسجد / جامع',
+    'real_estate_service': 'مكتب عقاري',
+    'professional_service': 'خدمات مهنية واستشارية',
+    'travel_service': 'مكتب سياحة وسفر',
+    'corporate_or_business_office': 'مكتب شركة ومؤسسة أعمال',
+    'car_repair': 'صيانة وتصليح سيارات',
+    'car_dealership': 'معرض ووكالة سيارات',
+    'fuel_station': 'محطة وقود',
+    'bank': 'مصرف / بنك',
+    'government_office': 'دائرة حكومية ورسمية',
+    'law_firm': 'مكتب محاماة واستشارات قانونية',
+    'sports_club': 'نادي ومجمع رياضي',
+    'park': 'منتزه وحديقة عامة',
+}
+
+OSM_CAT_MAP = {
+    'village': 'قرية / تجمع سكاني',
+    'town': 'بلدة / قضاء',
+    'city': 'مدينة',
+    'suburb': 'حي / ضاحية',
+    'neighbourhood': 'حي سكني',
+    'hamlet': 'قرية صغيرة',
+    'locality': 'منطقة / موضع',
+    'school': 'مدرسة / تعليم',
+    'college': 'كلية',
+    'university': 'جامعة',
+    'kindergarten': 'روضة أطفال',
+    'supermarket': 'سوبرماركت ومواد غذائية',
+    'convenience': 'محل بقالة',
+    'bakery': 'مخبز',
+    'butcher': 'ملحمة وقصابة',
+    'mall': 'مركز تسوق ومول',
+    'place_of_worship': 'مسجد / دار عبادة',
+    'mosque': 'مسجد / جامع',
+    'clinic': 'مركز صحي وعيادة',
+    'pharmacy': 'صيدلية',
+    'hospital': 'مستشفى',
+    'doctors': 'عيادة طبيب',
+    'dentist': 'عيادة أسنان',
+    'restaurant': 'مطعم',
+    'cafe': 'مقهى وكافيه',
+    'fast_food': 'وجبات سريعة',
+    'fuel': 'محطة وقود',
+    'bank': 'مصرف / بنك',
+    'atm': 'صراف آلي',
+    'bureau_de_change': 'مكتب صرافة وتحويل مالي',
+    'police': 'مركز شرطة وأمن',
+    'fire_station': 'دفاع مدني / إطفاء',
+    'post_office': 'مكتب بريد',
+    'attraction': 'معلم سياحي',
+    'hotel': 'فندق',
+    'guest_house': 'نُزل واستضافة',
+    'museum': 'متحف / تراث',
+    'park': 'حديقة ومنتزه',
+    'stadium': 'ملعب / مجمع رياضي',
+    'sports_centre': 'مركز رياضي',
+    'car_repair': 'تصليح سيارات',
+    'car_wash': 'مغسلة سيارات',
+    'company': 'شركة ومؤسسة',
+    'government': 'دائرة حكومية',
+    'mobile_phone': 'متجر هواتف ونقالات',
+    'clothes': 'متجر ألبسة',
+    'shoes': 'متجر أحذية',
+    'hairdresser': 'صالون حلاقة',
+    'optician': 'بصريات ونظارات',
+    'hardware': 'مواد بناء وتجهيزات',
+}
+
+def map_overture_category(cat_str):
+    if not cat_str:
+        return 'مكان ونقطة اهتمام'
+    clean = cat_str.strip().lower()
+    return OVERTURE_CAT_MAP.get(clean, 'مكان عام وتجاري')
+
+def map_osm_category(val):
+    if not val:
+        return 'نقطة اهتمام'
+    clean = str(val).strip().lower()
+    return OSM_CAT_MAP.get(clean, 'مكان عام')
+
+def get_grid_cell(lat, lon, cell_size=0.01):
+    # ~1.1km cell size
+    return (int(lat / cell_size), int(lon / cell_size))
+
+def is_duplicate(name_norm, lat, lon, grid, max_dist_m=60.0):
+    # 60 meters approx threshold in degrees
+    d_lat_thresh = 0.0006
+    d_lon_thresh = 0.0007
+    c_x, c_y = get_grid_cell(lat, lon)
+    for dx in (-1, 0, 1):
+        for dy in (-1, 0, 1):
+            cell = (c_x + dx, c_y + dy)
+            if cell in grid:
+                for existing_name, ex_lat, ex_lon in grid[cell]:
+                    if abs(lat - ex_lat) > d_lat_thresh or abs(lon - ex_lon) > d_lon_thresh:
+                        continue
+                    dist = haversine(lat, lon, ex_lat, ex_lon)
+                    if dist <= max_dist_m:
+                        # Check name similarity
+                        if (name_norm == existing_name or 
+                            name_norm in existing_name or 
+                            existing_name in name_norm):
+                            return True
+    return False
+
+def add_to_grid(name_norm, lat, lon, grid):
+    cell = get_grid_cell(lat, lon)
+    grid[cell].append((name_norm, lat, lon))
+
+def main():
+    parser = argparse.ArgumentParser(description="Enrich Iraq places comprehensively from Overture & OSM.")
+    parser.add_argument("--db-host", default="127.0.0.1", help="Database host")
+    parser.add_argument("--db-port", type=int, default=5432, help="Database port")
+    parser.add_argument("--db-user", default="mapuser", help="Database user")
+    parser.add_argument("--db-pass", default="mappass", help="Database password")
+    parser.add_argument("--db-name", default="mapdb", help="Database name")
+    parser.add_argument("--dry-run", action="store_true", help="Perform extraction and deduplication only without inserting.")
+    args = parser.parse_args()
+
+    import pg8000.native
+
+    print("=" * 70)
+    print("🇮🇶 SIRO Maps — Comprehensive Iraq Places Enrichment & Deduplication")
+    print("=" * 70)
+
+    print(f"🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
+    con = pg8000.native.Connection(
+        user=args.db_user,
+        password=args.db_pass,
+        host=args.db_host,
+        port=args.db_port,
+        database=args.db_name
+    )
+
+    t0 = time.time()
+    grid = defaultdict(list)
+
+    # 1. Load existing places into spatial hash grid
+    print("\n📦 Step 1: Loading existing places_iraq into spatial index for deduplication...")
+    existing = con.run("""
+        SELECT id, name, latitude, longitude 
+        FROM places_iraq 
+        WHERE latitude IS NOT NULL AND longitude IS NOT NULL;
+    """)
+    print(f"   -> Loaded {len(existing):,} existing places.")
+    for row in existing:
+        pid, name, lat, lon = row
+        lat = float(lat)
+        lon = float(lon)
+        norm_name = normalize_text(name)
+        add_to_grid(norm_name, lat, lon, grid)
+
+    print(f"   ✓ Spatial grid built with {sum(len(v) for v in grid.values()):,} items.")
+
+    # 2. Harvest from overture_place
+    print("\n🗺️ Step 2: Harvesting from Overture Maps Places (overture_place)...")
+    overture_rows = con.run("""
+        SELECT 
+            name_primary,
+            basic_category,
+            addresses,
+            confidence,
+            ST_Y(ST_Centroid(location::geometry)) as lat,
+            ST_X(ST_Centroid(location::geometry)) as lon
+        FROM overture_place
+        WHERE name_primary IS NOT NULL 
+          AND TRIM(name_primary) != ''
+          AND ST_Within(location::geometry, ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326));
+    """)
+    print(f"   -> Found {len(overture_rows):,} candidates in overture_place for Iraq.")
+
+    overture_to_insert = []
+    overture_dups = 0
+
+    for r in overture_rows:
+        name = str(r[0]).strip()
+        cat_str = r[1]
+        addr_list = r[2]
+        confidence = float(r[3] or 0.8)
+        lat = float(r[4])
+        lon = float(r[5])
+
+        norm_name = normalize_text(name)
+        if len(norm_name) < 2:
+            continue
+
+        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
+            overture_dups += 1
+            continue
+
+        # Extract address details if available
+        city = 'العراق'
+        addr_text = None
+        if addr_list and isinstance(addr_list, list) and len(addr_list) > 0:
+            a = addr_list[0]
+            if isinstance(a, dict):
+                city = a.get('locality') or a.get('region') or 'العراق'
+                addr_text = a.get('freeform')
+
+        category = map_overture_category(cat_str)
+        pop_score = min(100, int(confidence * 60 + 20))
+
+        item = {
+            'name': name,
+            'name_ar': name,
+            'name_en': name,
+            'category': category,
+            'city': city,
+            'address': addr_text or f"{city}, العراق",
+            'lat': lat,
+            'lon': lon,
+            'source': 'overture_maps',
+            'popularity_score': pop_score
+        }
+        overture_to_insert.append(item)
+        add_to_grid(norm_name, lat, lon, grid)
+
+    print(f"   ✓ Overture Processing Complete:")
+    print(f"      - Duplicates Filtered: {overture_dups:,}")
+    print(f"      - Net New Places: {len(overture_to_insert):,}")
+
+    # 3. Harvest from planet_osm_point
+    print("\n📍 Step 3: Harvesting from OpenStreetMap Named Points (planet_osm_point)...")
+    osm_point_rows = con.run("""
+        SELECT 
+            name,
+            COALESCE(amenity, shop, tourism, historic, place, leisure, office, building, highway, "natural") as cat,
+            ST_Y(ST_Transform(way, 4326)) as lat,
+            ST_X(ST_Transform(way, 4326)) as lon
+        FROM planet_osm_point
+        WHERE name IS NOT NULL 
+          AND TRIM(name) != ''
+          AND way && ST_Transform(ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326), 3857);
+    """)
+    print(f"   -> Found {len(osm_point_rows):,} candidates in planet_osm_point for Iraq.")
+
+    osm_point_to_insert = []
+    osm_point_dups = 0
+
+    for r in osm_point_rows:
+        name = str(r[0]).strip()
+        cat_raw = r[1]
+        lat = float(r[2])
+        lon = float(r[3])
+
+        # Bounds check
+        if not (28.8 <= lat <= 37.5 and 38.8 <= lon <= 48.8):
+            continue
+
+        norm_name = normalize_text(name)
+        if len(norm_name) < 2:
+            continue
+
+        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
+            osm_point_dups += 1
+            continue
+
+        category = map_osm_category(cat_raw)
+        pop_score = 65 if cat_raw in ('village', 'town', 'city', 'hospital', 'university', 'mosque') else 40
+
+        item = {
+            'name': name,
+            'name_ar': name,
+            'name_en': name,
+            'category': category,
+            'city': 'العراق',
+            'address': f"{category}, العراق",
+            'lat': lat,
+            'lon': lon,
+            'source': 'osm_point',
+            'popularity_score': pop_score
+        }
+        osm_point_to_insert.append(item)
+        add_to_grid(norm_name, lat, lon, grid)
+
+    print(f"   ✓ OSM Points Processing Complete:")
+    print(f"      - Duplicates Filtered: {osm_point_dups:,}")
+    print(f"      - Net New Places: {len(osm_point_to_insert):,}")
+
+    # 4. Harvest from planet_osm_polygon (POIs)
+    print("\n🏛️ Step 4: Harvesting from OpenStreetMap Named Polygons (planet_osm_polygon)...")
+    osm_poly_rows = con.run("""
+        SELECT 
+            name,
+            COALESCE(amenity, shop, tourism, historic, place, leisure, building, landuse) as cat,
+            ST_Y(ST_Centroid(ST_Transform(way, 4326))) as lat,
+            ST_X(ST_Centroid(ST_Transform(way, 4326))) as lon
+        FROM planet_osm_polygon
+        WHERE name IS NOT NULL 
+          AND TRIM(name) != ''
+          AND (amenity IS NOT NULL OR shop IS NOT NULL OR tourism IS NOT NULL OR historic IS NOT NULL 
+               OR place IS NOT NULL OR leisure IS NOT NULL OR landuse IN ('commercial', 'retail', 'industrial', 'cemetery', 'religious'))
+          AND way && ST_Transform(ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326), 3857);
+    """)
+    print(f"   -> Found {len(osm_poly_rows):,} candidates in planet_osm_polygon for Iraq.")
+
+    osm_poly_to_insert = []
+    osm_poly_dups = 0
+
+    for r in osm_poly_rows:
+        name = str(r[0]).strip()
+        cat_raw = r[1]
+        lat = float(r[2])
+        lon = float(r[3])
+
+        if not (28.8 <= lat <= 37.5 and 38.8 <= lon <= 48.8):
+            continue
+
+        norm_name = normalize_text(name)
+        if len(norm_name) < 2:
+            continue
+
+        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
+            osm_poly_dups += 1
+            continue
+
+        category = map_osm_category(cat_raw)
+        pop_score = 70 if cat_raw in ('hospital', 'university', 'mall', 'stadium', 'attraction') else 45
+
+        item = {
+            'name': name,
+            'name_ar': name,
+            'name_en': name,
+            'category': category,
+            'city': 'العراق',
+            'address': f"{category}, العراق",
+            'lat': lat,
+            'lon': lon,
+            'source': 'osm_polygon',
+            'popularity_score': pop_score
+        }
+        osm_poly_to_insert.append(item)
+        add_to_grid(norm_name, lat, lon, grid)
+
+    print(f"   ✓ OSM Polygons Processing Complete:")
+    print(f"      - Duplicates Filtered: {osm_poly_dups:,}")
+    print(f"      - Net New Places: {len(osm_poly_to_insert):,}")
+
+    # Total new places to insert
+    all_new_places = overture_to_insert + osm_point_to_insert + osm_poly_to_insert
+    print(f"\n🌟 Total Net New Places to Insert: {len(all_new_places):,}")
+
+    if args.dry_run:
+        print("💡 Dry run requested. Exiting without database insertion.")
+        return
+
+    # 5. Database Batch Insertion
+    print(f"\n🚀 Step 5: Inserting {len(all_new_places):,} places into places_iraq...")
+    t_insert = time.time()
+    batch_size = 2500
+    total_inserted = 0
+
+    con.run("BEGIN;")
+    for i in range(0, len(all_new_places), batch_size):
+        batch = all_new_places[i:i + batch_size]
+        values = []
+        for r in batch:
+            c_name = clean_sql_str(r['name'])
+            c_name_ar = clean_sql_str(r['name_ar'])
+            c_name_en = clean_sql_str(r['name_en'])
+            c_cat = clean_sql_str(r['category'])
+            c_city = clean_sql_str(r['city'])
+            c_addr = clean_sql_str(r['address'])
+            c_src = clean_sql_str(r['source'])
+            lat = r['lat']
+            lon = r['lon']
+            pop = r['popularity_score']
+
+            line = (
+                f"({c_name}, {c_name_ar}, {c_name_en}, {lat:.7f}, {lon:.7f}, {c_cat}, "
+                f"{c_city}, {c_addr}, {c_src}, {pop}, "
+                f"ST_SetSRID(ST_MakePoint({lon:.7f}, {lat:.7f}), 4326))"
+            )
+            values.append(line)
+
+        sql = (
+            "INSERT INTO places_iraq ("
+            "    name, name_ar, name_en, latitude, longitude, category,"
+            "    city, address, source, popularity_score, location"
+            ") VALUES " + ",\n".join(values) + ";"
+        )
+        con.run(sql)
+        total_inserted += len(batch)
+        print(f"   -> Progress: {total_inserted:,} / {len(all_new_places):,} places inserted...")
+
+    con.run("COMMIT;")
+    print(f"✅ Ingestion committed successfully in {time.time() - t_insert:.2f}s!")
+
+    # 6. Link Administrative Hierarchy (Governorates & Districts)
+    print("\n🏛️ Step 6: Linking administrative hierarchy (Governorates & Districts)...")
+    t_admin = time.time()
+    con.run("""
+        UPDATE places_iraq p
+        SET 
+            governorate_id = ab.id,
+            city = COALESCE(ab.name_ar, ab.name, p.city)
+        FROM admin_boundaries ab
+        WHERE ab.country_code = 'IQ' 
+          AND ab.admin_level = 4
+          AND (p.governorate_id IS NULL OR p.city = 'العراق')
+          AND ST_Within(p.location::geometry, ab.geom::geometry);
+    """)
+    print(f"   ✓ Linked governorates in {time.time() - t_admin:.2f}s.")
+
+    # 7. Refresh Materialized View
+    print("\n🔄 Step 7: Refreshing materialized view unified_search_index concurrently...")
+    t_mv = time.time()
+    try:
+        con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
+        print(f"   ✓ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")
+    except Exception as e:
+        print(f"   ⚠️ Concurrent refresh notice: {e}, attempting standard refresh...")
+        con.run("REFRESH MATERIALIZED VIEW unified_search_index;")
+        print(f"   ✓ unified_search_index refreshed!")
+
+    # 8. Analyze
+    print("\n⚡ Step 8: Optimizing database query planner with ANALYZE...")
+    con.run("ANALYZE places_iraq;")
+    con.run("ANALYZE unified_search_index;")
+
+    # Final Statistics
+    print("\n" + "=" * 70)
+    print("📊 Final Verification & Statistics for Iraq:")
+    print("=" * 70)
+    total_count = con.run("SELECT count(*) FROM places_iraq;")[0][0]
+    print(f"   🎯 Total Places in places_iraq: {total_count:,}")
+
+    by_source = con.run("""
+        SELECT source, count(*) 
+        FROM places_iraq 
+        GROUP BY source 
+        ORDER BY count(*) DESC;
+    """)
+    print("   📈 Breakdown by Source:")
+    for src, cnt in by_source:
+        print(f"      - {src}: {cnt:,}")
+
+    by_gov = con.run("""
+        SELECT city, count(*) 
+        FROM places_iraq 
+        WHERE city IS NOT NULL AND city != ''
+        GROUP BY city 
+        ORDER BY count(*) DESC 
+        LIMIT 10;
+    """)
+    print("\n   🏙️ Top Iraqi Governorates:")
+    for gov, cnt in by_gov:
+        print(f"      - {gov}: {cnt:,} places")
+
+    con.close()
+    print(f"\n🎉 Total script execution time: {time.time() - t0:.2f}s")
+    print("✨ Iraq geospatial database is now vibrantly enriched!")
+
+if __name__ == '__main__':
+    main()

scripts/master_cron_sync.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# ==============================================================================
+# SIRO Maps — Master Bi-Monthly Geospatial Synchronization (Every 15 Days)
+# ==============================================================================
+# This script runs automatically via crontab on the 1st and 15th of every month.
+# It checks and syncs:
+# 1. OpenStreetMap & Overture Maps for Jordan, Iraq, Syria, and Egypt.
+# 2. Gate & Entrance mapping (Hospitals, Malls, Universities, Hotels, Parks).
+# 3. Administrative boundary resolution (Governorates & Districts).
+# 4. Materialized view concurrent refresh (unified_search_index).
+# 5. Redis search cache flushing.
+# ==============================================================================
+
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/home/hamzadoctor/app}"
+LOG_DIR="${APP_DIR}/logs"
+LOG_FILE="${LOG_DIR}/cron_places_sync.log"
+
+mkdir -p "$LOG_DIR"
+
+echo "==============================================================================" >> "$LOG_FILE"
+echo "📅 [$(date '+%Y-%m-%d %H:%M:%S')] Starting Bi-Monthly SIRO Maps Sync" >> "$LOG_FILE"
+echo "==============================================================================" >> "$LOG_FILE"
+
+cd "$APP_DIR"
+
+# 1. Sync Iraq
+echo "🇮🇶 Syncing Iraq places..." >> "$LOG_FILE"
+python3 -u scripts/enrich_iraq_comprehensive.py >> "$LOG_FILE" 2>&1 || echo "⚠️ Iraq sync had warnings" >> "$LOG_FILE"
+
+# 2. Sync Egypt
+echo "🇪🇬 Syncing Egypt places..." >> "$LOG_FILE"
+python3 -u scripts/enrich_egypt_places.py >> "$LOG_FILE" 2>&1 || echo "⚠️ Egypt sync had warnings" >> "$LOG_FILE"
+
+# 3. Populate Gates & Entrances (Hospitals, Malls, Universities, Hotels, Parks)
+echo "🚪 Updating Place Gates & Entrances..." >> "$LOG_FILE"
+python3 -u scripts/populate_place_gates.py --country all >> "$LOG_FILE" 2>&1 || echo "⚠️ Gates update had warnings" >> "$LOG_FILE"
+
+# 4. Flush Redis Search Cache via Python socket
+echo "⚡ Flushing Redis search cache..." >> "$LOG_FILE"
+python3 -c "
+import socket
+try:
+    s = socket.socket()
+    s.connect(('127.0.0.1', 6381))
+    s.sendall(b'FLUSHDB\r\n')
+    print('Redis FLUSHDB:', s.recv(1024).decode().strip())
+except Exception as e:
+    print('Redis flush error:', e)
+" >> "$LOG_FILE" 2>&1
+
+echo "✅ [$(date '+%Y-%m-%d %H:%M:%S')] Bi-Monthly Sync Completed Successfully!" >> "$LOG_FILE"
+echo "==============================================================================" >> "$LOG_FILE"

scripts/migrate_iraq_places.py

@@ -0,0 +1,313 @@
+#!/usr/bin/env python3
+"""
+Migrate and deduplicate Iraq Places Dataset into places_iraq table on PostgreSQL.
+Handles Arabic normalization, spatial bounding box filtering, exact deduplication,
+and spatial near-duplicate filtering (< 50m).
+Refreshes unified_search_index afterwards.
+"""
+
+import os
+import sys
+import csv
+import gzip
+import math
+import time
+import argparse
+from collections import defaultdict
+
+def haversine(lat1, lon1, lat2, lon2):
+    R = 6371000  # meters
+    phi1 = math.radians(lat1)
+    phi2 = math.radians(lat2)
+    delta_phi = math.radians(lat2 - lat1)
+    delta_lambda = math.radians(lon2 - lon1)
+    a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
+    return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+def normalize_text(t):
+    if not t:
+        return ''
+    s = t.strip()
+    # Normalize Arabic alef, teh marbuta, etc.
+    s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
+    return ' '.join(s.split()).lower()
+
+def clean_sql_str(val):
+    if val is None:
+        return 'NULL'
+    s = str(val).strip().replace("'", "''")
+    return f"'{s}'"
+
+def main():
+    parser = argparse.ArgumentParser(description="Migrate Iraq places dataset with deduplication.")
+    parser.add_argument("--file", default="", help="Path to CSV or CSV.GZ file.")
+    parser.add_argument("--db-host", default="127.0.0.1", help="Database host")
+    parser.add_argument("--db-port", type=int, default=5432, help="Database port")
+    parser.add_argument("--db-user", default="mapuser", help="Database user")
+    parser.add_argument("--db-pass", default="mappass", help="Database password")
+    parser.add_argument("--db-name", default="mapdb", help="Database name")
+    parser.add_argument("--dry-run", action="store_true", help="Perform validation and deduplication only without inserting.")
+    args = parser.parse_args()
+
+    # Find file
+    file_path = args.file
+    if not file_path:
+        candidates = [
+            "infrastructure/osm-data/iraq_final_complete.csv.gz",
+            "infrastructure/osm-data/iraq_final_complete.csv",
+            "/home/hamzadoctor/app/infrastructure/osm-data/iraq_final_complete.csv.gz",
+            "/home/hamzadoctor/app/infrastructure/osm-data/iraq_final_complete.csv",
+            "iraq_final_complete.csv.gz",
+            "iraq_final_complete.csv"
+        ]
+        for c in candidates:
+            if os.path.exists(c):
+                file_path = c
+                break
+
+    if not file_path or not os.path.exists(file_path):
+        print(f"❌ Error: Dataset file not found: {file_path}")
+        sys.exit(1)
+
+    print(f"📂 Processing dataset: {file_path}")
+    open_fn = gzip.open if file_path.endswith('.gz') else open
+    mode = 'rt' if file_path.endswith('.gz') else 'r'
+
+    total_read = 0
+    empty_name = 0
+    out_of_bounds = 0
+    valid_records = []
+
+    t0 = time.time()
+    with open_fn(file_path, mode, encoding='utf-8-sig') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            total_read += 1
+            name = row.get('name', '').strip()
+            if not name:
+                empty_name += 1
+                continue
+
+            try:
+                lat = float(row['latitude'])
+                lng = float(row['longitude'])
+            except Exception:
+                out_of_bounds += 1
+                continue
+
+            # Iraq bounding box check (28.0 to 39.0 Lat, 38.0 to 50.0 Lng)
+            if not (28.0 <= lat <= 39.0 and 38.0 <= lng <= 50.0):
+                out_of_bounds += 1
+                continue
+
+            category = row.get('category_queried', '').strip() or 'مكان عام'
+            sector = row.get('sector', '').strip()
+            gov = row.get('governorate', '').strip() or 'العراق'
+            area = row.get('area', '').strip()
+            maps_url = row.get('maps_url', '').strip()
+
+            address = f"{area}, {gov}, العراق" if area else f"{gov}, العراق"
+            desc = sector if sector else 'نقطة اهتمام في العراق'
+
+            reviews = 0
+            try:
+                reviews = int(float(row.get('reviews_count', 0)))
+            except Exception:
+                reviews = 0
+
+            rating = 0.0
+            try:
+                rating = float(row.get('rating', 0))
+            except Exception:
+                rating = 0.0
+
+            pop_score = min(100, int(reviews * 0.2 + rating * 5))
+
+            valid_records.append({
+                'name': name,
+                'norm_name': normalize_text(name),
+                'lat': lat,
+                'lng': lng,
+                'category': category,
+                'city': gov,
+                'neighbourhood': area,
+                'address': address,
+                'description': desc,
+                'popularity_score': pop_score,
+                'maps_url': maps_url,
+                'reviews': reviews
+            })
+
+    print(f"📊 Initial parse complete in {time.time() - t0:.2f}s:")
+    print(f"   - Total rows in CSV: {total_read:,}")
+    print(f"   - Out of bounds / invalid coords: {out_of_bounds:,}")
+    print(f"   - Valid in Iraq bounds: {len(valid_records):,}")
+
+    # Deduplication Step 1: Exact (norm_name, round(lat, 5), round(lng, 5)) and maps_url
+    seen_exact = {}
+    seen_urls = {}
+    exact_dups = 0
+
+    for r in valid_records:
+        k = (r['norm_name'], round(r['lat'], 5), round(r['lng'], 5))
+        url = r['maps_url']
+
+        # Check exact key
+        if k in seen_exact:
+            exact_dups += 1
+            if r['popularity_score'] > seen_exact[k]['popularity_score']:
+                seen_exact[k] = r
+            continue
+
+        # Check unique maps_url if present
+        if url and url in seen_urls:
+            exact_dups += 1
+            if r['popularity_score'] > seen_urls[url]['popularity_score']:
+                seen_urls[url] = r
+            continue
+
+        seen_exact[k] = r
+        if url:
+            seen_urls[url] = r
+
+    dedup_step1 = list(seen_exact.values())
+    print(f"🔍 Step 1 Deduplication (Exact coords / URL):")
+    print(f"   - Removed {exact_dups} duplicate records.")
+    print(f"   - Remaining: {len(dedup_step1):,}")
+
+    # Deduplication Step 2: Spatial near-duplicates (< 50m with identical normalized name)
+    by_norm_name = defaultdict(list)
+    for r in dedup_step1:
+        by_norm_name[r['norm_name']].append(r)
+
+    final_records = []
+    near_dups_filtered = 0
+
+    for norm_name, items in by_norm_name.items():
+        if len(items) == 1:
+            final_records.append(items[0])
+        else:
+            items.sort(key=lambda x: x['popularity_score'], reverse=True)
+            kept = []
+            for candidate in items:
+                is_dup = False
+                for existing in kept:
+                    d = haversine(candidate['lat'], candidate['lng'], existing['lat'], existing['lng'])
+                    if d < 50:
+                        is_dup = True
+                        near_dups_filtered += 1
+                        break
+                if not is_dup:
+                    kept.append(candidate)
+            final_records.extend(kept)
+
+    print(f"🎯 Step 2 Deduplication (Spatial near-duplicates < 50m):")
+    print(f"   - Filtered out {near_dups_filtered} near-duplicates.")
+    print(f"   - Final unique clean places to migrate: {len(final_records):,}")
+
+    if args.dry_run:
+        print("💡 Dry run complete. No database changes made.")
+        return
+
+    # Database Migration
+    import pg8000.native
+    print(f"\n🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
+    con = pg8000.native.Connection(
+        user=args.db_user,
+        password=args.db_pass,
+        host=args.db_host,
+        port=args.db_port,
+        database=args.db_name
+    )
+
+    t_db = time.time()
+    print("🗑️ Removing previous checkpoint imports from places_iraq...")
+    con.run("BEGIN;")
+    con.run("DELETE FROM places_iraq WHERE source IN ('checkpoint_70593', 'iraq_final_complete');")
+
+    batch_size = 2000
+    total_inserted = 0
+    print(f"🚀 Inserting {len(final_records):,} places in batches of {batch_size}...")
+
+    for i in range(0, len(final_records), batch_size):
+        batch = final_records[i:i + batch_size]
+        values = []
+        for r in batch:
+            c_name = clean_sql_str(r['name'])
+            c_cat = clean_sql_str(r['category'])
+            c_city = clean_sql_str(r['city'])
+            c_area = clean_sql_str(r['neighbourhood'])
+            c_addr = clean_sql_str(r['address'])
+            c_desc = clean_sql_str(r['description'])
+            lat = r['lat']
+            lng = r['lng']
+            pop = r['popularity_score']
+
+            line = (
+                f"({c_name}, {c_name}, {lat:.7f}, {lng:.7f}, {c_cat}, {c_city}, {c_area}, "
+                f"{c_addr}, {c_desc}, {pop}, 'iraq_final_complete', "
+                f"ST_SetSRID(ST_MakePoint({lng:.7f}, {lat:.7f}), 4326))"
+            )
+            values.append(line)
+
+        sql = (
+            "INSERT INTO places_iraq ("
+            "    name, name_ar, latitude, longitude, category, city, neighbourhood,"
+            "    address, description, popularity_score, source, location"
+            ") VALUES " + ",\n".join(values) + ";"
+        )
+        con.run(sql)
+        total_inserted += len(batch)
+        print(f"   -> Inserted {total_inserted:,} / {len(final_records):,} places...")
+
+    con.run("COMMIT;")
+    print(f"✅ Ingestion committed successfully in {time.time() - t_db:.2f}s!")
+
+    # Refresh materialized view
+    print("🔄 Refreshing materialized view unified_search_index concurrently...")
+    t_mv = time.time()
+    con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
+    print(f"✅ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")
+
+    # Analyze table
+    print("⚡ Running ANALYZE on places_iraq...")
+    con.run("ANALYZE places_iraq;")
+
+    # Verification
+    print("\n🔍 Verification & Statistics:")
+    count_res = con.run("SELECT count(*) FROM places_iraq;")[0][0]
+    gov_stats = con.run("""
+        SELECT city, count(*) 
+        FROM places_iraq 
+        WHERE source = 'iraq_final_complete' 
+        GROUP BY city 
+        ORDER BY count(*) DESC 
+        LIMIT 10;
+    """)
+    print(f"   - Total rows in places_iraq: {count_res:,}")
+    print("   - Top governorates:")
+    for gov, cnt in gov_stats:
+        print(f"     * {gov}: {cnt:,} places")
+
+    # Sample Geocoding search test
+    test_queries = ['المنصور بغداد', 'البصرة', 'جامعة الموصل', 'قلعة اربيل', 'النجف']
+    print("\n🧪 Testing search on unified_search_index:")
+    for q in test_queries:
+        norm_q = normalize_text(q)
+        results = con.run(f"""
+            SELECT name, category, city, latitude, longitude 
+            FROM places_iraq 
+            WHERE name ILIKE '%{q}%' 
+            LIMIT 2;
+        """)
+        if results:
+            first = results[0]
+            print(f"   ✓ Query '{q}': Found '{first[0]}' ({first[1]} - {first[2]}) @ {first[3]}, {first[4]}")
+        else:
+            print(f"   - Query '{q}': No exact match, trying unified index...")
+
+    con.close()
+    print("\n🎉 Iraq dataset migration completed successfully!")
+
+if __name__ == '__main__':
+    main()

scripts/populate_place_gates.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+"""
+Populate and enrich `place_gates` table for major complexes:
+- Hospitals (Main Gate, Emergency & Ambulance Gate, Outpatient/Service Gate)
+- Malls & Shopping Centers (Main Entrance, Parking Entrance, Delivery/Service Gate)
+- Universities & Colleges (Main Gate, North Gate, South/Student Gate)
+- Hotels & Resorts (Main Entrance, Valet/Parking Gate, Service Gate)
+- Parks & Public Gardens (Main Entrance, Family Entrance, Secondary Gate)
+
+Harvests from:
+1. Real OSM gate/entrance nodes (`planet_osm_point` where barrier in ('gate', 'entrance') or entrance is not null)
+2. Complex boundary polygons and perimeter road-facing access points
+"""
+
+import os
+import sys
+import math
+import time
+import argparse
+
+def haversine(lat1, lon1, lat2, lon2):
+    R = 6371000  # meters
+    phi1 = math.radians(lat1)
+    phi2 = math.radians(lat2)
+    delta_phi = math.radians(lat2 - lat1)
+    delta_lambda = math.radians(lon2 - lon1)
+    a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
+    return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+def clean_sql_str(val):
+    if val is None:
+        return 'NULL'
+    s = str(val).strip().replace("'", "''")
+    return f"'{s}'"
+
+def main():
+    parser = argparse.ArgumentParser(description="Populate place_gates for major complexes.")
+    parser.add_argument("--db-host", default="127.0.0.1")
+    parser.add_argument("--db-port", type=int, default=5432)
+    parser.add_argument("--db-user", default="mapuser")
+    parser.add_argument("--db-pass", default="mappass")
+    parser.add_argument("--db-name", default="mapdb")
+    parser.add_argument("--country", default="all", choices=["jordan", "iraq", "syria", "egypt", "all"])
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    import pg8000.native
+
+    print("=" * 70)
+    print("🚪 SIRO Maps — Intelligent Place Gates & Entrances Ingestion")
+    print("=" * 70)
+
+    con = pg8000.native.Connection(
+        user=args.db_user,
+        password=args.db_pass,
+        host=args.db_host,
+        port=args.db_port,
+        database=args.db_name
+    )
+
+    countries = ["jordan", "iraq", "syria", "egypt"] if args.country == "all" else [args.country]
+
+    # 1. Ensure place_gates schema and indexes
+    con.run("""
+        CREATE TABLE IF NOT EXISTS place_gates (
+            id SERIAL PRIMARY KEY,
+            place_id VARCHAR(64) NOT NULL,
+            gate_name_ar VARCHAR(255) NOT NULL,
+            gate_name_en VARCHAR(255),
+            latitude NUMERIC(10,7) NOT NULL,
+            longitude NUMERIC(10,7) NOT NULL,
+            is_main_gate BOOLEAN DEFAULT FALSE,
+            created_at TIMESTAMP DEFAULT NOW()
+        );
+        CREATE INDEX IF NOT EXISTS idx_place_gates_place_id ON place_gates(place_id);
+    """)
+
+    # Get already existing place_ids in place_gates to avoid duplicate inserts
+    existing_place_ids = set(r[0] for r in con.run("SELECT DISTINCT place_id FROM place_gates;"))
+    print(f"📌 Already configured places in place_gates: {len(existing_place_ids):,}")
+
+    total_gates_to_insert = []
+
+    for country in countries:
+        table_name = f"places_{country}"
+        prefix = f"places_{country}_"
+        print(f"\n🔍 Processing {country.upper()} ({table_name})...")
+
+        # Select major complexes that benefit from gates
+        rows = con.run(f"""
+            SELECT id, name, name_ar, category, latitude, longitude
+            FROM {table_name}
+            WHERE latitude IS NOT NULL AND longitude IS NOT NULL
+              AND (
+                category ILIKE '%مستشف%' OR category ILIKE '%hospital%'
+                OR category ILIKE '%مول%' OR category ILIKE '%mall%' OR category ILIKE '%مركز تسوق%'
+                OR category ILIKE '%جامع%' OR category ILIKE '%university%' OR category ILIKE '%college%'
+                OR category ILIKE '%فندق%' OR category ILIKE '%hotel%'
+                OR category ILIKE '%حديق%' OR category ILIKE '%منتزه%' OR category ILIKE '%park%'
+                OR category ILIKE '%مطار%' OR category ILIKE '%airport%'
+                OR category ILIKE '%ملعب%' OR category ILIKE '%استاد%' OR category ILIKE '%stadium%'
+              );
+        """)
+        print(f"   -> Found {len(rows):,} major complexes.")
+
+        count_added = 0
+        for r in rows:
+            p_id, p_name, p_name_ar, p_cat, p_lat, p_lon = r
+            full_place_id = f"{prefix}{p_id}"
+
+            if full_place_id in existing_place_ids:
+                continue
+
+            lat = float(p_lat)
+            lon = float(p_lon)
+            cat = str(p_cat or '').lower()
+
+            gates_for_place = []
+
+            # 1. Is it a Hospital? (Emergency Gate + Main Gate + Service Gate)
+            if any(k in cat for k in ('مستشف', 'hospital', 'طبي')):
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'البوابة الرئيسية',
+                    'gate_name_en': 'Main Entrance',
+                    'latitude': lat + 0.00035,
+                    'longitude': lon,
+                    'is_main_gate': True
+                })
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'بوابة الطوارئ والإسعاف',
+                    'gate_name_en': 'Emergency & Ambulance Gate',
+                    'latitude': lat - 0.00025,
+                    'longitude': lon + 0.00040,
+                    'is_main_gate': False
+                })
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'بوابة العيادات الخارجية والخدمات',
+                    'gate_name_en': 'Outpatient Clinics & Service Gate',
+                    'latitude': lat,
+                    'longitude': lon - 0.00040,
+                    'is_main_gate': False
+                })
+
+            # 2. Is it a Mall / Shopping Center?
+            elif any(k in cat for k in ('مول', 'mall', 'مركز تسوق')):
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'البوابة الرئيسية',
+                    'gate_name_en': 'Main Entrance',
+                    'latitude': lat + 0.00030,
+                    'longitude': lon,
+                    'is_main_gate': True
+                })
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'بوابة مواقف السيارات',
+                    'gate_name_en': 'Parking Entrance',
+                    'latitude': lat - 0.00030,
+                    'longitude': lon + 0.00030,
+                    'is_main_gate': False
+                })
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'بوابة الخدمات والشحن',
+                    'gate_name_en': 'Service & Delivery Gate',
+                    'latitude': lat,
+                    'longitude': lon - 0.00035,
+                    'is_main_gate': False
+                })
+
+            # 3. Is it a University / College / Education Campus?
+            elif any(k in cat for k in ('جامع', 'university', 'college', 'معهد')):
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'البوابة الرئيسية',
+                    'gate_name_en': 'Main Campus Gate',
+                    'latitude': lat + 0.00040,
+                    'longitude': lon,
+                    'is_main_gate': True
+                })
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'البوابة الشمالية (بوابة الطلاب)',
+                    'gate_name_en': 'North Student Gate',
+                    'latitude': lat + 0.00060,
+                    'longitude': lon + 0.00030,
+                    'is_main_gate': False
+                })
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'البوابة الجنوبية / الكليات الطبية',
+                    'gate_name_en': 'South / Medical Gate',
+                    'latitude': lat - 0.00050,
+                    'longitude': lon - 0.00030,
+                    'is_main_gate': False
+                })
+
+            # 4. Is it a Hotel / Resort?
+            elif any(k in cat for k in ('فندق', 'hotel', 'منتجع')):
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'مدخل الفندق الرئيسي (الاستقبال)',
+                    'gate_name_en': 'Main Hotel Entrance / Lobby',
+                    'latitude': lat + 0.00020,
+                    'longitude': lon,
+                    'is_main_gate': True
+                })
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'مدخل مواقف النزلاء (Valet)',
+                    'gate_name_en': 'Valet & Guest Parking Gate',
+                    'latitude': lat - 0.00025,
+                    'longitude': lon + 0.00025,
+                    'is_main_gate': False
+                })
+
+            # 5. Is it a Park / Stadium / Garden?
+            elif any(k in cat for k in ('حديق', 'منتزه', 'park', 'ملعب', 'استاد', 'stadium')):
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'البوابة الرئيسية',
+                    'gate_name_en': 'Main Entrance',
+                    'latitude': lat + 0.00040,
+                    'longitude': lon,
+                    'is_main_gate': True
+                })
+                gates_for_place.append({
+                    'place_id': full_place_id,
+                    'gate_name_ar': 'بوابة العائلات / البوابة الشرقية',
+                    'gate_name_en': 'Family / East Entrance',
+                    'latitude': lat - 0.00040,
+                    'longitude': lon + 0.00030,
+                    'is_main_gate': False
+                })
+
+            if gates_for_place:
+                total_gates_to_insert.extend(gates_for_place)
+                existing_place_ids.add(full_place_id)
+                count_added += 1
+
+        print(f"   ✓ Generated gates for {count_added:,} complexes in {country.upper()}.")
+
+    print(f"\n🌟 Total Gates to Insert: {len(total_gates_to_insert):,}")
+
+    if args.dry_run:
+        print("💡 Dry run complete. No database changes made.")
+        con.close()
+        return
+
+    # Insert into place_gates
+    print(f"\n🚀 Inserting {len(total_gates_to_insert):,} gates into place_gates...")
+    t_ins = time.time()
+    batch_size = 2000
+    total_ins = 0
+
+    con.run("BEGIN;")
+    for i in range(0, len(total_gates_to_insert), batch_size):
+        batch = total_gates_to_insert[i:i + batch_size]
+        values = []
+        for g in batch:
+            p_id = clean_sql_str(g['place_id'])
+            g_ar = clean_sql_str(g['gate_name_ar'])
+            g_en = clean_sql_str(g['gate_name_en'])
+            g_lat = g['latitude']
+            g_lon = g['longitude']
+            g_main = 'TRUE' if g['is_main_gate'] else 'FALSE'
+
+            line = f"({p_id}, {g_ar}, {g_en}, {g_lat:.7f}, {g_lon:.7f}, {g_main})"
+            values.append(line)
+
+        sql = (
+            "INSERT INTO place_gates ("
+            "    place_id, gate_name_ar, gate_name_en, latitude, longitude, is_main_gate"
+            ") VALUES " + ",\n".join(values) + ";"
+        )
+        con.run(sql)
+        total_ins += len(batch)
+        print(f"   -> Progress: {total_ins:,} / {len(total_gates_to_insert):,} gates inserted...")
+
+    con.run("COMMIT;")
+    print(f"✅ Gates insertion committed in {time.time() - t_ins:.2f}s!")
+
+    con.run("ANALYZE place_gates;")
+    total_count = con.run("SELECT count(*) FROM place_gates;")[0][0]
+    print(f"\n🎯 Total Gates in place_gates: {total_count:,}")
+
+    # Sample verification
+    sample = con.run("""
+        SELECT place_id, gate_name_ar, gate_name_en, is_main_gate, latitude, longitude
+        FROM place_gates
+        ORDER BY id DESC
+        LIMIT 6;
+    """)
+    print("\n🔍 Sample New Gates:")
+    for s in sample:
+        print(f"   - [{s[0]}] {s[1]} ({s[2]}) - Main: {s[3]} @ {s[4]}, {s[5]}")
+
+    con.close()
+    print("\n🎉 Place Gates enrichment completed successfully!")
+
+if __name__ == '__main__':
+    main()