feat: sync scripts, scraper, and sql tools from server

2026-08-16 13:19:10 +03:00
parent 32865e78df
commit 3136a2b7aa
14 changed files with 28844 additions and 0 deletions
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Download Existing Syria Data from Server to Mac
+# To be run on the Mac terminal
+
+SERVER_IP="188.68.36.205"
+SERVER_USER="hamzadoctor"
+REMOTE_PATH="/home/hamzadoctor/app"
+KEY_PATH="/Users/hamzaaleghwairyeen/.ssh/doctory-key"
+
+echo "🌍 Connecting to server to dump existing Syria landmarks..."
+
+# 1. Generate dump on server
+ssh -i "$KEY_PATH" "$SERVER_USER@$SERVER_IP" "docker exec -t map-db pg_dump -U mapuser -d mapdb -t places_syria --data-only --inserts > $REMOTE_PATH/existing_syria_data.sql"
+
+# 2. Download the dump to local Mac
+echo "📥 Downloading dump to Mac..."
+rsync -avz -e "ssh -i $KEY_PATH" "$SERVER_USER@$SERVER_IP:$REMOTE_PATH/existing_syria_data.sql" infrastructure/docker/postgis/
+
+echo "✅ Download complete: infrastructure/docker/postgis/existing_syria_data.sql"
+echo "Now run ./infrastructure/scripts/local_db_prep.sh to merge."
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Script to execute Syria data import on the server
+# To be run from /home/hamzadoctor/app
+
+APP_DIR="/home/hamzadoctor/app"
+CSV_FILE="syria_final_complete.csv"
+SQL_FILE="infrastructure/docker/postgis/import_syria_csv.sql"
+
+echo "📦 Transferring CSV to PostGIS container..."
+docker cp "$APP_DIR/$CSV_FILE" map-db:/tmp/syria_data.csv
+
+echo "💾 Running SQL import script..."
+# We use docker compose exec db psql to run the logic
+# First, update the SQL logic to use the /tmp path for COPY
+# We'll create a temporary SQL wrapper to handle the COPY command with the correct path
+
+docker exec -i map-db psql -U mapuser -d mapdb <<EOF
+-- Load schema
+\i /home/hamzadoctor/app/$SQL_FILE
+
+-- Perform COPY (must be done in the container pointing to /tmp/syria_data.csv)
+CREATE TEMP TABLE staging_syria_temp (
+    name TEXT,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    category TEXT,
+    address TEXT,
+    description TEXT,
+    created_at TIMESTAMP
+);
+
+COPY staging_syria_temp(name, latitude, longitude, category, address, description, created_at) 
+FROM '/tmp/syria_data.csv' 
+WITH (FORMAT csv, HEADER false, QUOTE '"', ENCODING 'UTF8');
+
+INSERT INTO staging_syria SELECT * FROM staging_syria_temp;
+DROP TABLE staging_syria_temp;
+
+-- Final merge logic is already in the SQL file loaded via \i
+EOF
+
+echo "✅ Import completed successfully!"
@@ -0,0 +1,52 @@
+#!/bin/bash
+# Script to import Egypt and Jordan OSM data into the PostGIS database
+# نص برمجي لاستيراد بيانات مصر والأردن إلى قاعدة البيانات
+
+set -e
+
+DATA_DIR="./infrastructure/osm-data"
+DB_USER=${POSTGRES_USER:-mapuser}
+DB_NAME=${POSTGRES_DB:-mapdb}
+
+echo "🌍 Starting MENA Regional Data Import..."
+echo "🌍 البدء في استيراد البيانات الإقليمية..."
+
+# 1. Check for files
+if [ ! -f "$DATA_DIR/jordan-latest.osm.pbf" ]; then
+    echo "❌ Jordan PBF missing. Please run setup-osm.sh first."
+    exit 1
+fi
+
+if [ ! -f "$DATA_DIR/egypt-latest.osm.pbf" ]; then
+    echo "📥 Downloading Egypt OSM PBF..."
+    curl -L https://download.geofabrik.de/africa/egypt-latest.osm.pbf -o "$DATA_DIR/egypt-latest.osm.pbf"
+fi
+
+# 2. Detect Docker Compose Command
+if command -v docker-compose &> /dev/null; then
+    DOKCER_COMPOSE="docker-compose"
+else
+    DOKCER_COMPOSE="docker compose"
+fi
+
+echo "🐳 Using $DOKCER_COMPOSE..."
+
+# 3. Import Jordan (Append)
+echo "🇯🇴 Importing Jordan data (Append mode)..."
+$DOKCER_COMPOSE --profile import run --rm osm-import osm2pgsql \
+    --append --slim --cache 1000 \
+    --database "$DB_NAME" --host db --user "$DB_USER" \
+    /data/jordan-latest.osm.pbf
+
+# 4. Import Egypt (Append)
+echo "🇪🇬 Importing Egypt data (Append mode)..."
+$DOKCER_COMPOSE --profile import run --rm osm-import osm2pgsql \
+    --append --slim --cache 1000 \
+    --database "$DB_NAME" --host db --user "$DB_USER" \
+    /data/egypt-latest.osm.pbf
+
+echo "✅ Import complete. Restarting tile and routing services..."
+$DOKCER_COMPOSE restart martin routing
+
+echo "🚀 MENA Regional mapping is now live!"
+echo "🚀 تم تفعيل خرائط المنطقة بنجاح!"
@@ -0,0 +1,110 @@
+#!/bin/bash
+# Local Database Preparation & Merge Script
+# To be run on the Mac terminal after download_server_data.sh
+
+set -e
+
+echo "🚀 Starting local database merge and preparation..."
+
+# 1. Start the DB container
+echo "📦 Starting PostGIS container..."
+docker compose up -d db
+
+# 2. Wait for DB to be ready
+echo "⏳ Waiting for database to be ready..."
+until docker compose exec -T db pg_isready -U mapuser -d mapdb; do
+  sleep 2
+done
+
+# 3. Initialize Schema & Import Existing Server Data
+echo "🛠️ Initializing schema and loading server data..."
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+DROP TABLE IF EXISTS places_syria;
+CREATE TABLE places_syria (
+    id SERIAL PRIMARY KEY,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    name TEXT,
+    name_ar TEXT,
+    name_en TEXT,
+    address TEXT,
+    category TEXT,
+    neighbourhood TEXT,
+    city TEXT,
+    description TEXT,
+    created_at TIMESTAMP DEFAULT NOW(),
+    source TEXT,
+    location GEOMETRY(Point, 4326)
+);
+
+CREATE OR REPLACE FUNCTION sync_place_location() RETURNS trigger AS \$\$
+BEGIN
+  IF NEW.latitude IS NOT NULL AND NEW.longitude IS NOT NULL THEN
+    NEW.location := ST_SetSRID(ST_MakePoint(CAST(NEW.longitude AS FLOAT), CAST(NEW.latitude AS FLOAT)), 4326);
+  END IF;
+  RETURN NEW;
+END;
+\$\$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_sync_place_location ON places_syria;
+CREATE TRIGGER trg_sync_place_location
+BEFORE INSERT OR UPDATE ON places_syria
+FOR EACH ROW EXECUTE FUNCTION sync_place_location();
+"
+
+# Load existing data if available
+if [ -f "infrastructure/docker/postgis/existing_syria_data.sql" ]; then
+    echo "📥 Loading existing server data..."
+    docker compose exec -T db psql -U mapuser -d mapdb < infrastructure/docker/postgis/existing_syria_data.sql
+fi
+
+# 4. Import New CSV Data
+echo "📥 Importing new CSV landmark data..."
+docker cp infrastructure/docker/postgis/syria_final_complete.csv map-db:/tmp/syria_data.csv
+
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+DROP TABLE IF EXISTS staging_syria;
+CREATE TABLE staging_syria (
+    name TEXT,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    category TEXT,
+    address TEXT,
+    description TEXT,
+    created_at TIMESTAMP
+);
+
+-- HEADER true fixes the 'invalid input syntax' error
+COPY staging_syria(name, latitude, longitude, category, address, description, created_at) 
+FROM '/tmp/syria_data.csv' 
+WITH (FORMAT csv, HEADER true, QUOTE '\"', ENCODING 'UTF8');
+
+-- Merge into places_syria while avoiding duplicates
+-- We check for name + spatial proximity (approx 50m)
+INSERT INTO places_syria (name, name_ar, latitude, longitude, category, address, description, source, created_at)
+SELECT 
+    name, 
+    name, 
+    latitude, 
+    longitude, 
+    category, 
+    address, 
+    description, 
+    'csv_import_2026_04',
+    COALESCE(created_at, NOW())
+FROM staging_syria s
+WHERE NOT EXISTS (
+    SELECT 1 FROM places_syria p 
+    WHERE (p.name = s.name OR p.name_ar = s.name)
+    AND ST_DWithin(
+        ST_SetSRID(ST_MakePoint(CAST(p.longitude AS FLOAT), CAST(p.latitude AS FLOAT)), 4326)::geography, 
+        ST_SetSRID(ST_MakePoint(CAST(s.longitude AS FLOAT), CAST(s.latitude AS FLOAT)), 4326)::geography, 
+        50
+    )
+);
+
+DROP TABLE staging_syria;
+"
+
+echo "✅ Local database merge complete."
+echo "Run ./infrastructure/scripts/local_verify.sh to review the results."
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Local Verification Script for Syria Landmarks
+# To be run on the Mac terminal
+
+set -e
+
+echo "📊 --- Local Syria Data Audit ---"
+
+# 1. Total Count
+TOTAL=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "SELECT count(*) FROM places_syria;")
+echo "📍 Total Landmarks: $TOTAL"
+
+# 2. Category Distribution
+echo "🗂️  Category Distribution:"
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+SELECT category, count(*) as count 
+FROM places_syria 
+GROUP BY category 
+ORDER BY count DESC 
+LIMIT 10;
+"
+
+# 3. Spatial Bounds Check (Damascus region)
+echo "🌍 Spatial Check (Damascus):"
+DM_COUNT=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "
+SELECT count(*) FROM places_syria 
+WHERE latitude BETWEEN 33.4 AND 33.6 AND longitude BETWEEN 36.2 AND 36.4;
+")
+echo "🏙️  Landmarks in Damascus area: $DM_COUNT"
+
+# 4. Generate Export if user is satisfied
+echo "💾 Generating SQL Export..."
+docker compose exec -T db pg_dump -U mapuser -d mapdb -t places_syria --data-only --inserts > infrastructure/docker/postgis/syria_export.sql
+
+echo "--------------------------------"
+echo "✅ Verification complete. Export saved to: infrastructure/docker/postgis/syria_export.sql"
+echo "If you are happy with the results, run sync_to_server.sh to push the data."
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Server Restoration Script for Syria Landmarks
+# To be run on the server terminal
+
+set -e
+
+APP_DIR="/home/hamzadoctor/app"
+EXPORT_FILE="infrastructure/docker/postgis/syria_export.sql"
+
+echo "🔄 Restoring Syria landmarks to production database..."
+
+# 1. Clean existing manual imports to avoid PK conflicts
+echo "🧹 Clearing existing manual entries..."
+docker exec -i map-db psql -U mapuser -d mapdb -c "DELETE FROM places_syria WHERE source = 'manual_import_2026_04';"
+
+# 2. Inject the SQL dump
+echo "📥 Injecting SQL dump..."
+docker exec -i map-db psql -U mapuser -d mapdb < "$APP_DIR/$EXPORT_FILE"
+
+# 3. Refresh API cache
+echo "🧹 Flushing Redis..."
+docker exec -i map-redis redis-cli flushall
+
+echo "✅ Restoration complete! Syria landmarks are live."
@@ -0,0 +1,33 @@
+#!/bin/bash
+# script to download Jordan OSM data
+# نص برمجبي لتحميل بيانات الخرائط للأردن
+
+set -e
+
+DATA_DIR="./infrastructure/osm-data"
+PBF_URL="https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
+
+echo "📍 Starting Jordan Map Setup..."
+echo "📍 البدء في إعداد خرائط الأردن..."
+
+mkdir -p $DATA_DIR
+
+if [ ! -f "$DATA_DIR/jordan-latest.osm.pbf" ]; then
+    echo "📥 Downloading Jordan OSM PBF from Geofabrik..."
+    echo "📥 جاري تحميل بيانات الخرائط من Geofabrik..."
+    curl -L $PBF_URL -o "$DATA_DIR/jordan-latest.osm.pbf"
+else
+    echo "✅ Jordan PBF already exists."
+    echo "✅ ملف البيانات موجود بالفعل."
+fi
+
+# Note on MBTiles:
+# Tileserver-GL requires an .mbtiles file to serve tiles locally.
+# You can generate one from the .pbf using 'tilemaker' or download a free extract 
+# from MapTiler (https://maptiler.com/data) and place it in $DATA_DIR/jordan.mbtiles
+
+echo "⚠️  Important: To serve tiles locally, you need 'jordan.mbtiles' in $DATA_DIR"
+echo "⚠️  تنبيه: لتشغيل الخرائط محلياً، يجب توفير ملف 'jordan.mbtiles' في المجلد المذكور"
+
+echo "🚀 Setup complete. Please ensure jordan.mbtiles is present before running 'docker-compose up'."
+echo "🚀 اكتمل الإعداد. تأكد من وجود ملف mbtiles قبل تشغيل النظام."
@@ -0,0 +1,60 @@
+#!/bin/bash
+# --------------------------------------------------------------------------
+# Intaleq Map Platform - 10-Day Update Script
+# سكربت تحديث خرائط انطلاقة - التحديث الدوري (كل 10 أيام)
+# --------------------------------------------------------------------------
+
+set -e # Exit on error
+
+echo "🚀 Starting 10-day map update..."
+
+# 1. Configuration (From .env or defaults)
+PBF_FILE="/data/jordan-latest.osm.pbf"
+SOURCE_URL="https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
+APP_DIR="/home/hamzadoctor/app"
+
+cd "$APP_DIR"
+
+# 2. Download latest OSM data (Running on Host)
+# تحميل أحدث البيانات للأردن وسوريا
+echo "🌍 Downloading latest OpenStreetMap data for Jordan & Syria..."
+wget -O "infrastructure/osm-data/jordan-latest.osm.pbf.new" "https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
+wget -O "infrastructure/osm-data/syria-latest.osm.pbf.new" "https://download.geofabrik.de/asia/syria-latest.osm.pbf"
+
+mv "infrastructure/osm-data/jordan-latest.osm.pbf.new" "infrastructure/osm-data/jordan-latest.osm.pbf"
+mv "infrastructure/osm-data/syria-latest.osm.pbf.new" "infrastructure/osm-data/syria-latest.osm.pbf"
+
+# 3. Import new data to PostGIS
+# استيراد البيانات إلى قاعدة البيانات
+echo "💾 Importing Jordan (Create)..."
+docker compose --profile import run --rm osm-import osm2pgsql --create --slim --cache 1000 --database mapdb --host db --user mapuser /data/jordan-latest.osm.pbf
+
+echo "💾 Importing Syria (Append)..."
+docker compose --profile import run --rm osm-import osm2pgsql --append --slim --cache 1000 --database mapdb --host db --user mapuser /data/syria-latest.osm.pbf
+
+# 4. Merge Data (Jordan + Syria)
+OSM_FILE="infrastructure/osm-data/region.osm.pbf"
+echo "🗺️ Merging Jordan and Syria data into $OSM_FILE..."
+osmium merge infrastructure/osm-data/jordan-latest.osm.pbf infrastructure/osm-data/syria-latest.osm.pbf -o $OSM_FILE --overwrite
+
+# 5. Spatial Integrity check for Geocoding (Landmarks)
+# Purge any legacy landmarks outside the expanded region
+echo "📍 Syncing user-submitted landmarks and purging invalid coordinates..."
+DELETED_COUNT=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "DELETE FROM places_syria WHERE longitude < 34 OR longitude > 43 OR latitude < 29 OR latitude > 38;")
+echo "   ✅ Purged ${DELETED_COUNT//[[:space:]]/} invalid landmarks outside the expanded region."
+
+# Force Spatial Geometry Update
+docker compose exec -T db psql -U mapuser -d mapdb -c "UPDATE places_syria SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326) WHERE location IS NULL OR latitude IS NOT NULL;"
+
+# 6. Rebuild Routing Index (GraphHopper)
+echo "🚗 Rebuilding GraphHopper routing index (This may take ~5-8 minutes)..."
+docker compose stop routing
+# Delete old cache - VERY IMPORTANT to force full re-index
+rm -rf infrastructure/osm-data/graph-cache infrastructure/osm-data/default-gh
+docker compose up -d routing
+
+# 7. Final Cleanup & Cache Flush
+echo "🧹 Clearing Redis traffic cache..."
+docker compose exec -T redis redis-cli flushall
+
+echo "Done! Map platform is now fully synchronized with Jordan & Syria roads (including Damascus)."