2026-04-12-1

2026-04-12 22:35:38 +03:00
parent e3799c422c
commit 5ebd7ea3b1
84 changed files with 532 additions and 5136 deletions
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Download Existing Syria Data from Server to Mac
+# To be run on the Mac terminal
+
+SERVER_IP="188.68.36.205"
+SERVER_USER="hamzadoctor"
+REMOTE_PATH="/home/hamzadoctor/app"
+KEY_PATH="/Users/hamzaaleghwairyeen/.ssh/doctory-key"
+
+echo "🌍 Connecting to server to dump existing Syria landmarks..."
+
+# 1. Generate dump on server
+ssh -i "$KEY_PATH" "$SERVER_USER@$SERVER_IP" "docker exec -t map-db pg_dump -U mapuser -d mapdb -t places_syria --data-only --inserts > $REMOTE_PATH/existing_syria_data.sql"
+
+# 2. Download the dump to local Mac
+echo "📥 Downloading dump to Mac..."
+rsync -avz -e "ssh -i $KEY_PATH" "$SERVER_USER@$SERVER_IP:$REMOTE_PATH/existing_syria_data.sql" infrastructure/docker/postgis/
+
+echo "✅ Download complete: infrastructure/docker/postgis/existing_syria_data.sql"
+echo "Now run ./infrastructure/scripts/local_db_prep.sh to merge."
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Script to execute Syria data import on the server
+# To be run from /home/hamzadoctor/app
+
+APP_DIR="/home/hamzadoctor/app"
+CSV_FILE="syria_final_complete.csv"
+SQL_FILE="infrastructure/docker/postgis/import_syria_csv.sql"
+
+echo "📦 Transferring CSV to PostGIS container..."
+docker cp "$APP_DIR/$CSV_FILE" map-db:/tmp/syria_data.csv
+
+echo "💾 Running SQL import script..."
+# We use docker compose exec db psql to run the logic
+# First, update the SQL logic to use the /tmp path for COPY
+# We'll create a temporary SQL wrapper to handle the COPY command with the correct path
+
+docker exec -i map-db psql -U mapuser -d mapdb <<EOF
+-- Load schema
+\i /home/hamzadoctor/app/$SQL_FILE
+
+-- Perform COPY (must be done in the container pointing to /tmp/syria_data.csv)
+CREATE TEMP TABLE staging_syria_temp (
+    name TEXT,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    category TEXT,
+    address TEXT,
+    description TEXT,
+    created_at TIMESTAMP
+);
+
+COPY staging_syria_temp(name, latitude, longitude, category, address, description, created_at) 
+FROM '/tmp/syria_data.csv' 
+WITH (FORMAT csv, HEADER false, QUOTE '"', ENCODING 'UTF8');
+
+INSERT INTO staging_syria SELECT * FROM staging_syria_temp;
+DROP TABLE staging_syria_temp;
+
+-- Final merge logic is already in the SQL file loaded via \i
+EOF
+
+echo "✅ Import completed successfully!"
@@ -0,0 +1,110 @@
+#!/bin/bash
+# Local Database Preparation & Merge Script
+# To be run on the Mac terminal after download_server_data.sh
+
+set -e
+
+echo "🚀 Starting local database merge and preparation..."
+
+# 1. Start the DB container
+echo "📦 Starting PostGIS container..."
+docker compose up -d db
+
+# 2. Wait for DB to be ready
+echo "⏳ Waiting for database to be ready..."
+until docker compose exec -T db pg_isready -U mapuser -d mapdb; do
+  sleep 2
+done
+
+# 3. Initialize Schema & Import Existing Server Data
+echo "🛠️ Initializing schema and loading server data..."
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+DROP TABLE IF EXISTS places_syria;
+CREATE TABLE places_syria (
+    id SERIAL PRIMARY KEY,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    name TEXT,
+    name_ar TEXT,
+    name_en TEXT,
+    address TEXT,
+    category TEXT,
+    neighbourhood TEXT,
+    city TEXT,
+    description TEXT,
+    created_at TIMESTAMP DEFAULT NOW(),
+    source TEXT,
+    location GEOMETRY(Point, 4326)
+);
+
+CREATE OR REPLACE FUNCTION sync_place_location() RETURNS trigger AS \$\$
+BEGIN
+  IF NEW.latitude IS NOT NULL AND NEW.longitude IS NOT NULL THEN
+    NEW.location := ST_SetSRID(ST_MakePoint(CAST(NEW.longitude AS FLOAT), CAST(NEW.latitude AS FLOAT)), 4326);
+  END IF;
+  RETURN NEW;
+END;
+\$\$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_sync_place_location ON places_syria;
+CREATE TRIGGER trg_sync_place_location
+BEFORE INSERT OR UPDATE ON places_syria
+FOR EACH ROW EXECUTE FUNCTION sync_place_location();
+"
+
+# Load existing data if available
+if [ -f "infrastructure/docker/postgis/existing_syria_data.sql" ]; then
+    echo "📥 Loading existing server data..."
+    docker compose exec -T db psql -U mapuser -d mapdb < infrastructure/docker/postgis/existing_syria_data.sql
+fi
+
+# 4. Import New CSV Data
+echo "📥 Importing new CSV landmark data..."
+docker cp infrastructure/docker/postgis/syria_final_complete.csv map-db:/tmp/syria_data.csv
+
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+DROP TABLE IF EXISTS staging_syria;
+CREATE TABLE staging_syria (
+    name TEXT,
+    latitude DECIMAL(10, 8),
+    longitude DECIMAL(11, 8),
+    category TEXT,
+    address TEXT,
+    description TEXT,
+    created_at TIMESTAMP
+);
+
+-- HEADER true fixes the 'invalid input syntax' error
+COPY staging_syria(name, latitude, longitude, category, address, description, created_at) 
+FROM '/tmp/syria_data.csv' 
+WITH (FORMAT csv, HEADER true, QUOTE '\"', ENCODING 'UTF8');
+
+-- Merge into places_syria while avoiding duplicates
+-- We check for name + spatial proximity (approx 50m)
+INSERT INTO places_syria (name, name_ar, latitude, longitude, category, address, description, source, created_at)
+SELECT 
+    name, 
+    name, 
+    latitude, 
+    longitude, 
+    category, 
+    address, 
+    description, 
+    'csv_import_2026_04',
+    COALESCE(created_at, NOW())
+FROM staging_syria s
+WHERE NOT EXISTS (
+    SELECT 1 FROM places_syria p 
+    WHERE (p.name = s.name OR p.name_ar = s.name)
+    AND ST_DWithin(
+        ST_SetSRID(ST_MakePoint(CAST(p.longitude AS FLOAT), CAST(p.latitude AS FLOAT)), 4326)::geography, 
+        ST_SetSRID(ST_MakePoint(CAST(s.longitude AS FLOAT), CAST(s.latitude AS FLOAT)), 4326)::geography, 
+        50
+    )
+);
+
+DROP TABLE staging_syria;
+"
+
+echo "✅ Local database merge complete."
+echo "Run ./infrastructure/scripts/local_verify.sh to review the results."
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Local Verification Script for Syria Landmarks
+# To be run on the Mac terminal
+
+set -e
+
+echo "📊 --- Local Syria Data Audit ---"
+
+# 1. Total Count
+TOTAL=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "SELECT count(*) FROM places_syria;")
+echo "📍 Total Landmarks: $TOTAL"
+
+# 2. Category Distribution
+echo "🗂️  Category Distribution:"
+docker compose exec -T db psql -U mapuser -d mapdb -c "
+SELECT category, count(*) as count 
+FROM places_syria 
+GROUP BY category 
+ORDER BY count DESC 
+LIMIT 10;
+"
+
+# 3. Spatial Bounds Check (Damascus region)
+echo "🌍 Spatial Check (Damascus):"
+DM_COUNT=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "
+SELECT count(*) FROM places_syria 
+WHERE latitude BETWEEN 33.4 AND 33.6 AND longitude BETWEEN 36.2 AND 36.4;
+")
+echo "🏙️  Landmarks in Damascus area: $DM_COUNT"
+
+# 4. Generate Export if user is satisfied
+echo "💾 Generating SQL Export..."
+docker compose exec -T db pg_dump -U mapuser -d mapdb -t places_syria --data-only --inserts > infrastructure/docker/postgis/syria_export.sql
+
+echo "--------------------------------"
+echo "✅ Verification complete. Export saved to: infrastructure/docker/postgis/syria_export.sql"
+echo "If you are happy with the results, run sync_to_server.sh to push the data."
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Server Restoration Script for Syria Landmarks
+# To be run on the server terminal
+
+set -e
+
+APP_DIR="/home/hamzadoctor/app"
+EXPORT_FILE="infrastructure/docker/postgis/syria_export.sql"
+
+echo "🔄 Restoring Syria landmarks to production database..."
+
+# 1. Clean existing manual imports to avoid PK conflicts
+echo "🧹 Clearing existing manual entries..."
+docker exec -i map-db psql -U mapuser -d mapdb -c "DELETE FROM places_syria WHERE source = 'manual_import_2026_04';"
+
+# 2. Inject the SQL dump
+echo "📥 Injecting SQL dump..."
+docker exec -i map-db psql -U mapuser -d mapdb < "$APP_DIR/$EXPORT_FILE"
+
+# 3. Refresh API cache
+echo "🧹 Flushing Redis..."
+docker exec -i map-redis redis-cli flushall
+
+echo "✅ Restoration complete! Syria landmarks are live."