56 lines
1.8 KiB
Bash
56 lines
1.8 KiB
Bash
#!/bin/bash
|
|
set -e
|
|
|
|
# Configuration
|
|
CONTAINER_NAME="map-db"
|
|
DB_USER="postgres"
|
|
DB_NAME="map_saas"
|
|
CSV_FILE="infrastructure/docker/postgis/data_clean_syria.csv"
|
|
SQL_SCHEMA="infrastructure/docker/postgis/create_syria_table.sql"
|
|
|
|
# 1. Ensure the table exists
|
|
echo "Setting up Syria table..."
|
|
docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME < $SQL_SCHEMA
|
|
|
|
# 2. Upload CSV to container for fast bulk loading
|
|
echo "Copying CSV to database container..."
|
|
docker cp $CSV_FILE $CONTAINER_NAME:/tmp/data_clean_syria.csv
|
|
|
|
# 3. Import data using a temporary table to map columns correctly
|
|
echo "Importing data into PostGIS..."
|
|
docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME <<EOF
|
|
-- Create temporary table for CSV import
|
|
CREATE TEMP TABLE temp_syria_import (
|
|
search_query TEXT,
|
|
name TEXT,
|
|
rating TEXT,
|
|
reviews TEXT,
|
|
latitude DECIMAL,
|
|
longitude DECIMAL,
|
|
maps_url TEXT,
|
|
scraped_at TEXT
|
|
);
|
|
|
|
-- Copy from CSV (header row exists)
|
|
COPY temp_syria_import FROM '/tmp/data_clean_syria.csv' WITH (FORMAT csv, HEADER true, ENCODING 'UTF8');
|
|
|
|
-- Insert into main table
|
|
-- Mapping search_query -> neighbourhood
|
|
INSERT INTO places_syria (name, neighbourhood, latitude, longitude, created_at)
|
|
SELECT name, search_query, latitude, longitude,
|
|
CASE WHEN scraped_at IS NOT NULL AND scraped_at <> '' THEN scraped_at::timestamp ELSE CURRENT_TIMESTAMP END
|
|
FROM temp_syria_import;
|
|
|
|
-- Update PostGIS geometry column
|
|
UPDATE places_syria
|
|
SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)
|
|
WHERE location IS NULL AND latitude IS NOT NULL AND longitude IS NOT NULL;
|
|
|
|
-- Clean up
|
|
DROP TABLE temp_syria_import;
|
|
EOF
|
|
|
|
echo "✅ Import completed successfully!"
|
|
echo "Data sample from places_syria:"
|
|
docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME -c "SELECT name, neighbourhood, latitude, longitude FROM places_syria LIMIT 5;"
|