feat: add syria data import scripts and update geocoding search to include neighbourhood
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
CONTAINER_NAME="map-db"
|
||||
DB_USER="postgres"
|
||||
DB_NAME="map_saas"
|
||||
CSV_FILE="infrastructure/docker/postgis/data_clean_syria.csv"
|
||||
SQL_SCHEMA="infrastructure/docker/postgis/create_syria_table.sql"
|
||||
|
||||
# 1. Ensure the table exists
|
||||
echo "Setting up Syria table..."
|
||||
docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME < $SQL_SCHEMA
|
||||
|
||||
# 2. Upload CSV to container for fast bulk loading
|
||||
echo "Copying CSV to database container..."
|
||||
docker cp $CSV_FILE $CONTAINER_NAME:/tmp/data_clean_syria.csv
|
||||
|
||||
# 3. Import data using a temporary table to map columns correctly
|
||||
echo "Importing data into PostGIS..."
|
||||
docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME <<EOF
|
||||
-- Create temporary table for CSV import
|
||||
CREATE TEMP TABLE temp_syria_import (
|
||||
search_query TEXT,
|
||||
name TEXT,
|
||||
rating TEXT,
|
||||
reviews TEXT,
|
||||
latitude DECIMAL,
|
||||
longitude DECIMAL,
|
||||
maps_url TEXT,
|
||||
scraped_at TEXT
|
||||
);
|
||||
|
||||
-- Copy from CSV (header row exists)
|
||||
COPY temp_syria_import FROM '/tmp/data_clean_syria.csv' WITH (FORMAT csv, HEADER true, ENCODING 'UTF8');
|
||||
|
||||
-- Insert into main table
|
||||
-- Mapping search_query -> neighbourhood
|
||||
INSERT INTO places_syria (name, neighbourhood, latitude, longitude, created_at)
|
||||
SELECT name, search_query, latitude, longitude,
|
||||
CASE WHEN scraped_at IS NOT NULL AND scraped_at <> '' THEN scraped_at::timestamp ELSE CURRENT_TIMESTAMP END
|
||||
FROM temp_syria_import;
|
||||
|
||||
-- Update PostGIS geometry column
|
||||
UPDATE places_syria
|
||||
SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)
|
||||
WHERE location IS NULL AND latitude IS NOT NULL AND longitude IS NOT NULL;
|
||||
|
||||
-- Clean up
|
||||
DROP TABLE temp_syria_import;
|
||||
EOF
|
||||
|
||||
echo "✅ Import completed successfully!"
|
||||
echo "Data sample from places_syria:"
|
||||
docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME -c "SELECT name, neighbourhood, latitude, longitude FROM places_syria LIMIT 5;"
|
||||
Reference in New Issue
Block a user