feat: sync source-backed OSM facility entrances
This commit is contained in:
Hamza-Ayed
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
# وظيفة الملف: أداة تشغيل أو تجهيز باسم sync regional place names؛ تفاصيل خطواتها ومدخلاتها موضحة في دليل ملفات المشروع.
|
||||
# Refresh Overture places and a country-tagged OSM/Overture name layer.
|
||||
# Usage: bash infrastructure/scripts/sync_regional_place_names.sh [JO SY IQ EG SA AE OM QA KW BH YE ...]
|
||||
set -euo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/home/hamzadoctor/app}"
|
||||
DATA_DIR="${APP_DIR}/infrastructure/osm-data"
|
||||
cd "$APP_DIR"
|
||||
|
||||
declare -A OSM_RELATIONS=(
|
||||
[EG]=-1473947 [SA]=-307584 [AE]=-307763 [OM]=-305138
|
||||
[QA]=-305095 [KW]=-305099 [BH]=-378734 [YE]=-305092
|
||||
[JO]=-184818 [SY]=-184840 [IQ]=-304934
|
||||
)
|
||||
TARGETS=("$@")
|
||||
if (( $# == 0 )); then TARGETS=(JO SY IQ EG SA AE OM QA KW BH YE); fi
|
||||
|
||||
for code in "${TARGETS[@]}"; do
|
||||
[[ -n "${OSM_RELATIONS[$code]:-}" ]] || { echo "Unknown country: $code" >&2; exit 2; }
|
||||
done
|
||||
command -v ogr2ogr >/dev/null || { echo "ogr2ogr is required" >&2; exit 2; }
|
||||
test -x venv_overture/bin/overturemaps || { echo "Overture CLI is required" >&2; exit 2; }
|
||||
|
||||
psql_db() { docker compose exec -T db psql -v ON_ERROR_STOP=1 -U mapuser -d mapdb "$@"; }
|
||||
|
||||
psql_db <<'SQL'
|
||||
CREATE TABLE IF NOT EXISTS regional_place_names (
|
||||
country_code text NOT NULL,
|
||||
source text NOT NULL,
|
||||
source_id text NOT NULL,
|
||||
name text NOT NULL,
|
||||
location geometry(Point,4326) NOT NULL,
|
||||
PRIMARY KEY (country_code, source, source_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS regional_place_names_location_idx
|
||||
ON regional_place_names USING gist (location);
|
||||
CREATE INDEX IF NOT EXISTS regional_place_names_country_idx
|
||||
ON regional_place_names (country_code);
|
||||
CREATE INDEX IF NOT EXISTS overture_place_id_idx ON overture_place (id);
|
||||
SQL
|
||||
|
||||
for code in "${TARGETS[@]}"; do
|
||||
relation="${OSM_RELATIONS[$code]}"
|
||||
echo "[$code] determining country bounds"
|
||||
bbox="$(psql_db -Atc "SELECT concat_ws(',',ST_XMin(e),ST_YMin(e),ST_XMax(e),ST_YMax(e)) FROM (SELECT ST_Extent(ST_Transform(way,4326)) AS e FROM planet_osm_polygon WHERE osm_id=$relation AND boundary='administrative' AND admin_level='2') q")"
|
||||
[[ -n "$bbox" ]] || { echo "Missing country boundary: $code" >&2; exit 1; }
|
||||
output="$DATA_DIR/overture_places_${code}.geojson"
|
||||
echo "[$code] downloading Overture places for $bbox"
|
||||
if [[ ! -s "$output" ]]; then
|
||||
venv_overture/bin/overturemaps download --bbox="$bbox" -f geojson --type=place -o "$output"
|
||||
fi
|
||||
|
||||
echo "[$code] loading staging table"
|
||||
db_password="$(docker compose exec -T db printenv POSTGRES_PASSWORD)"
|
||||
PGPASSWORD="$db_password" ogr2ogr -f PostgreSQL \
|
||||
"PG:host=127.0.0.1 port=5432 user=mapuser dbname=mapdb" \
|
||||
"$output" -nln overture_place_stage -overwrite -nlt PROMOTE_TO_MULTI \
|
||||
-lco GEOMETRY_NAME=location
|
||||
|
||||
echo "[$code] merging named Overture places and refreshing map names"
|
||||
psql_db -v code="$code" -v relation="$relation" <<'SQL'
|
||||
BEGIN;
|
||||
WITH border AS (
|
||||
SELECT ST_Union(ST_Transform(way,4326)) geom FROM planet_osm_polygon
|
||||
WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
|
||||
)
|
||||
INSERT INTO overture_place
|
||||
(id,categories,confidence,websites,emails,socials,phones,brand,addresses,names,sources,
|
||||
operating_status,basic_category,taxonomy,version,location,name_primary)
|
||||
SELECT s.id,NULL,s.confidence,s.websites,s.emails,s.socials,s.phones,s.brand,s.addresses,
|
||||
s.names,s.sources,s.operating_status,s.basic_category,s.taxonomy,s.version,s.location,
|
||||
COALESCE(s.names->>'primary',s.names->>'common')
|
||||
FROM overture_place_stage s CROSS JOIN border b
|
||||
WHERE s.id IS NOT NULL AND s.location && b.geom AND ST_Intersects(s.location,b.geom)
|
||||
AND COALESCE(s.names->>'primary',s.names->>'common') IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM overture_place p WHERE p.id=s.id);
|
||||
|
||||
DELETE FROM regional_place_names WHERE country_code=:'code';
|
||||
WITH border AS (
|
||||
SELECT ST_Union(way) geom FROM planet_osm_polygon
|
||||
WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
|
||||
)
|
||||
INSERT INTO regional_place_names(country_code,source,source_id,name,location)
|
||||
SELECT :'code','osm_point',p.osm_id::text,p.name,ST_Transform(p.way,4326)
|
||||
FROM planet_osm_point p CROSS JOIN border b
|
||||
WHERE p.name IS NOT NULL AND p.name<>'' AND p.way && b.geom AND ST_Intersects(p.way,b.geom)
|
||||
ON CONFLICT DO NOTHING;
|
||||
WITH border AS (
|
||||
SELECT ST_Union(way) geom FROM planet_osm_polygon
|
||||
WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
|
||||
)
|
||||
INSERT INTO regional_place_names(country_code,source,source_id,name,location)
|
||||
SELECT :'code','osm_polygon',p.osm_id::text,p.name,ST_Transform(ST_PointOnSurface(p.way),4326)
|
||||
FROM planet_osm_polygon p CROSS JOIN border b
|
||||
WHERE p.name IS NOT NULL AND p.name<>'' AND p.way && b.geom
|
||||
AND ST_Intersects(ST_PointOnSurface(p.way),b.geom)
|
||||
ON CONFLICT DO NOTHING;
|
||||
WITH border AS (
|
||||
SELECT ST_Union(ST_Transform(way,4326)) geom FROM planet_osm_polygon
|
||||
WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
|
||||
)
|
||||
INSERT INTO regional_place_names(country_code,source,source_id,name,location)
|
||||
SELECT :'code','overture',p.id,COALESCE(p.name_primary,p.names->>'primary',p.names->>'common'),
|
||||
ST_PointOnSurface(p.location)
|
||||
FROM overture_place p CROSS JOIN border b
|
||||
WHERE p.id IS NOT NULL AND p.location && b.geom AND ST_Intersects(p.location,b.geom)
|
||||
AND COALESCE(p.name_primary,p.names->>'primary',p.names->>'common') IS NOT NULL
|
||||
ON CONFLICT DO NOTHING;
|
||||
COMMIT;
|
||||
DROP TABLE overture_place_stage;
|
||||
SQL
|
||||
if [[ "$code" == IQ ]]; then
|
||||
psql_db <<'SQL'
|
||||
INSERT INTO regional_place_names(country_code,source,source_id,name,location)
|
||||
SELECT 'IQ','custom_iraq',id::text,COALESCE(NULLIF(name_ar,''),NULLIF(name,'')),location
|
||||
FROM places_iraq
|
||||
WHERE source='iraq_final_complete' AND location IS NOT NULL
|
||||
AND COALESCE(NULLIF(name_ar,''),NULLIF(name,'')) IS NOT NULL
|
||||
ON CONFLICT DO NOTHING;
|
||||
SQL
|
||||
fi
|
||||
rm -f "$output"
|
||||
psql_db -Atc "SELECT '$code'||'|'||source||'|'||count(*) FROM regional_place_names WHERE country_code='$code' GROUP BY source ORDER BY source"
|
||||
done
|
||||
|
||||
docker compose exec -T redis redis-cli FLUSHDB >/dev/null
|
||||
Reference in New Issue
Block a user