128 lines
5.9 KiB
Bash
128 lines
5.9 KiB
Bash
#!/usr/bin/env bash
|
|
# وظيفة الملف: أداة تشغيل أو تجهيز باسم sync regional place names؛ تفاصيل خطواتها ومدخلاتها موضحة في دليل ملفات المشروع.
|
|
# Refresh Overture places and a country-tagged OSM/Overture name layer.
|
|
# Usage: bash infrastructure/scripts/sync_regional_place_names.sh [JO SY IQ EG SA AE OM QA KW BH YE ...]
|
|
set -euo pipefail
|
|
|
|
APP_DIR="${APP_DIR:-/home/hamzadoctor/app}"
|
|
DATA_DIR="${APP_DIR}/infrastructure/osm-data"
|
|
cd "$APP_DIR"
|
|
|
|
declare -A OSM_RELATIONS=(
|
|
[EG]=-1473947 [SA]=-307584 [AE]=-307763 [OM]=-305138
|
|
[QA]=-305095 [KW]=-305099 [BH]=-378734 [YE]=-305092
|
|
[JO]=-184818 [SY]=-184840 [IQ]=-304934
|
|
)
|
|
TARGETS=("$@")
|
|
if (( $# == 0 )); then TARGETS=(JO SY IQ EG SA AE OM QA KW BH YE); fi
|
|
|
|
for code in "${TARGETS[@]}"; do
|
|
[[ -n "${OSM_RELATIONS[$code]:-}" ]] || { echo "Unknown country: $code" >&2; exit 2; }
|
|
done
|
|
command -v ogr2ogr >/dev/null || { echo "ogr2ogr is required" >&2; exit 2; }
|
|
test -x venv_overture/bin/overturemaps || { echo "Overture CLI is required" >&2; exit 2; }
|
|
|
|
psql_db() { docker compose exec -T db psql -v ON_ERROR_STOP=1 -U mapuser -d mapdb "$@"; }
|
|
|
|
psql_db <<'SQL'
|
|
CREATE TABLE IF NOT EXISTS regional_place_names (
|
|
country_code text NOT NULL,
|
|
source text NOT NULL,
|
|
source_id text NOT NULL,
|
|
name text NOT NULL,
|
|
location geometry(Point,4326) NOT NULL,
|
|
PRIMARY KEY (country_code, source, source_id)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS regional_place_names_location_idx
|
|
ON regional_place_names USING gist (location);
|
|
CREATE INDEX IF NOT EXISTS regional_place_names_country_idx
|
|
ON regional_place_names (country_code);
|
|
CREATE INDEX IF NOT EXISTS overture_place_id_idx ON overture_place (id);
|
|
SQL
|
|
|
|
for code in "${TARGETS[@]}"; do
|
|
relation="${OSM_RELATIONS[$code]}"
|
|
echo "[$code] determining country bounds"
|
|
bbox="$(psql_db -Atc "SELECT concat_ws(',',ST_XMin(e),ST_YMin(e),ST_XMax(e),ST_YMax(e)) FROM (SELECT ST_Extent(ST_Transform(way,4326)) AS e FROM planet_osm_polygon WHERE osm_id=$relation AND boundary='administrative' AND admin_level='2') q")"
|
|
[[ -n "$bbox" ]] || { echo "Missing country boundary: $code" >&2; exit 1; }
|
|
output="$DATA_DIR/overture_places_${code}.geojson"
|
|
echo "[$code] downloading Overture places for $bbox"
|
|
if [[ ! -s "$output" ]]; then
|
|
venv_overture/bin/overturemaps download --bbox="$bbox" -f geojson --type=place -o "$output"
|
|
fi
|
|
|
|
echo "[$code] loading staging table"
|
|
db_password="$(docker compose exec -T db printenv POSTGRES_PASSWORD)"
|
|
PGPASSWORD="$db_password" ogr2ogr -f PostgreSQL \
|
|
"PG:host=127.0.0.1 port=5432 user=mapuser dbname=mapdb" \
|
|
"$output" -nln overture_place_stage -overwrite -nlt PROMOTE_TO_MULTI \
|
|
-lco GEOMETRY_NAME=location
|
|
|
|
echo "[$code] merging named Overture places and refreshing map names"
|
|
psql_db -v code="$code" -v relation="$relation" <<'SQL'
|
|
BEGIN;
|
|
WITH border AS (
|
|
SELECT ST_Union(ST_Transform(way,4326)) geom FROM planet_osm_polygon
|
|
WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
|
|
)
|
|
INSERT INTO overture_place
|
|
(id,categories,confidence,websites,emails,socials,phones,brand,addresses,names,sources,
|
|
operating_status,basic_category,taxonomy,version,location,name_primary)
|
|
SELECT s.id,NULL,s.confidence,s.websites,s.emails,s.socials,s.phones,s.brand,s.addresses,
|
|
s.names,s.sources,s.operating_status,s.basic_category,s.taxonomy,s.version,s.location,
|
|
COALESCE(s.names->>'primary',s.names->>'common')
|
|
FROM overture_place_stage s CROSS JOIN border b
|
|
WHERE s.id IS NOT NULL AND s.location && b.geom AND ST_Intersects(s.location,b.geom)
|
|
AND COALESCE(s.names->>'primary',s.names->>'common') IS NOT NULL
|
|
AND NOT EXISTS (SELECT 1 FROM overture_place p WHERE p.id=s.id);
|
|
|
|
DELETE FROM regional_place_names WHERE country_code=:'code';
|
|
WITH border AS (
|
|
SELECT ST_Union(way) geom FROM planet_osm_polygon
|
|
WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
|
|
)
|
|
INSERT INTO regional_place_names(country_code,source,source_id,name,location)
|
|
SELECT :'code','osm_point',p.osm_id::text,p.name,ST_Transform(p.way,4326)
|
|
FROM planet_osm_point p CROSS JOIN border b
|
|
WHERE p.name IS NOT NULL AND p.name<>'' AND p.way && b.geom AND ST_Intersects(p.way,b.geom)
|
|
ON CONFLICT DO NOTHING;
|
|
WITH border AS (
|
|
SELECT ST_Union(way) geom FROM planet_osm_polygon
|
|
WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
|
|
)
|
|
INSERT INTO regional_place_names(country_code,source,source_id,name,location)
|
|
SELECT :'code','osm_polygon',p.osm_id::text,p.name,ST_Transform(ST_PointOnSurface(p.way),4326)
|
|
FROM planet_osm_polygon p CROSS JOIN border b
|
|
WHERE p.name IS NOT NULL AND p.name<>'' AND p.way && b.geom
|
|
AND ST_Intersects(ST_PointOnSurface(p.way),b.geom)
|
|
ON CONFLICT DO NOTHING;
|
|
WITH border AS (
|
|
SELECT ST_Union(ST_Transform(way,4326)) geom FROM planet_osm_polygon
|
|
WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
|
|
)
|
|
INSERT INTO regional_place_names(country_code,source,source_id,name,location)
|
|
SELECT :'code','overture',p.id,COALESCE(p.name_primary,p.names->>'primary',p.names->>'common'),
|
|
ST_PointOnSurface(p.location)
|
|
FROM overture_place p CROSS JOIN border b
|
|
WHERE p.id IS NOT NULL AND p.location && b.geom AND ST_Intersects(p.location,b.geom)
|
|
AND COALESCE(p.name_primary,p.names->>'primary',p.names->>'common') IS NOT NULL
|
|
ON CONFLICT DO NOTHING;
|
|
COMMIT;
|
|
DROP TABLE overture_place_stage;
|
|
SQL
|
|
if [[ "$code" == IQ ]]; then
|
|
psql_db <<'SQL'
|
|
INSERT INTO regional_place_names(country_code,source,source_id,name,location)
|
|
SELECT 'IQ','custom_iraq',id::text,COALESCE(NULLIF(name_ar,''),NULLIF(name,'')),location
|
|
FROM places_iraq
|
|
WHERE source='iraq_final_complete' AND location IS NOT NULL
|
|
AND COALESCE(NULLIF(name_ar,''),NULLIF(name,'')) IS NOT NULL
|
|
ON CONFLICT DO NOTHING;
|
|
SQL
|
|
fi
|
|
rm -f "$output"
|
|
psql_db -Atc "SELECT '$code'||'|'||source||'|'||count(*) FROM regional_place_names WHERE country_code='$code' GROUP BY source ORDER BY source"
|
|
done
|
|
|
|
docker compose exec -T redis redis-cli FLUSHDB >/dev/null
|