feat: sync source-backed OSM facility entrances

infrastructure/scripts/sync_regional_place_names.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+# وظيفة الملف: أداة تشغيل أو تجهيز باسم sync regional place names؛ تفاصيل خطواتها ومدخلاتها موضحة في دليل ملفات المشروع.
+# Refresh Overture places and a country-tagged OSM/Overture name layer.
+# Usage: bash infrastructure/scripts/sync_regional_place_names.sh [JO SY IQ EG SA AE OM QA KW BH YE ...]
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/home/hamzadoctor/app}"
+DATA_DIR="${APP_DIR}/infrastructure/osm-data"
+cd "$APP_DIR"
+
+declare -A OSM_RELATIONS=(
+  [EG]=-1473947 [SA]=-307584 [AE]=-307763 [OM]=-305138
+  [QA]=-305095 [KW]=-305099 [BH]=-378734 [YE]=-305092
+  [JO]=-184818 [SY]=-184840 [IQ]=-304934
+)
+TARGETS=("$@")
+if (( $# == 0 )); then TARGETS=(JO SY IQ EG SA AE OM QA KW BH YE); fi
+
+for code in "${TARGETS[@]}"; do
+  [[ -n "${OSM_RELATIONS[$code]:-}" ]] || { echo "Unknown country: $code" >&2; exit 2; }
+done
+command -v ogr2ogr >/dev/null || { echo "ogr2ogr is required" >&2; exit 2; }
+test -x venv_overture/bin/overturemaps || { echo "Overture CLI is required" >&2; exit 2; }
+
+psql_db() { docker compose exec -T db psql -v ON_ERROR_STOP=1 -U mapuser -d mapdb "$@"; }
+
+psql_db <<'SQL'
+CREATE TABLE IF NOT EXISTS regional_place_names (
+  country_code text NOT NULL,
+  source text NOT NULL,
+  source_id text NOT NULL,
+  name text NOT NULL,
+  location geometry(Point,4326) NOT NULL,
+  PRIMARY KEY (country_code, source, source_id)
+);
+CREATE INDEX IF NOT EXISTS regional_place_names_location_idx
+  ON regional_place_names USING gist (location);
+CREATE INDEX IF NOT EXISTS regional_place_names_country_idx
+  ON regional_place_names (country_code);
+CREATE INDEX IF NOT EXISTS overture_place_id_idx ON overture_place (id);
+SQL
+
+for code in "${TARGETS[@]}"; do
+  relation="${OSM_RELATIONS[$code]}"
+  echo "[$code] determining country bounds"
+  bbox="$(psql_db -Atc "SELECT concat_ws(',',ST_XMin(e),ST_YMin(e),ST_XMax(e),ST_YMax(e)) FROM (SELECT ST_Extent(ST_Transform(way,4326)) AS e FROM planet_osm_polygon WHERE osm_id=$relation AND boundary='administrative' AND admin_level='2') q")"
+  [[ -n "$bbox" ]] || { echo "Missing country boundary: $code" >&2; exit 1; }
+  output="$DATA_DIR/overture_places_${code}.geojson"
+  echo "[$code] downloading Overture places for $bbox"
+  if [[ ! -s "$output" ]]; then
+    venv_overture/bin/overturemaps download --bbox="$bbox" -f geojson --type=place -o "$output"
+  fi
+
+  echo "[$code] loading staging table"
+  db_password="$(docker compose exec -T db printenv POSTGRES_PASSWORD)"
+  PGPASSWORD="$db_password" ogr2ogr -f PostgreSQL \
+    "PG:host=127.0.0.1 port=5432 user=mapuser dbname=mapdb" \
+    "$output" -nln overture_place_stage -overwrite -nlt PROMOTE_TO_MULTI \
+    -lco GEOMETRY_NAME=location
+
+  echo "[$code] merging named Overture places and refreshing map names"
+  psql_db -v code="$code" -v relation="$relation" <<'SQL'
+BEGIN;
+WITH border AS (
+  SELECT ST_Union(ST_Transform(way,4326)) geom FROM planet_osm_polygon
+  WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
+)
+INSERT INTO overture_place
+ (id,categories,confidence,websites,emails,socials,phones,brand,addresses,names,sources,
+  operating_status,basic_category,taxonomy,version,location,name_primary)
+SELECT s.id,NULL,s.confidence,s.websites,s.emails,s.socials,s.phones,s.brand,s.addresses,
+       s.names,s.sources,s.operating_status,s.basic_category,s.taxonomy,s.version,s.location,
+       COALESCE(s.names->>'primary',s.names->>'common')
+FROM overture_place_stage s CROSS JOIN border b
+WHERE s.id IS NOT NULL AND s.location && b.geom AND ST_Intersects(s.location,b.geom)
+  AND COALESCE(s.names->>'primary',s.names->>'common') IS NOT NULL
+  AND NOT EXISTS (SELECT 1 FROM overture_place p WHERE p.id=s.id);
+
+DELETE FROM regional_place_names WHERE country_code=:'code';
+WITH border AS (
+  SELECT ST_Union(way) geom FROM planet_osm_polygon
+  WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
+)
+INSERT INTO regional_place_names(country_code,source,source_id,name,location)
+SELECT :'code','osm_point',p.osm_id::text,p.name,ST_Transform(p.way,4326)
+FROM planet_osm_point p CROSS JOIN border b
+WHERE p.name IS NOT NULL AND p.name<>'' AND p.way && b.geom AND ST_Intersects(p.way,b.geom)
+ON CONFLICT DO NOTHING;
+WITH border AS (
+  SELECT ST_Union(way) geom FROM planet_osm_polygon
+  WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
+)
+INSERT INTO regional_place_names(country_code,source,source_id,name,location)
+SELECT :'code','osm_polygon',p.osm_id::text,p.name,ST_Transform(ST_PointOnSurface(p.way),4326)
+FROM planet_osm_polygon p CROSS JOIN border b
+WHERE p.name IS NOT NULL AND p.name<>'' AND p.way && b.geom
+  AND ST_Intersects(ST_PointOnSurface(p.way),b.geom)
+ON CONFLICT DO NOTHING;
+WITH border AS (
+  SELECT ST_Union(ST_Transform(way,4326)) geom FROM planet_osm_polygon
+  WHERE osm_id = :'relation'::bigint AND boundary='administrative' AND admin_level='2'
+)
+INSERT INTO regional_place_names(country_code,source,source_id,name,location)
+SELECT :'code','overture',p.id,COALESCE(p.name_primary,p.names->>'primary',p.names->>'common'),
+       ST_PointOnSurface(p.location)
+FROM overture_place p CROSS JOIN border b
+WHERE p.id IS NOT NULL AND p.location && b.geom AND ST_Intersects(p.location,b.geom)
+  AND COALESCE(p.name_primary,p.names->>'primary',p.names->>'common') IS NOT NULL
+ON CONFLICT DO NOTHING;
+COMMIT;
+DROP TABLE overture_place_stage;
+SQL
+  if [[ "$code" == IQ ]]; then
+    psql_db <<'SQL'
+INSERT INTO regional_place_names(country_code,source,source_id,name,location)
+SELECT 'IQ','custom_iraq',id::text,COALESCE(NULLIF(name_ar,''),NULLIF(name,'')),location
+FROM places_iraq
+WHERE source='iraq_final_complete' AND location IS NOT NULL
+  AND COALESCE(NULLIF(name_ar,''),NULLIF(name,'')) IS NOT NULL
+ON CONFLICT DO NOTHING;
+SQL
+  fi
+  rm -f "$output"
+  psql_db -Atc "SELECT '$code'||'|'||source||'|'||count(*) FROM regional_place_names WHERE country_code='$code' GROUP BY source ORDER BY source"
+done
+
+docker compose exec -T redis redis-cli FLUSHDB >/dev/null

infrastructure/scripts/update-data.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# وظيفة الملف: أداة تشغيل أو تجهيز باسم update data؛ تفاصيل خطواتها ومدخلاتها موضحة في دليل ملفات المشروع.
 # --------------------------------------------------------------------------
 # Intaleq Map Platform - 10-Day Update Script
 # سكربت تحديث خرائط انطلاقة - التحديث الدوري (كل 10 أيام)
@@ -11,62 +12,83 @@
 
 set -e # Exit on error
 
-echo "🚀 Starting 10-day map update (v3 — Jordan + Syria + Egypt + Iraq + Delta)..."
-
 APP_DIR="/home/hamzadoctor/app"
 DATA_DIR="${APP_DIR}/infrastructure/osm-data"
+DATA_SCOPE_FILE="${DATA_DIR}/map-data-scope"
+# An explicitly supplied scope takes priority. After a successful regional
+# expansion, persist it so the scheduled 10-day job cannot drop those regions.
+MAP_DATA_SCOPE="${MAP_DATA_SCOPE:-$(cat "${DATA_SCOPE_FILE}" 2>/dev/null || printf 'core')}"
+
+case "${MAP_DATA_SCOPE}" in
+    core|gcc|gcc-yemen) ;;
+    *)
+        echo "❌ MAP_DATA_SCOPE must be one of: core, gcc, gcc-yemen (got: ${MAP_DATA_SCOPE})"
+        exit 2
+        ;;
+esac
+
+echo "🚀 Starting map update (scope: ${MAP_DATA_SCOPE})..."
+
 # ✅ FIX: Correct filename that GraphHopper actually reads (was: region.osm.pbf)
 MASTER_FILE="${DATA_DIR}/master_map.osm.pbf"
 DELTA_FILE="${DATA_DIR}/delta.osm"
 
 cd "${APP_DIR}"
 
-# ── Step 1: Download all four country PBFs ─────────────────────────────────
-echo "🌍 Downloading Jordan, Syria, Egypt & Iraq PBF data from Geofabrik..."
+if ! command -v osmium >/dev/null 2>&1; then
+    echo "❌ osmium-tool is required to merge routing extracts. Install it with: sudo apt-get install -y osmium-tool"
+    exit 3
+fi
 
-wget -q --show-progress -O "${DATA_DIR}/jordan-latest.osm.pbf.new" \
+# ── Step 1: Select and download the country PBFs ───────────────────────────
+# GCC is Geofabrik's combined Saudi Arabia, UAE, Qatar, Oman, Kuwait and Bahrain
+# extract. Keeping the stage explicit prevents a scheduled update from silently
+# enlarging the production routing graph.
+COUNTRIES=(jordan syria egypt iraq)
+PBF_FILES=(
+    "jordan-latest.osm.pbf"
+    "syria-latest.osm.pbf"
+    "egypt-latest.osm.pbf"
+    "iraq-latest.osm.pbf"
+)
+PBF_URLS=(
     "https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
-
-# ✅ FIX: كان يحمّل مصر إلى ملف سوريا ثم يعيد تحميل سوريا فوقها — هدر 177MB كل دورة
-wget -q --show-progress -O "${DATA_DIR}/syria-latest.osm.pbf.new" \
     "https://download.geofabrik.de/asia/syria-latest.osm.pbf"
-wget -q --show-progress -O "${DATA_DIR}/egypt-latest.osm.pbf.new" \
     "https://download.geofabrik.de/africa/egypt-latest.osm.pbf"
-wget -q --show-progress -O "${DATA_DIR}/iraq-latest.osm.pbf.new" \
     "https://download.geofabrik.de/asia/iraq-latest.osm.pbf"
+)
 
-mv "${DATA_DIR}/jordan-latest.osm.pbf.new" "${DATA_DIR}/jordan-latest.osm.pbf"
-mv "${DATA_DIR}/syria-latest.osm.pbf.new"  "${DATA_DIR}/syria-latest.osm.pbf"
-mv "${DATA_DIR}/egypt-latest.osm.pbf.new"  "${DATA_DIR}/egypt-latest.osm.pbf"
-mv "${DATA_DIR}/iraq-latest.osm.pbf.new"   "${DATA_DIR}/iraq-latest.osm.pbf"
+if [[ "${MAP_DATA_SCOPE}" == "gcc" || "${MAP_DATA_SCOPE}" == "gcc-yemen" ]]; then
+    COUNTRIES+=(gcc-states)
+    PBF_FILES+=(gcc-states-latest.osm.pbf)
+    PBF_URLS+=("https://download.geofabrik.de/asia/gcc-states-latest.osm.pbf")
+fi
+
+if [[ "${MAP_DATA_SCOPE}" == "gcc-yemen" ]]; then
+    COUNTRIES+=(yemen)
+    PBF_FILES+=(yemen-latest.osm.pbf)
+    PBF_URLS+=("https://download.geofabrik.de/asia/yemen-latest.osm.pbf")
+fi
+
+echo "🌍 Downloading ${COUNTRIES[*]} PBF data from Geofabrik..."
+for i in "${!PBF_FILES[@]}"; do
+    wget -q --show-progress -O "${DATA_DIR}/${PBF_FILES[$i]}.new" "${PBF_URLS[$i]}"
+    mv "${DATA_DIR}/${PBF_FILES[$i]}.new" "${DATA_DIR}/${PBF_FILES[$i]}"
+done
 
 echo "✅ Downloads complete."
 
-# ── Step 2: Import all four countries into PostGIS ─────────────────────────
-echo "💾 Importing Jordan into PostGIS (--create resets planet_osm_* tables)..."
-docker compose --profile import run --rm osm-import \
-    osm2pgsql --create --slim --cache 1000 \
-    --database mapdb --host db --user mapuser \
-    /data/jordan-latest.osm.pbf
-
-echo "💾 Importing Syria into PostGIS (--append)..."
-docker compose --profile import run --rm osm-import \
-    osm2pgsql --append --slim --cache 1000 \
-    --database mapdb --host db --user mapuser \
-    /data/syria-latest.osm.pbf
-
-# ✅ FIX: Egypt was missing from the update cycle — added here
-echo "💾 Importing Egypt into PostGIS (--append)..."
-docker compose --profile import run --rm osm-import \
-    osm2pgsql --append --slim --cache 1000 \
-    --database mapdb --host db --user mapuser \
-    /data/egypt-latest.osm.pbf
-
-echo "💾 Importing Iraq into PostGIS (--append)..."
-docker compose --profile import run --rm osm-import \
-    osm2pgsql --append --slim --cache 1000 \
-    --database mapdb --host db --user mapuser \
-    /data/iraq-latest.osm.pbf
+# ── Step 2: Import into PostGIS ────────────────────────────────────────────
+# The first extract replaces planet_osm_*; every remaining extract appends.
+for i in "${!PBF_FILES[@]}"; do
+    IMPORT_MODE="--append"
+    [[ "$i" == "0" ]] && IMPORT_MODE="--create"
+    echo "💾 Importing ${COUNTRIES[$i]} into PostGIS (${IMPORT_MODE})..."
+    docker compose --profile import run --rm osm-import \
+        osm2pgsql "${IMPORT_MODE}" --slim --cache 1000 \
+        --database mapdb --host db --user mapuser \
+        "/data/${PBF_FILES[$i]}"
+done
 
 # ── Step 2b: Keep routing topology tables ─────────────────────────────────
 # Approved roads must share real OSM node IDs with the base graph. Deleting
@@ -75,15 +97,19 @@ echo "🧭 Preserving OSM node/way topology for approved-road routing..."
 docker compose exec -T db psql -U mapuser -d mapdb -c \
     "DROP TABLE IF EXISTS planet_osm_rels;"
 
-# ── Step 3: Merge all four PBFs into master_map.osm.pbf ────────────────────
+# OSM 624165122 is a misclassified 5.94 km² water polygon covering central Zarqa.
+# Remove it after every full OSM import so it cannot reappear in the rendered map.
+docker compose exec -T db psql -U mapuser -d mapdb -c \
+    "DELETE FROM planet_osm_polygon WHERE osm_id = 624165122;"
+
+# ── Step 3: Merge all selected PBFs into master_map.osm.pbf ───────────────
 # ✅ FIX: Old script wrote to region.osm.pbf — GraphHopper reads master_map.osm.pbf
-echo "🗺️ Merging Jordan + Syria + Egypt + Iraq → ${MASTER_FILE}..."
-osmium merge \
-    "${DATA_DIR}/jordan-latest.osm.pbf" \
-    "${DATA_DIR}/syria-latest.osm.pbf" \
-    "${DATA_DIR}/egypt-latest.osm.pbf" \
-    "${DATA_DIR}/iraq-latest.osm.pbf" \
-    -o "${MASTER_FILE}" --overwrite
+echo "🗺️ Merging ${COUNTRIES[*]} → ${MASTER_FILE}..."
+MERGE_INPUTS=()
+for pbf_file in "${PBF_FILES[@]}"; do
+    MERGE_INPUTS+=("${DATA_DIR}/${pbf_file}")
+done
+osmium merge "${MERGE_INPUTS[@]}" -o "${MASTER_FILE}" --overwrite
 
 echo "✅ Master PBF built: $(du -sh ${MASTER_FILE} | cut -f1)"
 
@@ -117,7 +143,9 @@ docker compose stop api web dashboard martin || true
 trap 'echo "▶️  إعادة تشغيل الحاويات المؤقتة..."; docker compose up -d api web dashboard martin || true' EXIT
 
 docker compose stop routing
-rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
+# GraphHopper creates its graph files as root inside the container, so the
+# host user needs sudo to reliably remove a stale graph before rebuilding.
+sudo rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
 docker compose up -d routing
 echo "✅ GraphHopper restarting from ${MASTER_FILE}."
 
@@ -149,6 +177,23 @@ fi
 echo "🧹 Flushing Redis traffic cache..."
 docker compose exec -T redis redis-cli flushall
 
+# Keep named OSM and Overture places in step with the 10-day road refresh.
+# This includes Egypt, GCC countries and Yemen; the importer uses country
+# borders and preserves the existing curated place tables.
+bash "${APP_DIR}/infrastructure/scripts/sync_regional_place_names.sh"
+docker compose exec -T db psql -v ON_ERROR_STOP=1 -U mapuser -d mapdb \
+    < "${APP_DIR}/infrastructure/sql/10_regional_place_search.sql"
+
+# Import only explicit OSM entrance/gate nodes from the same 11-country PBF.
+# No synthetic offsets are created; a node is attached only when its coordinate
+# is covered by exactly one named target facility polygon.
+python3 -u "${APP_DIR}/scripts/sync_osm_place_gates.py" \
+    --pbf "${DATA_DIR}/master_map.osm.pbf"
+docker compose exec -T redis redis-cli flushdb
+
+# Persist only after every import, graph rebuild and verification has succeeded.
+printf '%s\n' "${MAP_DATA_SCOPE}" > "${DATA_SCOPE_FILE}"
+
 echo ""
-echo "✅ 10-day update complete — Jordan + Syria + Egypt + Iraq + approved delta applied."
+echo "✅ 10-day update complete — ${COUNTRIES[*]}, approved road delta, regional names, and OSM gates synchronized."
 echo "   GraphHopper is rebuilding in the background. Allow 5-10 min for routing to be ready."

infrastructure/sql/05_poi_gates_clustering.sql

@@ -0,0 +1,28 @@
+-- وظيفة الملف: استعلام أو تغيير قاعدة بيانات يخص 05 poi gates clustering؛ راجع ترتيب التنفيذ قبل تطبيقه.
+-- 1. Create the place_gates table
+CREATE TABLE IF NOT EXISTS place_gates (
+    id SERIAL PRIMARY KEY,
+    place_id VARCHAR(255) NOT NULL, -- references unified_search_index.id
+    gate_name_ar VARCHAR(255) NOT NULL,
+    gate_name_en VARCHAR(255),
+    latitude NUMERIC(10, 7) NOT NULL,
+    longitude NUMERIC(10, 7) NOT NULL,
+    is_main_gate BOOLEAN DEFAULT false,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX idx_place_gates_place_id ON place_gates(place_id);
+
+-- No gate seed rows belong here: service doors must come from an explicit,
+-- source-backed entrance node. See scripts/sync_osm_place_gates.py.
+
+-- Prepare schema for future K-Means Clustering (Sero Trip Drop-offs)
+CREATE TABLE IF NOT EXISTS sero_dropoffs (
+    id SERIAL PRIMARY KEY,
+    place_id VARCHAR(255) NOT NULL,
+    latitude NUMERIC(10, 7) NOT NULL,
+    longitude NUMERIC(10, 7) NOT NULL,
+    timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX idx_sero_dropoffs_place_id ON sero_dropoffs(place_id);

infrastructure/sql/10_regional_place_search.sql

@@ -0,0 +1,8 @@
+-- وظيفة الملف: استعلام أو تغيير قاعدة بيانات يخص 10 regional place search؛ راجع ترتيب التنفيذ قبل تطبيقه.
+-- Run outside a transaction so these indexes do not block map reads/writes.
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE INDEX CONCURRENTLY IF NOT EXISTS regional_place_names_normalized_trgm_idx
+  ON regional_place_names USING gin (normalize_arabic(name) gin_trgm_ops);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS regional_place_names_country_prefix_idx
+  ON regional_place_names (country_code, (normalize_arabic(name)) text_pattern_ops);
+ANALYZE regional_place_names;