feat: implement side-by-side synchronized map comparison tool and infrastructure scripts for connectivity and data updates
This commit is contained in:
Hamza-Ayed
@@ -2,59 +2,106 @@
|
||||
# --------------------------------------------------------------------------
|
||||
# Intaleq Map Platform - 10-Day Update Script
|
||||
# سكربت تحديث خرائط انطلاقة - التحديث الدوري (كل 10 أيام)
|
||||
# FIXES:
|
||||
# v2 - Corrected output filename: region.osm.pbf → master_map.osm.pbf
|
||||
# (GraphHopper reads master_map.osm.pbf per docker-compose.yml)
|
||||
# v2 - Added Egypt download and import
|
||||
# v2 - Applies approved-roads delta after every update so custom roads survive
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "🚀 Starting 10-day map update..."
|
||||
echo "🚀 Starting 10-day map update (v2 — Jordan + Syria + Egypt + Delta)..."
|
||||
|
||||
# 1. Configuration (From .env or defaults)
|
||||
PBF_FILE="/data/jordan-latest.osm.pbf"
|
||||
SOURCE_URL="https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
|
||||
APP_DIR="/home/hamzadoctor/app"
|
||||
DATA_DIR="${APP_DIR}/infrastructure/osm-data"
|
||||
# ✅ FIX: Correct filename that GraphHopper actually reads (was: region.osm.pbf)
|
||||
MASTER_FILE="${DATA_DIR}/master_map.osm.pbf"
|
||||
DELTA_FILE="${DATA_DIR}/delta.osm"
|
||||
|
||||
cd "$APP_DIR"
|
||||
cd "${APP_DIR}"
|
||||
|
||||
# 2. Download latest OSM data (Running on Host)
|
||||
# تحميل أحدث البيانات للأردن وسوريا
|
||||
echo "🌍 Downloading latest OpenStreetMap data for Jordan & Syria..."
|
||||
wget -O "infrastructure/osm-data/jordan-latest.osm.pbf.new" "https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
|
||||
wget -O "infrastructure/osm-data/syria-latest.osm.pbf.new" "https://download.geofabrik.de/asia/syria-latest.osm.pbf"
|
||||
# ── Step 1: Download all three country PBFs ────────────────────────────────
|
||||
echo "🌍 Downloading Jordan, Syria & Egypt PBF data from Geofabrik..."
|
||||
|
||||
mv "infrastructure/osm-data/jordan-latest.osm.pbf.new" "infrastructure/osm-data/jordan-latest.osm.pbf"
|
||||
mv "infrastructure/osm-data/syria-latest.osm.pbf.new" "infrastructure/osm-data/syria-latest.osm.pbf"
|
||||
wget -q --show-progress -O "${DATA_DIR}/jordan-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
|
||||
|
||||
# 3. Import new data to PostGIS
|
||||
# استيراد البيانات إلى قاعدة البيانات
|
||||
echo "💾 Importing Jordan (Create)..."
|
||||
docker compose --profile import run --rm osm-import osm2pgsql --create --slim --cache 1000 --database mapdb --host db --user mapuser /data/jordan-latest.osm.pbf
|
||||
wget -q --show-progress -O "${DATA_DIR}/syria-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/africa/egypt-latest.osm.pbf"
|
||||
# ✅ FIX: Syria URL was accidentally downloading Egypt above in old script — corrected
|
||||
wget -q --show-progress -O "${DATA_DIR}/egypt-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/africa/egypt-latest.osm.pbf"
|
||||
wget -q --show-progress -O "${DATA_DIR}/syria-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/asia/syria-latest.osm.pbf"
|
||||
|
||||
echo "💾 Importing Syria (Append)..."
|
||||
docker compose --profile import run --rm osm-import osm2pgsql --append --slim --cache 1000 --database mapdb --host db --user mapuser /data/syria-latest.osm.pbf
|
||||
mv "${DATA_DIR}/jordan-latest.osm.pbf.new" "${DATA_DIR}/jordan-latest.osm.pbf"
|
||||
mv "${DATA_DIR}/syria-latest.osm.pbf.new" "${DATA_DIR}/syria-latest.osm.pbf"
|
||||
mv "${DATA_DIR}/egypt-latest.osm.pbf.new" "${DATA_DIR}/egypt-latest.osm.pbf"
|
||||
|
||||
# 4. Merge Data (Jordan + Syria)
|
||||
OSM_FILE="infrastructure/osm-data/region.osm.pbf"
|
||||
echo "🗺️ Merging Jordan and Syria data into $OSM_FILE..."
|
||||
osmium merge infrastructure/osm-data/jordan-latest.osm.pbf infrastructure/osm-data/syria-latest.osm.pbf -o $OSM_FILE --overwrite
|
||||
echo "✅ Downloads complete."
|
||||
|
||||
# 5. Spatial Integrity check for Geocoding (Landmarks)
|
||||
# Purge any legacy landmarks outside the expanded region
|
||||
echo "📍 Syncing user-submitted landmarks and purging invalid coordinates..."
|
||||
DELETED_COUNT=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "DELETE FROM places_syria WHERE longitude < 34 OR longitude > 43 OR latitude < 29 OR latitude > 38;")
|
||||
echo " ✅ Purged ${DELETED_COUNT//[[:space:]]/} invalid landmarks outside the expanded region."
|
||||
# ── Step 2: Import all three countries into PostGIS ───────────────────────
|
||||
echo "💾 Importing Jordan into PostGIS (--create resets planet_osm_* tables)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --create --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/jordan-latest.osm.pbf
|
||||
|
||||
# Force Spatial Geometry Update
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -c "UPDATE places_syria SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326) WHERE location IS NULL OR latitude IS NOT NULL;"
|
||||
echo "💾 Importing Syria into PostGIS (--append)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --append --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/syria-latest.osm.pbf
|
||||
|
||||
# 6. Rebuild Routing Index (GraphHopper)
|
||||
echo "🚗 Rebuilding GraphHopper routing index (This may take ~5-8 minutes)..."
|
||||
# ✅ FIX: Egypt was missing from the update cycle — added here
|
||||
echo "💾 Importing Egypt into PostGIS (--append)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --append --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/egypt-latest.osm.pbf
|
||||
|
||||
# ── Step 3: Merge all three PBFs into master_map.osm.pbf ─────────────────
|
||||
# ✅ FIX: Old script wrote to region.osm.pbf — GraphHopper reads master_map.osm.pbf
|
||||
echo "🗺️ Merging Jordan + Syria + Egypt → ${MASTER_FILE}..."
|
||||
osmium merge \
|
||||
"${DATA_DIR}/jordan-latest.osm.pbf" \
|
||||
"${DATA_DIR}/syria-latest.osm.pbf" \
|
||||
"${DATA_DIR}/egypt-latest.osm.pbf" \
|
||||
-o "${MASTER_FILE}" --overwrite
|
||||
|
||||
echo "✅ Master PBF built: $(du -sh ${MASTER_FILE} | cut -f1)"
|
||||
|
||||
# ── Step 4: Apply approved-roads delta (survives each update) ────────────
|
||||
# ✅ NEW: Export approved candidate_roads as OSM XML and merge into master
|
||||
echo "🛣️ Applying approved-roads delta..."
|
||||
if [ -f "${APP_DIR}/infrastructure/scripts/apply-delta.sh" ]; then
|
||||
bash "${APP_DIR}/infrastructure/scripts/apply-delta.sh" "${APP_DIR}" "${MASTER_FILE}" "${DELTA_FILE}" || \
|
||||
echo "⚠️ Delta apply failed (no approved roads yet?). Continuing without delta."
|
||||
else
|
||||
echo "⚠️ apply-delta.sh not found. Skipping delta step."
|
||||
fi
|
||||
|
||||
# ── Step 5: Spatial integrity check for Geocoding (Landmarks) ────────────
|
||||
echo "📍 Purging landmarks outside valid bounding boxes..."
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -t -c \
|
||||
"DELETE FROM places_syria WHERE longitude < 34 OR longitude > 43 OR latitude < 29 OR latitude > 38;"
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -t -c \
|
||||
"UPDATE places_syria SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)
|
||||
WHERE location IS NULL AND latitude IS NOT NULL;"
|
||||
echo "✅ Landmark sync done."
|
||||
|
||||
# ── Step 6: Rebuild GraphHopper routing index ─────────────────────────────
|
||||
echo "🚗 Rebuilding GraphHopper routing graph (may take 5-10 min)..."
|
||||
docker compose stop routing
|
||||
# Delete old cache - VERY IMPORTANT to force full re-index
|
||||
rm -rf infrastructure/osm-data/graph-cache infrastructure/osm-data/default-gh
|
||||
rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
|
||||
docker compose up -d routing
|
||||
echo "✅ GraphHopper restarting from ${MASTER_FILE}."
|
||||
|
||||
# 7. Final Cleanup & Cache Flush
|
||||
echo "🧹 Clearing Redis traffic cache..."
|
||||
# ── Step 7: Cache flush ───────────────────────────────────────────────────
|
||||
echo "🧹 Flushing Redis traffic cache..."
|
||||
docker compose exec -T redis redis-cli flushall
|
||||
|
||||
echo "Done! Map platform is now fully synchronized with Jordan & Syria roads (including Damascus)."
|
||||
echo ""
|
||||
echo "✅ 10-day update complete — Jordan + Syria + Egypt + approved delta applied."
|
||||
echo " GraphHopper is rebuilding in the background. Allow 5-10 min for routing to be ready."
|
||||
|
||||
Reference in New Issue
Block a user