feat: sync source-backed OSM facility entrances
This commit is contained in:
Hamza-Ayed
@@ -1,4 +1,5 @@
|
||||
#!/bin/bash
|
||||
# وظيفة الملف: أداة تشغيل أو تجهيز باسم update data؛ تفاصيل خطواتها ومدخلاتها موضحة في دليل ملفات المشروع.
|
||||
# --------------------------------------------------------------------------
|
||||
# Intaleq Map Platform - 10-Day Update Script
|
||||
# سكربت تحديث خرائط انطلاقة - التحديث الدوري (كل 10 أيام)
|
||||
@@ -11,62 +12,83 @@
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "🚀 Starting 10-day map update (v3 — Jordan + Syria + Egypt + Iraq + Delta)..."
|
||||
|
||||
APP_DIR="/home/hamzadoctor/app"
|
||||
DATA_DIR="${APP_DIR}/infrastructure/osm-data"
|
||||
DATA_SCOPE_FILE="${DATA_DIR}/map-data-scope"
|
||||
# An explicitly supplied scope takes priority. After a successful regional
|
||||
# expansion, persist it so the scheduled 10-day job cannot drop those regions.
|
||||
MAP_DATA_SCOPE="${MAP_DATA_SCOPE:-$(cat "${DATA_SCOPE_FILE}" 2>/dev/null || printf 'core')}"
|
||||
|
||||
case "${MAP_DATA_SCOPE}" in
|
||||
core|gcc|gcc-yemen) ;;
|
||||
*)
|
||||
echo "❌ MAP_DATA_SCOPE must be one of: core, gcc, gcc-yemen (got: ${MAP_DATA_SCOPE})"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "🚀 Starting map update (scope: ${MAP_DATA_SCOPE})..."
|
||||
|
||||
# ✅ FIX: Correct filename that GraphHopper actually reads (was: region.osm.pbf)
|
||||
MASTER_FILE="${DATA_DIR}/master_map.osm.pbf"
|
||||
DELTA_FILE="${DATA_DIR}/delta.osm"
|
||||
|
||||
cd "${APP_DIR}"
|
||||
|
||||
# ── Step 1: Download all four country PBFs ─────────────────────────────────
|
||||
echo "🌍 Downloading Jordan, Syria, Egypt & Iraq PBF data from Geofabrik..."
|
||||
if ! command -v osmium >/dev/null 2>&1; then
|
||||
echo "❌ osmium-tool is required to merge routing extracts. Install it with: sudo apt-get install -y osmium-tool"
|
||||
exit 3
|
||||
fi
|
||||
|
||||
wget -q --show-progress -O "${DATA_DIR}/jordan-latest.osm.pbf.new" \
|
||||
# ── Step 1: Select and download the country PBFs ───────────────────────────
|
||||
# GCC is Geofabrik's combined Saudi Arabia, UAE, Qatar, Oman, Kuwait and Bahrain
|
||||
# extract. Keeping the stage explicit prevents a scheduled update from silently
|
||||
# enlarging the production routing graph.
|
||||
COUNTRIES=(jordan syria egypt iraq)
|
||||
PBF_FILES=(
|
||||
"jordan-latest.osm.pbf"
|
||||
"syria-latest.osm.pbf"
|
||||
"egypt-latest.osm.pbf"
|
||||
"iraq-latest.osm.pbf"
|
||||
)
|
||||
PBF_URLS=(
|
||||
"https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
|
||||
|
||||
# ✅ FIX: كان يحمّل مصر إلى ملف سوريا ثم يعيد تحميل سوريا فوقها — هدر 177MB كل دورة
|
||||
wget -q --show-progress -O "${DATA_DIR}/syria-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/asia/syria-latest.osm.pbf"
|
||||
wget -q --show-progress -O "${DATA_DIR}/egypt-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/africa/egypt-latest.osm.pbf"
|
||||
wget -q --show-progress -O "${DATA_DIR}/iraq-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/asia/iraq-latest.osm.pbf"
|
||||
)
|
||||
|
||||
mv "${DATA_DIR}/jordan-latest.osm.pbf.new" "${DATA_DIR}/jordan-latest.osm.pbf"
|
||||
mv "${DATA_DIR}/syria-latest.osm.pbf.new" "${DATA_DIR}/syria-latest.osm.pbf"
|
||||
mv "${DATA_DIR}/egypt-latest.osm.pbf.new" "${DATA_DIR}/egypt-latest.osm.pbf"
|
||||
mv "${DATA_DIR}/iraq-latest.osm.pbf.new" "${DATA_DIR}/iraq-latest.osm.pbf"
|
||||
if [[ "${MAP_DATA_SCOPE}" == "gcc" || "${MAP_DATA_SCOPE}" == "gcc-yemen" ]]; then
|
||||
COUNTRIES+=(gcc-states)
|
||||
PBF_FILES+=(gcc-states-latest.osm.pbf)
|
||||
PBF_URLS+=("https://download.geofabrik.de/asia/gcc-states-latest.osm.pbf")
|
||||
fi
|
||||
|
||||
if [[ "${MAP_DATA_SCOPE}" == "gcc-yemen" ]]; then
|
||||
COUNTRIES+=(yemen)
|
||||
PBF_FILES+=(yemen-latest.osm.pbf)
|
||||
PBF_URLS+=("https://download.geofabrik.de/asia/yemen-latest.osm.pbf")
|
||||
fi
|
||||
|
||||
echo "🌍 Downloading ${COUNTRIES[*]} PBF data from Geofabrik..."
|
||||
for i in "${!PBF_FILES[@]}"; do
|
||||
wget -q --show-progress -O "${DATA_DIR}/${PBF_FILES[$i]}.new" "${PBF_URLS[$i]}"
|
||||
mv "${DATA_DIR}/${PBF_FILES[$i]}.new" "${DATA_DIR}/${PBF_FILES[$i]}"
|
||||
done
|
||||
|
||||
echo "✅ Downloads complete."
|
||||
|
||||
# ── Step 2: Import all four countries into PostGIS ─────────────────────────
|
||||
echo "💾 Importing Jordan into PostGIS (--create resets planet_osm_* tables)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --create --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/jordan-latest.osm.pbf
|
||||
|
||||
echo "💾 Importing Syria into PostGIS (--append)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --append --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/syria-latest.osm.pbf
|
||||
|
||||
# ✅ FIX: Egypt was missing from the update cycle — added here
|
||||
echo "💾 Importing Egypt into PostGIS (--append)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --append --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/egypt-latest.osm.pbf
|
||||
|
||||
echo "💾 Importing Iraq into PostGIS (--append)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --append --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/iraq-latest.osm.pbf
|
||||
# ── Step 2: Import into PostGIS ────────────────────────────────────────────
|
||||
# The first extract replaces planet_osm_*; every remaining extract appends.
|
||||
for i in "${!PBF_FILES[@]}"; do
|
||||
IMPORT_MODE="--append"
|
||||
[[ "$i" == "0" ]] && IMPORT_MODE="--create"
|
||||
echo "💾 Importing ${COUNTRIES[$i]} into PostGIS (${IMPORT_MODE})..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql "${IMPORT_MODE}" --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
"/data/${PBF_FILES[$i]}"
|
||||
done
|
||||
|
||||
# ── Step 2b: Keep routing topology tables ─────────────────────────────────
|
||||
# Approved roads must share real OSM node IDs with the base graph. Deleting
|
||||
@@ -75,15 +97,19 @@ echo "🧭 Preserving OSM node/way topology for approved-road routing..."
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -c \
|
||||
"DROP TABLE IF EXISTS planet_osm_rels;"
|
||||
|
||||
# ── Step 3: Merge all four PBFs into master_map.osm.pbf ────────────────────
|
||||
# OSM 624165122 is a misclassified 5.94 km² water polygon covering central Zarqa.
|
||||
# Remove it after every full OSM import so it cannot reappear in the rendered map.
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -c \
|
||||
"DELETE FROM planet_osm_polygon WHERE osm_id = 624165122;"
|
||||
|
||||
# ── Step 3: Merge all selected PBFs into master_map.osm.pbf ───────────────
|
||||
# ✅ FIX: Old script wrote to region.osm.pbf — GraphHopper reads master_map.osm.pbf
|
||||
echo "🗺️ Merging Jordan + Syria + Egypt + Iraq → ${MASTER_FILE}..."
|
||||
osmium merge \
|
||||
"${DATA_DIR}/jordan-latest.osm.pbf" \
|
||||
"${DATA_DIR}/syria-latest.osm.pbf" \
|
||||
"${DATA_DIR}/egypt-latest.osm.pbf" \
|
||||
"${DATA_DIR}/iraq-latest.osm.pbf" \
|
||||
-o "${MASTER_FILE}" --overwrite
|
||||
echo "🗺️ Merging ${COUNTRIES[*]} → ${MASTER_FILE}..."
|
||||
MERGE_INPUTS=()
|
||||
for pbf_file in "${PBF_FILES[@]}"; do
|
||||
MERGE_INPUTS+=("${DATA_DIR}/${pbf_file}")
|
||||
done
|
||||
osmium merge "${MERGE_INPUTS[@]}" -o "${MASTER_FILE}" --overwrite
|
||||
|
||||
echo "✅ Master PBF built: $(du -sh ${MASTER_FILE} | cut -f1)"
|
||||
|
||||
@@ -117,7 +143,9 @@ docker compose stop api web dashboard martin || true
|
||||
trap 'echo "▶️ إعادة تشغيل الحاويات المؤقتة..."; docker compose up -d api web dashboard martin || true' EXIT
|
||||
|
||||
docker compose stop routing
|
||||
rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
|
||||
# GraphHopper creates its graph files as root inside the container, so the
|
||||
# host user needs sudo to reliably remove a stale graph before rebuilding.
|
||||
sudo rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
|
||||
docker compose up -d routing
|
||||
echo "✅ GraphHopper restarting from ${MASTER_FILE}."
|
||||
|
||||
@@ -149,6 +177,23 @@ fi
|
||||
echo "🧹 Flushing Redis traffic cache..."
|
||||
docker compose exec -T redis redis-cli flushall
|
||||
|
||||
# Keep named OSM and Overture places in step with the 10-day road refresh.
|
||||
# This includes Egypt, GCC countries and Yemen; the importer uses country
|
||||
# borders and preserves the existing curated place tables.
|
||||
bash "${APP_DIR}/infrastructure/scripts/sync_regional_place_names.sh"
|
||||
docker compose exec -T db psql -v ON_ERROR_STOP=1 -U mapuser -d mapdb \
|
||||
< "${APP_DIR}/infrastructure/sql/10_regional_place_search.sql"
|
||||
|
||||
# Import only explicit OSM entrance/gate nodes from the same 11-country PBF.
|
||||
# No synthetic offsets are created; a node is attached only when its coordinate
|
||||
# is covered by exactly one named target facility polygon.
|
||||
python3 -u "${APP_DIR}/scripts/sync_osm_place_gates.py" \
|
||||
--pbf "${DATA_DIR}/master_map.osm.pbf"
|
||||
docker compose exec -T redis redis-cli flushdb
|
||||
|
||||
# Persist only after every import, graph rebuild and verification has succeeded.
|
||||
printf '%s\n' "${MAP_DATA_SCOPE}" > "${DATA_SCOPE_FILE}"
|
||||
|
||||
echo ""
|
||||
echo "✅ 10-day update complete — Jordan + Syria + Egypt + Iraq + approved delta applied."
|
||||
echo "✅ 10-day update complete — ${COUNTRIES[*]}, approved road delta, regional names, and OSM gates synchronized."
|
||||
echo " GraphHopper is rebuilding in the background. Allow 5-10 min for routing to be ready."
|
||||
|
||||
Reference in New Issue
Block a user