200 lines
10 KiB
Bash
200 lines
10 KiB
Bash
#!/bin/bash
|
|
# وظيفة الملف: أداة تشغيل أو تجهيز باسم update data؛ تفاصيل خطواتها ومدخلاتها موضحة في دليل ملفات المشروع.
|
|
# --------------------------------------------------------------------------
|
|
# Intaleq Map Platform - 10-Day Update Script
|
|
# سكربت تحديث خرائط انطلاقة - التحديث الدوري (كل 10 أيام)
|
|
# FIXES:
|
|
# v2 - Corrected output filename: region.osm.pbf → master_map.osm.pbf
|
|
# (GraphHopper reads master_map.osm.pbf per docker-compose.yml)
|
|
# v2 - Added Egypt download and import
|
|
# v2 - Applies approved-roads delta after every update so custom roads survive
|
|
# --------------------------------------------------------------------------
|
|
|
|
set -e # Exit on error
|
|
|
|
APP_DIR="/home/hamzadoctor/app"
|
|
DATA_DIR="${APP_DIR}/infrastructure/osm-data"
|
|
DATA_SCOPE_FILE="${DATA_DIR}/map-data-scope"
|
|
# An explicitly supplied scope takes priority. After a successful regional
|
|
# expansion, persist it so the scheduled 10-day job cannot drop those regions.
|
|
MAP_DATA_SCOPE="${MAP_DATA_SCOPE:-$(cat "${DATA_SCOPE_FILE}" 2>/dev/null || printf 'core')}"
|
|
|
|
case "${MAP_DATA_SCOPE}" in
|
|
core|gcc|gcc-yemen) ;;
|
|
*)
|
|
echo "❌ MAP_DATA_SCOPE must be one of: core, gcc, gcc-yemen (got: ${MAP_DATA_SCOPE})"
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
echo "🚀 Starting map update (scope: ${MAP_DATA_SCOPE})..."
|
|
|
|
# ✅ FIX: Correct filename that GraphHopper actually reads (was: region.osm.pbf)
|
|
MASTER_FILE="${DATA_DIR}/master_map.osm.pbf"
|
|
DELTA_FILE="${DATA_DIR}/delta.osm"
|
|
|
|
cd "${APP_DIR}"
|
|
|
|
if ! command -v osmium >/dev/null 2>&1; then
|
|
echo "❌ osmium-tool is required to merge routing extracts. Install it with: sudo apt-get install -y osmium-tool"
|
|
exit 3
|
|
fi
|
|
|
|
# ── Step 1: Select and download the country PBFs ───────────────────────────
|
|
# GCC is Geofabrik's combined Saudi Arabia, UAE, Qatar, Oman, Kuwait and Bahrain
|
|
# extract. Keeping the stage explicit prevents a scheduled update from silently
|
|
# enlarging the production routing graph.
|
|
COUNTRIES=(jordan syria egypt iraq)
|
|
PBF_FILES=(
|
|
"jordan-latest.osm.pbf"
|
|
"syria-latest.osm.pbf"
|
|
"egypt-latest.osm.pbf"
|
|
"iraq-latest.osm.pbf"
|
|
)
|
|
PBF_URLS=(
|
|
"https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
|
|
"https://download.geofabrik.de/asia/syria-latest.osm.pbf"
|
|
"https://download.geofabrik.de/africa/egypt-latest.osm.pbf"
|
|
"https://download.geofabrik.de/asia/iraq-latest.osm.pbf"
|
|
)
|
|
|
|
if [[ "${MAP_DATA_SCOPE}" == "gcc" || "${MAP_DATA_SCOPE}" == "gcc-yemen" ]]; then
|
|
COUNTRIES+=(gcc-states)
|
|
PBF_FILES+=(gcc-states-latest.osm.pbf)
|
|
PBF_URLS+=("https://download.geofabrik.de/asia/gcc-states-latest.osm.pbf")
|
|
fi
|
|
|
|
if [[ "${MAP_DATA_SCOPE}" == "gcc-yemen" ]]; then
|
|
COUNTRIES+=(yemen)
|
|
PBF_FILES+=(yemen-latest.osm.pbf)
|
|
PBF_URLS+=("https://download.geofabrik.de/asia/yemen-latest.osm.pbf")
|
|
fi
|
|
|
|
echo "🌍 Downloading ${COUNTRIES[*]} PBF data from Geofabrik..."
|
|
for i in "${!PBF_FILES[@]}"; do
|
|
wget -q --show-progress -O "${DATA_DIR}/${PBF_FILES[$i]}.new" "${PBF_URLS[$i]}"
|
|
mv "${DATA_DIR}/${PBF_FILES[$i]}.new" "${DATA_DIR}/${PBF_FILES[$i]}"
|
|
done
|
|
|
|
echo "✅ Downloads complete."
|
|
|
|
# ── Step 2: Import into PostGIS ────────────────────────────────────────────
|
|
# The first extract replaces planet_osm_*; every remaining extract appends.
|
|
for i in "${!PBF_FILES[@]}"; do
|
|
IMPORT_MODE="--append"
|
|
[[ "$i" == "0" ]] && IMPORT_MODE="--create"
|
|
echo "💾 Importing ${COUNTRIES[$i]} into PostGIS (${IMPORT_MODE})..."
|
|
docker compose --profile import run --rm osm-import \
|
|
osm2pgsql "${IMPORT_MODE}" --slim --cache 1000 \
|
|
--database mapdb --host db --user mapuser \
|
|
"/data/${PBF_FILES[$i]}"
|
|
done
|
|
|
|
# ── Step 2b: Keep routing topology tables ─────────────────────────────────
|
|
# Approved roads must share real OSM node IDs with the base graph. Deleting
|
|
# nodes/ways makes Martin draw a road but leaves GraphHopper unable to connect it.
|
|
echo "🧭 Preserving OSM node/way topology for approved-road routing..."
|
|
docker compose exec -T db psql -U mapuser -d mapdb -c \
|
|
"DROP TABLE IF EXISTS planet_osm_rels;"
|
|
|
|
# OSM 624165122 is a misclassified 5.94 km² water polygon covering central Zarqa.
|
|
# Remove it after every full OSM import so it cannot reappear in the rendered map.
|
|
docker compose exec -T db psql -U mapuser -d mapdb -c \
|
|
"DELETE FROM planet_osm_polygon WHERE osm_id = 624165122;"
|
|
|
|
# ── Step 3: Merge all selected PBFs into master_map.osm.pbf ───────────────
|
|
# ✅ FIX: Old script wrote to region.osm.pbf — GraphHopper reads master_map.osm.pbf
|
|
echo "🗺️ Merging ${COUNTRIES[*]} → ${MASTER_FILE}..."
|
|
MERGE_INPUTS=()
|
|
for pbf_file in "${PBF_FILES[@]}"; do
|
|
MERGE_INPUTS+=("${DATA_DIR}/${pbf_file}")
|
|
done
|
|
osmium merge "${MERGE_INPUTS[@]}" -o "${MASTER_FILE}" --overwrite
|
|
|
|
echo "✅ Master PBF built: $(du -sh ${MASTER_FILE} | cut -f1)"
|
|
|
|
# ── Step 4: Apply approved-roads delta (survives each update) ────────────
|
|
# ✅ NEW: Export approved candidate_roads as OSM XML and merge into master
|
|
echo "🛣️ Applying approved-roads delta..."
|
|
if [ -f "${APP_DIR}/infrastructure/scripts/apply-delta.sh" ]; then
|
|
bash "${APP_DIR}/infrastructure/scripts/apply-delta.sh" "${APP_DIR}" "${MASTER_FILE}" "${DELTA_FILE}" || \
|
|
echo "⚠️ Delta apply failed (no approved roads yet?). Continuing without delta."
|
|
else
|
|
echo "⚠️ apply-delta.sh not found. Skipping delta step."
|
|
fi
|
|
|
|
# ── Step 5: Spatial integrity check for Geocoding (Landmarks) ────────────
|
|
echo "📍 Purging landmarks outside valid bounding boxes..."
|
|
docker compose exec -T db psql -U mapuser -d mapdb -t -c \
|
|
"DELETE FROM places_syria WHERE longitude < 34 OR longitude > 43 OR latitude < 29 OR latitude > 38;"
|
|
docker compose exec -T db psql -U mapuser -d mapdb -t -c \
|
|
"UPDATE places_syria SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)
|
|
WHERE location IS NULL AND latitude IS NOT NULL;"
|
|
echo "✅ Landmark sync done."
|
|
|
|
# ── Step 6: Rebuild GraphHopper routing index ─────────────────────────────
|
|
# ذروة البناء لأربع دول ~3GB. على سيرفر بـ7.8GB الهامش ضيق، فنُخلي الذاكرة
|
|
# للحاويات غير الضرورية أثناء البناء فقط ثم نعيدها. db و redis يبقيان لأن
|
|
# الخطوات التالية تحتاجهما.
|
|
echo "🚗 Rebuilding GraphHopper routing graph (may take 5-10 min)..."
|
|
echo "⏸️ إيقاف مؤقت للحاويات غير الضرورية لتحرير الذاكرة أثناء البناء..."
|
|
docker compose stop api web dashboard martin || true
|
|
# نُعيدها مهما كانت نتيجة البناء — بما في ذلك الخروج المبكر عند الفشل
|
|
trap 'echo "▶️ إعادة تشغيل الحاويات المؤقتة..."; docker compose up -d api web dashboard martin || true' EXIT
|
|
|
|
docker compose stop routing
|
|
# GraphHopper creates its graph files as root inside the container, so the
|
|
# host user needs sudo to reliably remove a stale graph before rebuilding.
|
|
sudo rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
|
|
docker compose up -d routing
|
|
echo "✅ GraphHopper restarting from ${MASTER_FILE}."
|
|
|
|
# ── Step 6b: بوابة تحقق — اتجاهات السير ──────────────────────────────────
|
|
# انحدار 26/07/2026: سقوط قاعدة "!car_access → 0" جعل المحرك يتجاهل كل oneway
|
|
# وكل دوار لأيام دون أن يلاحظ أحد. لا نُنهي التحديث دون إثبات أنها تعمل.
|
|
# التفاصيل: docs/ROUTING_ONEWAY_AR.md
|
|
echo "⏳ انتظار جهوزية المحرك قبل فحص الاتجاهات..."
|
|
GH_READY=0
|
|
for _ in $(seq 1 90); do
|
|
if curl -fsS http://localhost:8989/health >/dev/null 2>&1; then GH_READY=1; break; fi
|
|
sleep 10
|
|
done
|
|
|
|
if [ "$GH_READY" = "0" ]; then
|
|
echo "❌ المحرك لم يجهز خلال 15 دقيقة — راجع: docker compose logs -f routing"
|
|
echo " (السبب المتكرر: exited with code 137 أي نفاد ذاكرة أثناء البناء)"
|
|
exit 1
|
|
fi
|
|
|
|
if ! APP_DIR="${APP_DIR}" bash "${APP_DIR}/infrastructure/scripts/verify-oneway.sh"; then
|
|
echo ""
|
|
echo "❌❌ المحرك يتجاهل اتجاهات السير — التوجيه غير صالح للاستخدام."
|
|
echo " اقرأ: docs/ROUTING_ONEWAY_AR.md"
|
|
exit 1
|
|
fi
|
|
|
|
# ── Step 7: Cache flush ───────────────────────────────────────────────────
|
|
echo "🧹 Flushing Redis traffic cache..."
|
|
docker compose exec -T redis redis-cli flushall
|
|
|
|
# Keep named OSM and Overture places in step with the 10-day road refresh.
|
|
# This includes Egypt, GCC countries and Yemen; the importer uses country
|
|
# borders and preserves the existing curated place tables.
|
|
bash "${APP_DIR}/infrastructure/scripts/sync_regional_place_names.sh"
|
|
docker compose exec -T db psql -v ON_ERROR_STOP=1 -U mapuser -d mapdb \
|
|
< "${APP_DIR}/infrastructure/sql/10_regional_place_search.sql"
|
|
|
|
# Import only explicit OSM entrance/gate nodes from the same 11-country PBF.
|
|
# No synthetic offsets are created; a node is attached only when its coordinate
|
|
# is covered by exactly one named target facility polygon.
|
|
python3 -u "${APP_DIR}/scripts/sync_osm_place_gates.py" \
|
|
--pbf "${DATA_DIR}/master_map.osm.pbf"
|
|
docker compose exec -T redis redis-cli flushdb
|
|
|
|
# Persist only after every import, graph rebuild and verification has succeeded.
|
|
printf '%s\n' "${MAP_DATA_SCOPE}" > "${DATA_SCOPE_FILE}"
|
|
|
|
echo ""
|
|
echo "✅ 10-day update complete — ${COUNTRIES[*]}, approved road delta, regional names, and OSM gates synchronized."
|
|
echo " GraphHopper is rebuilding in the background. Allow 5-10 min for routing to be ready."
|