feat: introduce routing status tracking for approved roads and telemetry batch queue limits

infrastructure/scripts/apply-delta.sh

@@ -34,6 +34,7 @@ WITH ways AS (
          start_node, end_node,
          ROW_NUMBER() OVER (ORDER BY id) AS way_seq
   FROM approved_roads
+  WHERE routing_status IN ('queued', 'syncing', 'routed')
 ),
 pts AS (
   SELECT w.way_seq, w.confidence, w.drivers, w.highway, w.oneway, w.start_node, w.end_node,

infrastructure/scripts/check_routing_sync.sh

@@ -1,31 +1,53 @@
 #!/bin/bash
 # --------------------------------------------------------------------------
 # check_routing_sync.sh
-# Runs via cron every minute to check if the admin dashboard requested
-# a routing graph rebuild (due to a newly approved road).
+# Runs via cron every minute. The Redis request is cleared only after the
+# rebuilt engine is healthy, so failed rebuilds remain queued for retry.
 # --------------------------------------------------------------------------
 
-APP_DIR="/home/hamzadoctor/app"
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/home/hamzadoctor/app}"
+DATA_DIR="${APP_DIR}/infrastructure/osm-data"
+LOCK_FILE="/tmp/map-saas-routing-sync.lock"
 cd "${APP_DIR}"
 
+exec 9>"${LOCK_FILE}"
+if ! flock -n 9; then
+    exit 0
+fi
+
 # Read flag from Redis using the redis container
 SYNC_REQ=$(docker compose exec -T redis redis-cli get routing_sync_requested | tr -d '\r')
 
 if [ "$SYNC_REQ" = "1" ] || [ "$SYNC_REQ" = "\"1\"" ]; then
-    echo "$(date): Sync requested! Triggering delta apply and GH rebuild..."
-    
-    # 1. Delete the flag immediately so we don't trigger it again
-    docker compose exec -T redis redis-cli del routing_sync_requested
-    
-    # 2. Apply delta
-    if [ -f "${APP_DIR}/infrastructure/scripts/apply-delta.sh" ]; then
-        bash "${APP_DIR}/infrastructure/scripts/apply-delta.sh"
+    echo "$(date -Is): Sync requested; rebuilding connected approved roads..."
+    docker compose exec -T db psql -U mapuser -d mapdb -c \
+        "UPDATE approved_roads SET routing_status='syncing', routing_error=NULL WHERE routing_status='queued';"
+
+    if ! bash "${APP_DIR}/infrastructure/scripts/apply-delta.sh" "${APP_DIR}" "${DATA_DIR}/master_map.osm.pbf" "${DATA_DIR}/delta.osm"; then
+        docker compose exec -T db psql -U mapuser -d mapdb -c \
+            "UPDATE approved_roads SET routing_status='queued', routing_error='Delta export or merge failed; queued for retry.' WHERE routing_status='syncing';"
+        exit 1
     fi
-    
-    # 3. Restart GraphHopper to rebuild index
+
     docker compose stop routing
-    rm -rf "${APP_DIR}/infrastructure/osm-data/graph-cache" "${APP_DIR}/infrastructure/osm-data/default-gh"
+    rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
     docker compose up -d routing
-    
-    echo "$(date): Routing rebuild triggered successfully."
+
+    for _ in $(seq 1 90); do
+        if curl -fsS http://localhost:8989/health >/dev/null 2>&1; then
+            docker compose exec -T db psql -U mapuser -d mapdb -c \
+                "UPDATE approved_roads SET routing_status='routed', routing_error=NULL, routed_at=NOW() WHERE routing_status='syncing';"
+            docker compose exec -T redis redis-cli del routing_sync_requested >/dev/null
+            echo "$(date -Is): Routing graph is ready; queued roads are routable."
+            exit 0
+        fi
+        sleep 10
+    done
+
+    docker compose exec -T db psql -U mapuser -d mapdb -c \
+        "UPDATE approved_roads SET routing_status='queued', routing_error='GraphHopper did not become healthy; queued for retry.' WHERE routing_status='syncing';"
+    echo "$(date -Is): GraphHopper did not become healthy; retaining sync request for retry." >&2
+    exit 1
 fi

infrastructure/scripts/update-data.sh

@@ -68,15 +68,12 @@ docker compose --profile import run --rm osm-import \
     --database mapdb --host db --user mapuser \
     /data/iraq-latest.osm.pbf
 
-# ── Step 2b: إسقاط جداول osm2pgsql الوسيطة ────────────────────────────────
-# planet_osm_nodes/ways/rels ينشئها --slim لدعم --append فقط، ولا يقرأها
-# التطبيق ولا martin إطلاقاً (جداول الإخراج هي line/polygon/point/roads).
-# قياس 30/07/2026: nodes 5031MB + ways 1400MB ≈ 6.4GB نائمة بين الدورات.
-# آمن لأن --create في الخطوة 2 يعيد إنشاءها من الصفر كل تشغيل.
-# ⚠️ لا تشغّل osm2pgsql --append بعد هذه النقطة حتى الدورة القادمة.
-echo "🧹 إسقاط جداول osm2pgsql الوسيطة (تُعاد في الدورة القادمة)..."
+# ── Step 2b: Keep routing topology tables ─────────────────────────────────
+# Approved roads must share real OSM node IDs with the base graph. Deleting
+# nodes/ways makes Martin draw a road but leaves GraphHopper unable to connect it.
+echo "🧭 Preserving OSM node/way topology for approved-road routing..."
 docker compose exec -T db psql -U mapuser -d mapdb -c \
-    "DROP TABLE IF EXISTS planet_osm_nodes, planet_osm_ways, planet_osm_rels;"
+    "DROP TABLE IF EXISTS planet_osm_rels;"
 
 # ── Step 3: Merge all four PBFs into master_map.osm.pbf ────────────────────
 # ✅ FIX: Old script wrote to region.osm.pbf — GraphHopper reads master_map.osm.pbf