feat: introduce routing status tracking for approved roads and telemetry batch queue limits

infrastructure/scripts/check_routing_sync.sh

@@ -1,31 +1,53 @@
 #!/bin/bash
 # --------------------------------------------------------------------------
 # check_routing_sync.sh
-# Runs via cron every minute to check if the admin dashboard requested
-# a routing graph rebuild (due to a newly approved road).
+# Runs via cron every minute. The Redis request is cleared only after the
+# rebuilt engine is healthy, so failed rebuilds remain queued for retry.
 # --------------------------------------------------------------------------
 
-APP_DIR="/home/hamzadoctor/app"
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/home/hamzadoctor/app}"
+DATA_DIR="${APP_DIR}/infrastructure/osm-data"
+LOCK_FILE="/tmp/map-saas-routing-sync.lock"
 cd "${APP_DIR}"
 
+exec 9>"${LOCK_FILE}"
+if ! flock -n 9; then
+    exit 0
+fi
+
 # Read flag from Redis using the redis container
 SYNC_REQ=$(docker compose exec -T redis redis-cli get routing_sync_requested | tr -d '\r')
 
 if [ "$SYNC_REQ" = "1" ] || [ "$SYNC_REQ" = "\"1\"" ]; then
-    echo "$(date): Sync requested! Triggering delta apply and GH rebuild..."
-    
-    # 1. Delete the flag immediately so we don't trigger it again
-    docker compose exec -T redis redis-cli del routing_sync_requested
-    
-    # 2. Apply delta
-    if [ -f "${APP_DIR}/infrastructure/scripts/apply-delta.sh" ]; then
-        bash "${APP_DIR}/infrastructure/scripts/apply-delta.sh"
+    echo "$(date -Is): Sync requested; rebuilding connected approved roads..."
+    docker compose exec -T db psql -U mapuser -d mapdb -c \
+        "UPDATE approved_roads SET routing_status='syncing', routing_error=NULL WHERE routing_status='queued';"
+
+    if ! bash "${APP_DIR}/infrastructure/scripts/apply-delta.sh" "${APP_DIR}" "${DATA_DIR}/master_map.osm.pbf" "${DATA_DIR}/delta.osm"; then
+        docker compose exec -T db psql -U mapuser -d mapdb -c \
+            "UPDATE approved_roads SET routing_status='queued', routing_error='Delta export or merge failed; queued for retry.' WHERE routing_status='syncing';"
+        exit 1
     fi
-    
-    # 3. Restart GraphHopper to rebuild index
+
     docker compose stop routing
-    rm -rf "${APP_DIR}/infrastructure/osm-data/graph-cache" "${APP_DIR}/infrastructure/osm-data/default-gh"
+    rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
     docker compose up -d routing
-    
-    echo "$(date): Routing rebuild triggered successfully."
+
+    for _ in $(seq 1 90); do
+        if curl -fsS http://localhost:8989/health >/dev/null 2>&1; then
+            docker compose exec -T db psql -U mapuser -d mapdb -c \
+                "UPDATE approved_roads SET routing_status='routed', routing_error=NULL, routed_at=NOW() WHERE routing_status='syncing';"
+            docker compose exec -T redis redis-cli del routing_sync_requested >/dev/null
+            echo "$(date -Is): Routing graph is ready; queued roads are routable."
+            exit 0
+        fi
+        sleep 10
+    done
+
+    docker compose exec -T db psql -U mapuser -d mapdb -c \
+        "UPDATE approved_roads SET routing_status='queued', routing_error='GraphHopper did not become healthy; queued for retry.' WHERE routing_status='syncing';"
+    echo "$(date -Is): GraphHopper did not become healthy; retaining sync request for retry." >&2
+    exit 1
 fi