feat: implement side-by-side synchronized map comparison tool and infrastructure scripts for connectivity and data updates
This commit is contained in:
Hamza-Ayed
@@ -0,0 +1,115 @@
|
||||
#!/bin/bash
|
||||
# --------------------------------------------------------------------------
|
||||
# apply-delta.sh
|
||||
# Exports approved candidate_roads from PostGIS as OSM XML, then merges
|
||||
# them into master_map.osm.pbf so GraphHopper routes on them.
|
||||
#
|
||||
# Usage: bash apply-delta.sh [APP_DIR] [MASTER_PBF] [DELTA_OSM]
|
||||
# --------------------------------------------------------------------------
|
||||
set -e
|
||||
|
||||
APP_DIR="${1:-/home/hamzadoctor/app}"
|
||||
MASTER_PBF="${2:-${APP_DIR}/infrastructure/osm-data/master_map.osm.pbf}"
|
||||
DELTA_OSM="${3:-${APP_DIR}/infrastructure/osm-data/delta.osm}"
|
||||
MERGED_PBF="${MASTER_PBF%.pbf}_merged.pbf"
|
||||
|
||||
cd "${APP_DIR}"
|
||||
|
||||
echo "🛣️ Exporting approved roads from PostGIS → ${DELTA_OSM}..."
|
||||
|
||||
# Build valid OSM XML in pure SQL (nodes declared before ways, matching negative
|
||||
# IDs). No python/psycopg2 is needed in the db container. Source is approved_roads
|
||||
# (snapped-to-network geometry, EPSG:4326); each LineString becomes one <way>.
|
||||
#
|
||||
# Junction connectivity: if a road has start_node/end_node (real OSM node ids the
|
||||
# API resolved at approval time), the first/last <nd ref> points at that REAL node
|
||||
# instead of a fresh one — so after osmium-merge the way shares a node with the
|
||||
# existing highway and GraphHopper routes THROUGH it. Interior vertices (and any
|
||||
# endpoint with no resolved node) get fresh negative-id nodes as before.
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -t -A -X > "${DELTA_OSM}" 2>/dev/null <<'SQL'
|
||||
WITH ways AS (
|
||||
SELECT id, confidence, "uniqueDriverCount" AS drivers,
|
||||
COALESCE(highway, 'residential') AS highway,
|
||||
start_node, end_node,
|
||||
ROW_NUMBER() OVER (ORDER BY id) AS way_seq
|
||||
FROM approved_roads
|
||||
),
|
||||
pts AS (
|
||||
SELECT w.way_seq, w.confidence, w.drivers, w.highway, w.start_node, w.end_node,
|
||||
dp.path[1] AS pt_order,
|
||||
COUNT(*) OVER (PARTITION BY w.way_seq) AS npts,
|
||||
ST_Y(dp.geom) AS lat, ST_X(dp.geom) AS lon
|
||||
FROM ways w
|
||||
JOIN approved_roads ar ON ar.id = w.id,
|
||||
LATERAL ST_DumpPoints(ar.geometry) AS dp
|
||||
),
|
||||
pts_numbered AS (
|
||||
SELECT *, ROW_NUMBER() OVER (ORDER BY way_seq, pt_order) AS gseq FROM pts
|
||||
),
|
||||
pts_ref AS (
|
||||
SELECT *,
|
||||
CASE WHEN pt_order = 1 AND start_node IS NOT NULL THEN start_node
|
||||
WHEN pt_order = npts AND end_node IS NOT NULL THEN end_node
|
||||
ELSE -1000000 - gseq END AS node_ref,
|
||||
NOT ( (pt_order = 1 AND start_node IS NOT NULL)
|
||||
OR (pt_order = npts AND end_node IS NOT NULL) ) AS emit_node
|
||||
FROM pts_numbered
|
||||
),
|
||||
nodes_xml AS (
|
||||
SELECT string_agg(
|
||||
format(' <node id="%s" version="1" lat="%s" lon="%s"/>', node_ref, lat, lon),
|
||||
E'\n' ORDER BY gseq) AS x
|
||||
FROM pts_ref WHERE emit_node
|
||||
),
|
||||
ways_xml AS (
|
||||
SELECT string_agg(w.x, E'\n') AS x FROM (
|
||||
SELECT format(
|
||||
' <way id="%s" version="1">%s%s%s%s%s </way>',
|
||||
-way_seq,
|
||||
string_agg(format(E'\n <nd ref="%s"/>', node_ref), '' ORDER BY pt_order),
|
||||
format(E'\n <tag k="highway" v="%s"/>', min(highway)),
|
||||
E'\n <tag k="source" v="intaleq:telemetry"/>',
|
||||
format(E'\n <tag k="confidence" v="%s"/>', round(min(confidence)::numeric, 2)),
|
||||
format(E'\n <tag k="intaleq:drivers" v="%s"/>\n', min(drivers))
|
||||
) AS x
|
||||
FROM pts_ref GROUP BY way_seq
|
||||
) w
|
||||
)
|
||||
SELECT format(
|
||||
E'<?xml version="1.0" encoding="UTF-8"?>\n<osm version="0.6" generator="intaleq-delta">\n%s\n%s\n</osm>',
|
||||
COALESCE((SELECT x FROM nodes_xml), ''),
|
||||
COALESCE((SELECT x FROM ways_xml), '')
|
||||
);
|
||||
SQL
|
||||
|
||||
# Bail out cleanly if there are no approved roads yet.
|
||||
if ! grep -q '<way ' "${DELTA_OSM}" 2>/dev/null; then
|
||||
echo "ℹ️ No approved roads to export. Skipping delta merge."
|
||||
rm -f "${DELTA_OSM}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "✅ Exported $(grep -c '<way ' "${DELTA_OSM}") approved road(s) to OSM XML."
|
||||
|
||||
# Merge delta into master PBF using osmium
|
||||
if command -v osmium &> /dev/null; then
|
||||
# osmium merge requires inputs sorted by (type, id). Our SQL emits nodes in
|
||||
# descending-id order, so sort the delta first — otherwise merge is undefined.
|
||||
SORTED_OSM="${DELTA_OSM%.osm}_sorted.osm"
|
||||
echo "🔃 Sorting delta (osmium requires sorted input)..."
|
||||
osmium sort "${DELTA_OSM}" -o "${SORTED_OSM}" --overwrite
|
||||
|
||||
echo "🔀 Merging delta → ${MASTER_PBF} with osmium..."
|
||||
osmium merge "${MASTER_PBF}" "${SORTED_OSM}" -o "${MERGED_PBF}" --overwrite
|
||||
mv "${MERGED_PBF}" "${MASTER_PBF}"
|
||||
rm -f "${SORTED_OSM}"
|
||||
echo "✅ Master PBF updated with approved roads (endpoints share real OSM nodes where resolved)."
|
||||
else
|
||||
echo "⚠️ osmium not found on host."
|
||||
# On Debian/Ubuntu: apt-get install -y osmium-tool
|
||||
# On Mac: brew install osmium-tool
|
||||
echo " Please install osmium-tool: https://osmcode.org/osmium-tool/"
|
||||
echo " Approved roads saved to ${DELTA_OSM} for manual merge."
|
||||
fi
|
||||
|
||||
echo "🏁 Delta apply complete."
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
#!/bin/bash
|
||||
# --------------------------------------------------------------------------
|
||||
# check-node-connectivity.sh (Phase B-2 live validation)
|
||||
# فحص جاهزية اتصال التقاطعات بالتوجيه على قاعدة البيانات الحية
|
||||
#
|
||||
# GraphHopper only connects ways that SHARE an OSM node id. This script verifies
|
||||
# that the osm2pgsql `--slim` middle tables on THIS database can be used to resolve
|
||||
# the real node id nearest each approved-road endpoint (the mechanism the API uses
|
||||
# in connectApprovedRoad + apply-delta.sh). Run it on the DB host after an import.
|
||||
#
|
||||
# Usage: bash infrastructure/scripts/check-node-connectivity.sh
|
||||
# Exit 0 = ready, Exit 1 = middle tables unusable (roads draw but won't route-connect)
|
||||
# --------------------------------------------------------------------------
|
||||
set -euo pipefail
|
||||
|
||||
q() { docker compose exec -T db psql -U mapuser -d mapdb -t -A -X -c "$1" 2>/dev/null | tr -d '\r'; }
|
||||
trim() { echo "$1" | tr -d '[:space:]'; }
|
||||
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo " Node-connectivity diagnostic — osm2pgsql middle tables"
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
|
||||
FAIL=0
|
||||
|
||||
# ── 1. Required columns present? ──────────────────────────────────────────
|
||||
echo ""
|
||||
echo "[1] Schema check:"
|
||||
for pair in "planet_osm_ways:nodes" "planet_osm_nodes:lat" "planet_osm_nodes:lon"; do
|
||||
tbl="${pair%%:*}"; col="${pair##*:}"
|
||||
n=$(trim "$(q "SELECT COUNT(*) FROM information_schema.columns WHERE table_name='${tbl}' AND column_name='${col}'")")
|
||||
if [ "$n" = "1" ]; then
|
||||
echo " ✓ ${tbl}.${col}"
|
||||
else
|
||||
echo " ✗ ${tbl}.${col} MISSING"
|
||||
FAIL=1
|
||||
fi
|
||||
done
|
||||
if [ "$FAIL" = "1" ]; then
|
||||
echo ""
|
||||
echo " ⚠️ Middle tables are not in the expected shape. Most common cause:"
|
||||
echo " the import used --flat-nodes (node locations go to a file, not a table)."
|
||||
echo " Endpoint auto-connection will be skipped (roads still draw on tiles)."
|
||||
echo " To enable it, re-import with --slim and WITHOUT --flat-nodes."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── 2. Node coordinate reconstruction (scaling sanity) ────────────────────
|
||||
echo ""
|
||||
echo "[2] Node coordinate reconstruction (expect a sane lon/lat in your region):"
|
||||
q "SELECT ' node '||id||' -> lon='||round((lon/1e7)::numeric,6)||' lat='||round((lat/1e7)::numeric,6)
|
||||
FROM planet_osm_nodes
|
||||
WHERE lon BETWEEN 240000000 AND 430000000 AND lat BETWEEN 210000000 AND 380000000
|
||||
LIMIT 3"
|
||||
|
||||
# ── 3. Nearest-highway-node lookup around Amman city centre ───────────────
|
||||
echo ""
|
||||
echo "[3] Nearest-highway-node lookup near (35.91, 31.95):"
|
||||
RESULT=$(q "
|
||||
WITH p AS (SELECT ST_Transform(ST_SetSRID(ST_MakePoint(35.91, 31.95), 4326), 3857) AS pt)
|
||||
SELECT n.id||' | lon='||round((n.lon/1e7)::numeric,6)||' lat='||round((n.lat/1e7)::numeric,6)
|
||||
FROM p
|
||||
JOIN planet_osm_line l ON l.highway IS NOT NULL AND l.way && ST_Expand(p.pt, 300)
|
||||
JOIN planet_osm_ways w ON w.id = l.osm_id
|
||||
CROSS JOIN LATERAL unnest(w.nodes) AS wn(node_id)
|
||||
JOIN planet_osm_nodes n ON n.id = wn.node_id
|
||||
ORDER BY ST_SetSRID(ST_MakePoint(n.lon/1e7, n.lat/1e7), 4326) <-> ST_Transform(p.pt, 4326)
|
||||
LIMIT 1")
|
||||
if [ -n "$(trim "$RESULT")" ]; then
|
||||
echo " ✓ resolved node → ${RESULT}"
|
||||
else
|
||||
echo " ✗ no highway node found near Amman — is OSM data imported for this region?"
|
||||
FAIL=1
|
||||
fi
|
||||
|
||||
# ── 4. Current approved_roads connection status ───────────────────────────
|
||||
echo ""
|
||||
echo "[4] approved_roads connection status:"
|
||||
if [ "$(trim "$(q "SELECT COUNT(*) FROM information_schema.tables WHERE table_name='approved_roads'")")" = "1" ]; then
|
||||
q "SELECT ' total='||COUNT(*)||' with start_node='||COUNT(start_node)||' with end_node='||COUNT(end_node) FROM approved_roads"
|
||||
else
|
||||
echo " (approved_roads not created yet — approve a candidate first)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
if [ "$FAIL" = "0" ]; then
|
||||
echo " ✅ READY — endpoint auto-connection will work on this database."
|
||||
echo " New approvals get start_node/end_node; apply-delta.sh shares them,"
|
||||
echo " and GraphHopper routes THROUGH approved roads after the next rebuild."
|
||||
exit 0
|
||||
else
|
||||
echo " ❌ NOT READY — see messages above."
|
||||
exit 1
|
||||
fi
|
||||
@@ -2,59 +2,106 @@
|
||||
# --------------------------------------------------------------------------
|
||||
# Intaleq Map Platform - 10-Day Update Script
|
||||
# سكربت تحديث خرائط انطلاقة - التحديث الدوري (كل 10 أيام)
|
||||
# FIXES:
|
||||
# v2 - Corrected output filename: region.osm.pbf → master_map.osm.pbf
|
||||
# (GraphHopper reads master_map.osm.pbf per docker-compose.yml)
|
||||
# v2 - Added Egypt download and import
|
||||
# v2 - Applies approved-roads delta after every update so custom roads survive
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "🚀 Starting 10-day map update..."
|
||||
echo "🚀 Starting 10-day map update (v2 — Jordan + Syria + Egypt + Delta)..."
|
||||
|
||||
# 1. Configuration (From .env or defaults)
|
||||
PBF_FILE="/data/jordan-latest.osm.pbf"
|
||||
SOURCE_URL="https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
|
||||
APP_DIR="/home/hamzadoctor/app"
|
||||
DATA_DIR="${APP_DIR}/infrastructure/osm-data"
|
||||
# ✅ FIX: Correct filename that GraphHopper actually reads (was: region.osm.pbf)
|
||||
MASTER_FILE="${DATA_DIR}/master_map.osm.pbf"
|
||||
DELTA_FILE="${DATA_DIR}/delta.osm"
|
||||
|
||||
cd "$APP_DIR"
|
||||
cd "${APP_DIR}"
|
||||
|
||||
# 2. Download latest OSM data (Running on Host)
|
||||
# تحميل أحدث البيانات للأردن وسوريا
|
||||
echo "🌍 Downloading latest OpenStreetMap data for Jordan & Syria..."
|
||||
wget -O "infrastructure/osm-data/jordan-latest.osm.pbf.new" "https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
|
||||
wget -O "infrastructure/osm-data/syria-latest.osm.pbf.new" "https://download.geofabrik.de/asia/syria-latest.osm.pbf"
|
||||
# ── Step 1: Download all three country PBFs ────────────────────────────────
|
||||
echo "🌍 Downloading Jordan, Syria & Egypt PBF data from Geofabrik..."
|
||||
|
||||
mv "infrastructure/osm-data/jordan-latest.osm.pbf.new" "infrastructure/osm-data/jordan-latest.osm.pbf"
|
||||
mv "infrastructure/osm-data/syria-latest.osm.pbf.new" "infrastructure/osm-data/syria-latest.osm.pbf"
|
||||
wget -q --show-progress -O "${DATA_DIR}/jordan-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/asia/jordan-latest.osm.pbf"
|
||||
|
||||
# 3. Import new data to PostGIS
|
||||
# استيراد البيانات إلى قاعدة البيانات
|
||||
echo "💾 Importing Jordan (Create)..."
|
||||
docker compose --profile import run --rm osm-import osm2pgsql --create --slim --cache 1000 --database mapdb --host db --user mapuser /data/jordan-latest.osm.pbf
|
||||
wget -q --show-progress -O "${DATA_DIR}/syria-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/africa/egypt-latest.osm.pbf"
|
||||
# ✅ FIX: Syria URL was accidentally downloading Egypt above in old script — corrected
|
||||
wget -q --show-progress -O "${DATA_DIR}/egypt-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/africa/egypt-latest.osm.pbf"
|
||||
wget -q --show-progress -O "${DATA_DIR}/syria-latest.osm.pbf.new" \
|
||||
"https://download.geofabrik.de/asia/syria-latest.osm.pbf"
|
||||
|
||||
echo "💾 Importing Syria (Append)..."
|
||||
docker compose --profile import run --rm osm-import osm2pgsql --append --slim --cache 1000 --database mapdb --host db --user mapuser /data/syria-latest.osm.pbf
|
||||
mv "${DATA_DIR}/jordan-latest.osm.pbf.new" "${DATA_DIR}/jordan-latest.osm.pbf"
|
||||
mv "${DATA_DIR}/syria-latest.osm.pbf.new" "${DATA_DIR}/syria-latest.osm.pbf"
|
||||
mv "${DATA_DIR}/egypt-latest.osm.pbf.new" "${DATA_DIR}/egypt-latest.osm.pbf"
|
||||
|
||||
# 4. Merge Data (Jordan + Syria)
|
||||
OSM_FILE="infrastructure/osm-data/region.osm.pbf"
|
||||
echo "🗺️ Merging Jordan and Syria data into $OSM_FILE..."
|
||||
osmium merge infrastructure/osm-data/jordan-latest.osm.pbf infrastructure/osm-data/syria-latest.osm.pbf -o $OSM_FILE --overwrite
|
||||
echo "✅ Downloads complete."
|
||||
|
||||
# 5. Spatial Integrity check for Geocoding (Landmarks)
|
||||
# Purge any legacy landmarks outside the expanded region
|
||||
echo "📍 Syncing user-submitted landmarks and purging invalid coordinates..."
|
||||
DELETED_COUNT=$(docker compose exec -T db psql -U mapuser -d mapdb -t -c "DELETE FROM places_syria WHERE longitude < 34 OR longitude > 43 OR latitude < 29 OR latitude > 38;")
|
||||
echo " ✅ Purged ${DELETED_COUNT//[[:space:]]/} invalid landmarks outside the expanded region."
|
||||
# ── Step 2: Import all three countries into PostGIS ───────────────────────
|
||||
echo "💾 Importing Jordan into PostGIS (--create resets planet_osm_* tables)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --create --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/jordan-latest.osm.pbf
|
||||
|
||||
# Force Spatial Geometry Update
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -c "UPDATE places_syria SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326) WHERE location IS NULL OR latitude IS NOT NULL;"
|
||||
echo "💾 Importing Syria into PostGIS (--append)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --append --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/syria-latest.osm.pbf
|
||||
|
||||
# 6. Rebuild Routing Index (GraphHopper)
|
||||
echo "🚗 Rebuilding GraphHopper routing index (This may take ~5-8 minutes)..."
|
||||
# ✅ FIX: Egypt was missing from the update cycle — added here
|
||||
echo "💾 Importing Egypt into PostGIS (--append)..."
|
||||
docker compose --profile import run --rm osm-import \
|
||||
osm2pgsql --append --slim --cache 1000 \
|
||||
--database mapdb --host db --user mapuser \
|
||||
/data/egypt-latest.osm.pbf
|
||||
|
||||
# ── Step 3: Merge all three PBFs into master_map.osm.pbf ─────────────────
|
||||
# ✅ FIX: Old script wrote to region.osm.pbf — GraphHopper reads master_map.osm.pbf
|
||||
echo "🗺️ Merging Jordan + Syria + Egypt → ${MASTER_FILE}..."
|
||||
osmium merge \
|
||||
"${DATA_DIR}/jordan-latest.osm.pbf" \
|
||||
"${DATA_DIR}/syria-latest.osm.pbf" \
|
||||
"${DATA_DIR}/egypt-latest.osm.pbf" \
|
||||
-o "${MASTER_FILE}" --overwrite
|
||||
|
||||
echo "✅ Master PBF built: $(du -sh ${MASTER_FILE} | cut -f1)"
|
||||
|
||||
# ── Step 4: Apply approved-roads delta (survives each update) ────────────
|
||||
# ✅ NEW: Export approved candidate_roads as OSM XML and merge into master
|
||||
echo "🛣️ Applying approved-roads delta..."
|
||||
if [ -f "${APP_DIR}/infrastructure/scripts/apply-delta.sh" ]; then
|
||||
bash "${APP_DIR}/infrastructure/scripts/apply-delta.sh" "${APP_DIR}" "${MASTER_FILE}" "${DELTA_FILE}" || \
|
||||
echo "⚠️ Delta apply failed (no approved roads yet?). Continuing without delta."
|
||||
else
|
||||
echo "⚠️ apply-delta.sh not found. Skipping delta step."
|
||||
fi
|
||||
|
||||
# ── Step 5: Spatial integrity check for Geocoding (Landmarks) ────────────
|
||||
echo "📍 Purging landmarks outside valid bounding boxes..."
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -t -c \
|
||||
"DELETE FROM places_syria WHERE longitude < 34 OR longitude > 43 OR latitude < 29 OR latitude > 38;"
|
||||
docker compose exec -T db psql -U mapuser -d mapdb -t -c \
|
||||
"UPDATE places_syria SET location = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)
|
||||
WHERE location IS NULL AND latitude IS NOT NULL;"
|
||||
echo "✅ Landmark sync done."
|
||||
|
||||
# ── Step 6: Rebuild GraphHopper routing index ─────────────────────────────
|
||||
echo "🚗 Rebuilding GraphHopper routing graph (may take 5-10 min)..."
|
||||
docker compose stop routing
|
||||
# Delete old cache - VERY IMPORTANT to force full re-index
|
||||
rm -rf infrastructure/osm-data/graph-cache infrastructure/osm-data/default-gh
|
||||
rm -rf "${DATA_DIR}/graph-cache" "${DATA_DIR}/default-gh"
|
||||
docker compose up -d routing
|
||||
echo "✅ GraphHopper restarting from ${MASTER_FILE}."
|
||||
|
||||
# 7. Final Cleanup & Cache Flush
|
||||
echo "🧹 Clearing Redis traffic cache..."
|
||||
# ── Step 7: Cache flush ───────────────────────────────────────────────────
|
||||
echo "🧹 Flushing Redis traffic cache..."
|
||||
docker compose exec -T redis redis-cli flushall
|
||||
|
||||
echo "Done! Map platform is now fully synchronized with Jordan & Syria roads (including Damascus)."
|
||||
echo ""
|
||||
echo "✅ 10-day update complete — Jordan + Syria + Egypt + approved delta applied."
|
||||
echo " GraphHopper is rebuilding in the background. Allow 5-10 min for routing to be ready."
|
||||
|
||||
Reference in New Issue
Block a user