chore: add build artifacts, map assets, and routing data

This commit is contained in:
Hamza-Ayed
2026-09-19 12:34:24 +03:00
parent aa2b9f131f
commit 43ad2e0ad4
327 changed files with 52421 additions and 1810 deletions
+179
View File
@@ -0,0 +1,179 @@
#!/usr/bin/env bash
# ============================================================================
# بناء حزمة التوجيه المحلية (Valhalla Routing Package) للأردن
# ============================================================================
# English:
# Builds an on-device Valhalla routing package for Jordan from the same OSM
# data used by the server-side GraphHopper engine, with real SRTM elevation
# baked into every edge (grade/ascend/descend), admin street context, and a
# tile extract tar that the tactical app downloads and routes against 100%
# offline via valhalla-mobile.
#
# Output: infrastructure/osm-data/routing-packages/
# - jordan-valhalla-<version>.tar (valhalla_tiles.tar + admins.sqlite)
# - jordan-routing-manifest.json (version, sha256, sizeBytes, builtAt)
#
# Runs in two modes:
# 1. Host mode : ./infrastructure/scripts/build-valhalla-tiles.sh (needs docker)
# 2. Container : docker compose --profile tiles run --rm valhalla-tiles
# ============================================================================
set -euo pipefail
VERSION="${JORDAN_ROUTING_VERSION:-jordan-1.0.0-$(date +%Y%m%d)}"
VALHALLA_IMG="ghcr.io/valhalla/valhalla:latest"
# Jordan bounding box with a small buffer so border corridors connect cleanly.
S=29.05; W=34.75; N=32.80; E=39.45
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ROOT="$( cd "$SCRIPT_DIR/../.." && pwd )"
IN_CONTAINER=false
if [ -f /.dockerenv ] || [ "${VALHALLA_IN_CONTAINER:-0}" = "1" ]; then
IN_CONTAINER=true
fi
if [ "$IN_CONTAINER" = true ]; then
# Compose mounts ./infrastructure/osm-data at /data
DATA_DIR="/data"
else
DATA_DIR="$ROOT/infrastructure/osm-data"
fi
OUT_DIR="$DATA_DIR/routing-packages"
WORK_DIR="$DATA_DIR/valhalla-work"
MASTER_PBF="$DATA_DIR/master_map.osm.pbf"
log() { echo "🧭 $1"; }
run_vh() {
# Run a command inside the valhalla toolchain (locally or through docker).
if [ "$IN_CONTAINER" = true ]; then
bash -c "$1"
else
docker run --rm -v "$DATA_DIR":/data -w /data --entrypoint bash "$VALHALLA_IMG" -lc "$1"
fi
}
if [ ! -f "$MASTER_PBF" ]; then
echo "❌ master_map.osm.pbf not found at: $MASTER_PBF" >&2
echo " ضع ملف OSM المدمج في infrastructure/osm-data/master_map.osm.pbf أولاً." >&2
exit 1
fi
mkdir -p "$OUT_DIR" "$WORK_DIR"
log "[1/6] Cutting Jordan extract ($W,$S → $E,$N) ..."
if run_vh 'command -v osmium >/dev/null 2>&1' 2>/dev/null; then
run_vh "osmium extract --bbox $W,$S,$E,$N --strategy smart --overwrite \
/data/master_map.osm.pbf -o /data/valhalla-work/jordan_routing.osm.pbf"
else
log " osmium غير متوفر — جاري استخدام مستخرج Geofabrik للأردن (نفس مصدر OSM)."
curl -fL --retry 3 "https://download.geofabrik.de/asia/jordan-latest.osm.pbf" \
-o "$WORK_DIR/jordan_routing.osm.pbf"
fi
log "[2/6] Generating config + downloading SRTM elevation for Jordan ..."
run_vh '
set -euo pipefail
cd /data/valhalla-work
mkdir -p /data/valhalla-work/valhalla_tiles /data/valhalla-work/elevation
valhalla_build_config \
--mjolnir-tile-dir /data/valhalla-work/valhalla_tiles \
--mjolnir-concurrency "$(nproc)" \
--mjolnir-tile-extract /data/valhalla-work/valhalla_tiles.tar \
--mjolnir-admin /data/valhalla-work/admins.sqlite \
--additional-data-elevation /data/valhalla-work/elevation \
> /data/valhalla-work/valhalla.json
# SRTM 30m elevation for Jordan bbox
valhalla_build_elevation -c /data/valhalla-work/valhalla.json -b 34.75,29.05,39.45,32.80 || true
'
log "[3/6] Building admin database (governorate/street context) ..."
run_vh 'valhalla_build_admins --config /data/valhalla-work/valhalla.json /data/valhalla-work/jordan_routing.osm.pbf'
log "[4/6] Building routing graph tiles (roads + oneway + elevation grades) ..."
run_vh 'valhalla_build_tiles --config /data/valhalla-work/valhalla.json -j "$(nproc)" /data/valhalla-work/jordan_routing.osm.pbf'
log "[5/6] Creating tile extract (tar) and verifying a test route ..."
run_vh '
set -euo pipefail
cd /data/valhalla-work
rm -f /data/valhalla-work/valhalla_tiles.tar
valhalla_build_extract --config /data/valhalla-work/valhalla.json -v --overwrite
# فحص سلامة: عمان ← العقبة يجب أن ينجح بالكامل على الغراف المحلي.
valhalla_service /data/valhalla-work/valhalla.json route \
"{\"locations\":[{\"lat\":31.9539,\"lon\":35.9106},{\"lat\":29.5320,\"lon\":35.0060}],\"costing\":\"auto\",\"units\":\"kilometers\",\"directions_options\":{\"language\":\"ar\"}}" \
> /data/valhalla-work/test_route.json 2>/dev/null || true
python3 - <<'PY'
import json, sys
try:
with open("/data/valhalla-work/test_route.json") as f:
d = json.load(f)
t = d.get("trip", {})
status = t.get("status")
msg = t.get("status_message", "unknown error")
if status == 0:
dist = round(t.get("summary", {}).get("length", 0), 1)
maneuvers = sum(len(l.get("maneuvers", [])) for l in t.get("legs", []))
print(f" ✅ test route OK: {dist} km | {maneuvers} maneuvers")
else:
print(f" ⚠️ test route status: {status} ({msg})")
except Exception as e:
print(f" ℹ️ test route check note: {e}")
PY
'
log "[5.5/6] Extracting road graph (jordan_roads.db) for on-device SQLite routing ..."
PY_SCRIPT="$SCRIPT_DIR/extract_roads_graph.py"
if [ ! -f "$PY_SCRIPT" ] && [ -f "$DATA_DIR/../infrastructure/scripts/extract_roads_graph.py" ]; then
PY_SCRIPT="$DATA_DIR/../infrastructure/scripts/extract_roads_graph.py"
fi
if [ -f "$PY_SCRIPT" ]; then
python3 "$PY_SCRIPT" "$WORK_DIR/jordan_routing.osm.pbf" "$WORK_DIR/jordan_roads.db" || true
fi
log "[6/6] Packaging + manifest ..."
FINAL_TAR="$OUT_DIR/jordan-valhalla-$VERSION.tar"
rm -f "$FINAL_TAR"
PACK_ITEMS="valhalla_tiles.tar admins.sqlite"
if [ -f "$WORK_DIR/jordan_roads.db" ]; then
PACK_ITEMS="$PACK_ITEMS jordan_roads.db"
log " Included jordan_roads.db in package."
fi
tar -cf "$FINAL_TAR" -C "$WORK_DIR" $PACK_ITEMS
if [ "$IN_CONTAINER" = false ]; then
SIZE=$(stat -f%z "$FINAL_TAR" 2>/dev/null || stat -c%s "$FINAL_TAR")
SHA256=$(shasum -a 256 "$FINAL_TAR" | awk '{print $1}')
else
SIZE=$(wc -c < "$FINAL_TAR" | tr -d " ")
SHA256=$(sha256sum "$FINAL_TAR" | awk "{print \$1}")
fi
cat > "$OUT_DIR/jordan-routing-manifest.json" <<MANIFEST
{
"packageId": "jordan-valhalla-routing",
"version": "$VERSION",
"fileName": "jordan-valhalla-$VERSION.tar",
"sizeBytes": $SIZE,
"sha256": "$SHA256",
"engine": "valhalla",
"elevation": "srtm-30m",
"bbox": {"south": $S, "west": $W, "north": $N, "east": $E},
"builtAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
}
MANIFEST
chmod -R a+rwX "$OUT_DIR" 2>/dev/null || true
rm -rf "$WORK_DIR"
echo ""
echo "✅ Done. حزمة التوجيه المحلية جاهزة:"
ls -lh "$FINAL_TAR" "$OUT_DIR/jordan-routing-manifest.json"
@@ -0,0 +1,226 @@
#!/usr/bin/env python3
"""
Extract Jordan OSM Road Network into SQLite Database for 100% On-Device Offline Routing.
Creates 'jordan_roads.db' containing all nodes, edges, geometries, street names, and spatial indexing.
"""
import sys
import os
import sqlite3
import json
import math
HIGHWAY_SPEEDS = {
'motorway': 110,
'motorway_link': 70,
'trunk': 90,
'trunk_link': 60,
'primary': 75,
'primary_link': 50,
'secondary': 60,
'secondary_link': 45,
'tertiary': 50,
'tertiary_link': 35,
'unclassified': 40,
'residential': 35,
'living_street': 20,
'track': 25,
'service': 25,
}
def haversine_dist(lat1, lon1, lat2, lon2):
R = 6371000.0
phi1 = math.radians(lat1)
phi2 = math.radians(lat2)
dphi = math.radians(lat2 - lat1)
dlambda = math.radians(lon2 - lon1)
a = math.sin(dphi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlambda / 2)**2
return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
def build_roads_db(pbf_path, out_db_path):
print(f"🛣️ Extracting roads from {pbf_path} into {out_db_path}...")
try:
import osmium
except ImportError:
print("⚠️ pyosmium not available, installing or using alternative...")
import subprocess
subprocess.check_call([sys.executable, "-m", "pip", "install", "osmium"])
import osmium
if os.path.exists(out_db_path):
os.remove(out_db_path)
conn = sqlite3.connect(out_db_path)
cur = conn.cursor()
cur.execute("PRAGMA journal_mode = WAL;")
cur.execute("PRAGMA synchronous = NORMAL;")
cur.execute("""
CREATE TABLE nodes (
id INTEGER PRIMARY KEY,
lat REAL NOT NULL,
lng REAL NOT NULL
);
""")
cur.execute("""
CREATE TABLE edges (
id INTEGER PRIMARY KEY AUTOINCREMENT,
osm_way_id INTEGER,
from_node INTEGER NOT NULL,
to_node INTEGER NOT NULL,
name TEXT,
highway TEXT NOT NULL,
speed_kmh INTEGER NOT NULL,
oneway INTEGER NOT NULL,
length_m REAL NOT NULL,
geom_json TEXT NOT NULL,
min_lat REAL,
min_lng REAL,
max_lat REAL,
max_lng REAL
);
""")
cur.execute("CREATE INDEX idx_edges_from ON edges(from_node);")
cur.execute("CREATE INDEX idx_edges_to ON edges(to_node);")
cur.execute("CREATE INDEX idx_edges_bbox ON edges(min_lat, max_lat, min_lng, max_lng);")
# Pass 1: Collect road ways and needed node IDs
class RoadWayHandler(osmium.SimpleHandler):
def __init__(self):
super().__init__()
self.needed_nodes = set()
self.ways = []
def way(self, w):
highway = w.tags.get('highway')
if not highway or highway not in HIGHWAY_SPEEDS:
return
# Skip pedestrian-only
if highway in ('footway', 'pedestrian', 'path', 'steps', 'cycleway'):
return
name = w.tags.get('name:ar') or w.tags.get('name') or ''
oneway_tag = w.tags.get('oneway', 'no')
oneway = 1 if oneway_tag in ('yes', '1', 'true') else (-1 if oneway_tag == '-1' else 0)
node_refs = [n.ref for n in w.nodes]
if len(node_refs) < 2:
return
for ref in node_refs:
self.needed_nodes.add(ref)
self.ways.append({
'id': w.id,
'highway': highway,
'name': name,
'oneway': oneway,
'nodes': node_refs
})
print("📖 Pass 1: Scanning road ways...")
way_handler = RoadWayHandler()
way_handler.apply_file(pbf_path)
print(f" Found {len(way_handler.ways):,} road ways and {len(way_handler.needed_nodes):,} road nodes.")
# Pass 2: Extract node coordinates
node_coords = {}
class NodeHandler(osmium.SimpleHandler):
def __init__(self, needed):
super().__init__()
self.needed = needed
def node(self, n):
if n.id in self.needed:
node_coords[n.id] = (round(n.location.lat, 6), round(n.location.lon, 6))
print("📖 Pass 2: Resolving node coordinates...")
node_handler = NodeHandler(way_handler.needed_nodes)
node_handler.apply_file(pbf_path, locations=False)
print(f" Cached coordinates for {len(node_coords):,} nodes.")
# Insert nodes into DB
print("💾 Inserting nodes into SQLite...")
cur.executemany("INSERT OR IGNORE INTO nodes (id, lat, lng) VALUES (?, ?, ?);",
[(nid, lat, lng) for nid, (lat, lng) in node_coords.items()])
# Split ways into edges between intersections
print("✂️ Segmenting ways into routable edges...")
# Count node degrees to identify intersection nodes
node_degree = {}
for w in way_handler.ways:
for nid in w['nodes']:
node_degree[nid] = node_degree.get(nid, 0) + 1
edges_to_insert = []
for w in way_handler.ways:
w_nodes = w['nodes']
highway = w['highway']
speed = HIGHWAY_SPEEDS.get(highway, 40)
name = w['name']
oneway = w['oneway']
current_segment = []
for i, nid in enumerate(w_nodes):
if nid not in node_coords:
continue
current_segment.append(nid)
# Split at intersections or end of way
is_endpoint = (i == 0 or i == len(w_nodes) - 1)
is_intersection = node_degree.get(nid, 0) > 1
if len(current_segment) >= 2 and (is_intersection or is_endpoint):
u = current_segment[0]
v = current_segment[-1]
if u != v:
# Calculate geometry and length
coords = [node_coords[x] for x in current_segment if x in node_coords]
if len(coords) >= 2:
length_m = 0.0
min_lat = min(c[0] for c in coords)
max_lat = max(c[0] for c in coords)
min_lng = min(c[1] for c in coords)
max_lng = max(c[1] for c in coords)
for j in range(len(coords) - 1):
length_m += haversine_dist(coords[j][0], coords[j][1], coords[j+1][0], coords[j+1][1])
geom_json = json.dumps([[c[0], c[1]] for c in coords], separators=(',', ':'))
# Forward edge
if oneway >= 0:
edges_to_insert.append((w['id'], u, v, name, highway, speed, oneway, round(length_m, 1), geom_json, min_lat, min_lng, max_lat, max_lng))
# Reverse edge
if oneway <= 0:
rev_geom = json.dumps([[c[0], c[1]] for c in reversed(coords)], separators=(',', ':'))
edges_to_insert.append((w['id'], v, u, name, highway, speed, oneway, round(length_m, 1), rev_geom, min_lat, min_lng, max_lat, max_lng))
current_segment = [nid]
print(f"💾 Inserting {len(edges_to_insert):,} routable edges into SQLite...")
cur.executemany("""
INSERT INTO edges (osm_way_id, from_node, to_node, name, highway, speed_kmh, oneway, length_m, geom_json, min_lat, min_lng, max_lat, max_lng)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
""", edges_to_insert)
conn.commit()
# Indexing and vacuum
print("⚡ Optimizing database & building indices...")
cur.execute("ANALYZE;")
cur.execute("VACUUM;")
conn.close()
size_mb = os.path.getsize(out_db_path) / (1024 * 1024)
print(f"✅ Successfully built {out_db_path} ({size_mb:.1f} MB) with {len(edges_to_insert):,} edges!")
if __name__ == "__main__":
pbf = sys.argv[1] if len(sys.argv) > 1 else "/data/valhalla-work/jordan_routing.osm.pbf"
out = sys.argv[2] if len(sys.argv) > 2 else "/data/routing-packages/jordan_roads.db"
build_roads_db(pbf, out)
+50 -5
View File
@@ -115,14 +115,19 @@ pip install --quiet overturemaps
# ── تعريف النطاقات القابلة للتشغيل ────────────────────────────────────────
run_jordan() { process jordan_north "$BBOX_JORDAN_NORTH" division_area
process jordan_north "$BBOX_JORDAN_NORTH" place
process jordan_north "$BBOX_JORDAN_NORTH" land_cover
process jordan_south "$BBOX_JORDAN_SOUTH" division_area
process jordan_south "$BBOX_JORDAN_SOUTH" place; }
process jordan_south "$BBOX_JORDAN_SOUTH" place
process jordan_south "$BBOX_JORDAN_SOUTH" land_cover; }
run_syria() { process syria "$BBOX_SYRIA" division_area
process syria "$BBOX_SYRIA" place; }
process syria "$BBOX_SYRIA" place
process syria "$BBOX_SYRIA" land_cover; }
run_egypt() { process egypt "$BBOX_EGYPT" division_area
process egypt "$BBOX_EGYPT" place; }
process egypt "$BBOX_EGYPT" place
process egypt "$BBOX_EGYPT" land_cover; }
run_iraq() { process iraq "$BBOX_IRAQ" division_area
process iraq "$BBOX_IRAQ" place; }
process iraq "$BBOX_IRAQ" place
process iraq "$BBOX_IRAQ" land_cover; }
run_amman() { process amman "$BBOX_AMMAN" building
process amman "$BBOX_AMMAN" segment; }
@@ -151,7 +156,7 @@ done
# ── فهارس مكانية + تسطيح الأسماء (idempotent) ─────────────────────────────
# name_primary إلزامي: عمود names من نوع json وبلاطات MVT لا تحمل كائنات، فلا
# يستطيع الستايل قراءة الاسم منه. التفاصيل: infrastructure/sql/09_overture_name_primary.sql
for t in overture_building overture_segment overture_place; do
for t in overture_building overture_segment overture_place overture_land_cover; do
psql_run -c "
DO \$\$
BEGIN
@@ -167,4 +172,44 @@ for t in overture_building overture_segment overture_place; do
"
done
# ── إنشاء واستخراج الجروف والموانع التكتيكية آلياً في PostGIS ─────────────
echo "⛰️ توليد واستخراج الجروف والموانع الصخرية التكتيكية في PostGIS..."
psql_run -c "
DO \$\$
BEGIN
-- 1. جدول الموانع الصخرية والتكتيكية المدمجة
CREATE TABLE IF NOT EXISTS tactical_terrain_obstacles (
id SERIAL PRIMARY KEY,
osm_id BIGINT,
obstacle_type VARCHAR(64),
severity VARCHAR(32),
name VARCHAR(255),
geometry GEOMETRY(Geometry, 4326),
created_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_tactical_obs_geom ON tactical_terrain_obstacles USING gist (geometry);
CREATE INDEX IF NOT EXISTS idx_tactical_obs_type ON tactical_terrain_obstacles (obstacle_type);
-- 2. دمج المقاطع الصخرية والموانع من OSM و Overture
IF to_regclass('public.planet_osm_line') IS NOT NULL THEN
INSERT INTO tactical_terrain_obstacles (osm_id, obstacle_type, severity, name, geometry)
SELECT
osm_id,
COALESCE(natural, barrier, man_made, waterway) AS obstacle_type,
CASE
WHEN natural = 'cliff' THEN 'SEVERE_NO_GO'
WHEN barrier = 'retaining_wall' THEN 'RESTRICTED'
WHEN barrier IN ('ditch', 'berm') THEN 'TACTICAL_BARRIER'
WHEN waterway = 'wadi' THEN 'DRAINAGE_DEFILE'
ELSE 'OBSTACLE'
END,
name,
geometry
FROM planet_osm_line
WHERE (natural IN ('cliff', 'ridge', 'arete') OR barrier IN ('retaining_wall', 'berm', 'ditch') OR waterway IN ('wadi', 'waterfall'))
ON CONFLICT DO NOTHING;
END IF;
END \$\$;
"
echo "🎉 اكتمل. البلاطات تُخدم عبر martin تلقائياً (WATCH_DB=true)."