feat: add sovereign map server migration runbook, seeding scripts, and tactical maneuver compare view
This commit is contained in:
Hamza-Ayed
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sovereign 3D DEM Tile Seeder for Jordan (سيرفر سيادي لتخزين تضاريس الأردن ثلاثية الأبعاد)
|
||||
Downloads Terrarium 3D DEM elevation tiles for the Jordan National Bounding Box
|
||||
and packages them into a sovereign MBTiles SQLite database and/or local tile directory.
|
||||
|
||||
Bounding Box (Jordan):
|
||||
West: 34.8°, South: 29.1°, East: 39.3°, North: 33.4°
|
||||
Zoom Levels: 0 to 14 (overscaled to z20 by MapLibre GPU)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
import sqlite3
|
||||
import argparse
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
# National Jordan Geographic Envelope
|
||||
JORDAN_BBOX = {
|
||||
'min_lon': 34.8,
|
||||
'min_lat': 29.1,
|
||||
'max_lon': 39.3,
|
||||
'max_lat': 33.4
|
||||
}
|
||||
|
||||
DEM_URL_TEMPLATE = "https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png"
|
||||
|
||||
def deg2num(lat_deg, lon_deg, zoom):
|
||||
lat_rad = math.radians(lat_deg)
|
||||
n = 2.0 ** zoom
|
||||
xtile = int((lon_deg + 180.0) / 360.0 * n)
|
||||
ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
|
||||
return (xtile, ytile)
|
||||
|
||||
def init_mbtiles(db_path, min_zoom, max_zoom):
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
cur.execute("PRAGMA synchronous = NORMAL")
|
||||
cur.execute("PRAGMA journal_mode = WAL")
|
||||
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS metadata (
|
||||
name TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS tiles (
|
||||
zoom_level INTEGER,
|
||||
tile_column INTEGER,
|
||||
tile_row INTEGER,
|
||||
tile_data BLOB,
|
||||
PRIMARY KEY (zoom_level, tile_column, tile_row)
|
||||
);
|
||||
""")
|
||||
|
||||
meta = [
|
||||
('name', 'Jordan Sovereign 3D Elevation DEM (Terrarium)'),
|
||||
('type', 'baselayer'),
|
||||
('version', '1.0'),
|
||||
('description', 'High-accuracy Sovereign 3D DEM Terrarium Tiles for the Hashemite Kingdom of Jordan'),
|
||||
('format', 'png'),
|
||||
('bounds', f"{JORDAN_BBOX['min_lon']},{JORDAN_BBOX['min_lat']},{JORDAN_BBOX['max_lon']},{JORDAN_BBOX['max_lat']}"),
|
||||
('minzoom', str(min_zoom)),
|
||||
('maxzoom', str(max_zoom)),
|
||||
('attribution', '© Intaleq Sovereign Spatial Engine | Mapzen Terrarium')
|
||||
]
|
||||
for k, v in meta:
|
||||
cur.execute("INSERT OR REPLACE INTO metadata (name, value) VALUES (?, ?)", (k, v))
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
def download_tile(z, x, y, retries=3):
|
||||
url = DEM_URL_TEMPLATE.format(z=z, x=x, y=y)
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={'User-Agent': 'Intaleq-Sovereign-Map-Seeder/1.0'}
|
||||
)
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
if response.status == 200:
|
||||
return (z, x, y, response.read(), None)
|
||||
except Exception as e:
|
||||
if attempt == retries - 1:
|
||||
return (z, x, y, None, str(e))
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
return (z, x, y, None, "Timeout")
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Jordan Sovereign 3D DEM Tile Seeder")
|
||||
parser.add_argument("--min-zoom", type=int, default=0, help="Minimum zoom level (default: 0)")
|
||||
parser.add_argument("--max-zoom", type=int, default=14, help="Maximum zoom level (default: 14)")
|
||||
parser.add_argument("--output-mbtiles", type=str, default="data/jordan-dem.mbtiles", help="Output MBTiles path")
|
||||
parser.add_argument("--output-dir", type=str, default="", help="Optional output directory for raw {z}/{x}/{y}.png files")
|
||||
parser.add_argument("--workers", type=int, default=16, help="Concurrent worker threads (default: 16)")
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(os.path.dirname(os.path.abspath(args.output_mbtiles)), exist_ok=True)
|
||||
if args.output_dir:
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
|
||||
print(f"🚀 Initializing Sovereign 3D DEM Tile Seeder for Jordan...")
|
||||
print(f"📍 Geographic Bounding Box: {JORDAN_BBOX}")
|
||||
print(f"🔍 Zoom range: {args.min_zoom} -> {args.max_zoom}")
|
||||
print(f"💾 Target MBTiles: {args.output_mbtiles}")
|
||||
|
||||
conn = init_mbtiles(args.output_mbtiles, args.min_zoom, args.max_zoom)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pre-calculate tile list
|
||||
tiles_to_download = []
|
||||
for z in range(args.min_zoom, args.max_zoom + 1):
|
||||
x1, y2 = deg2num(JORDAN_BBOX['min_lat'], JORDAN_BBOX['min_lon'], z)
|
||||
x2, y1 = deg2num(JORDAN_BBOX['max_lat'], JORDAN_BBOX['max_lon'], z)
|
||||
min_x, max_x = min(x1, x2), max(x1, x2)
|
||||
min_y, max_y = min(y1, y2), max(y1, y2)
|
||||
for x in range(min_x, max_x + 1):
|
||||
for y in range(min_y, max_y + 1):
|
||||
# Check if tile already exists in MBTiles
|
||||
# Note: MBTiles uses TMS y-coordinates: tms_y = (2^z - 1) - y
|
||||
tms_y = (1 << z) - 1 - y
|
||||
cursor.execute("SELECT 1 FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?", (z, x, tms_y))
|
||||
if not cursor.fetchone():
|
||||
tiles_to_download.append((z, x, y))
|
||||
|
||||
total = len(tiles_to_download)
|
||||
print(f"📦 Total new tiles to fetch: {total:,}")
|
||||
if total == 0:
|
||||
print("✅ All tiles are already cached and present in the sovereign database!")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
batch = []
|
||||
start_time = time.time()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as executor:
|
||||
futures = {executor.submit(download_tile, z, x, y): (z, x, y) for z, x, y in tiles_to_download}
|
||||
|
||||
for future in as_completed(futures):
|
||||
z, x, y, data, err = future.result()
|
||||
if data:
|
||||
tms_y = (1 << z) - 1 - y
|
||||
batch.append((z, x, tms_y, data))
|
||||
downloaded += 1
|
||||
|
||||
# If raw dir specified, also save file
|
||||
if args.output_dir:
|
||||
tile_file_dir = os.path.join(args.output_dir, str(z), str(x))
|
||||
os.makedirs(tile_file_dir, exist_ok=True)
|
||||
with open(os.path.join(tile_file_dir, f"{y}.png"), "wb") as f:
|
||||
f.write(data)
|
||||
else:
|
||||
errors += 1
|
||||
|
||||
if len(batch) >= 200:
|
||||
cursor.executemany("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", batch)
|
||||
conn.commit()
|
||||
batch.clear()
|
||||
elapsed = time.time() - start_time
|
||||
speed = downloaded / elapsed if elapsed > 0 else 0
|
||||
pct = (downloaded + errors) / total * 100
|
||||
print(f"⏳ Progress: {downloaded:,}/{total:,} ({pct:.1f}%) | Speed: {speed:.1f} tiles/s | Errors: {errors}")
|
||||
|
||||
if batch:
|
||||
cursor.executemany("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", batch)
|
||||
conn.commit()
|
||||
|
||||
conn.close()
|
||||
elapsed = time.time() - start_time
|
||||
print(f"\n🎉 Finished! Downloaded: {downloaded:,} tiles in {elapsed:.1f}s. Errors: {errors}.")
|
||||
print(f"🛡️ Sovereign MBTiles stored at: {args.output_mbtiles}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user