feat: add sovereign map server migration runbook, seeding scripts, and tactical maneuver compare view
This commit is contained in:
Hamza-Ayed
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# ==============================================================================
|
||||
# Sovereign Map Platform - Full Autonomous Backup System
|
||||
# منصة الخرائط السيادية - نظام النسخ الاحتياطي الشامل والأتمتة لنقل السيرفر
|
||||
# ==============================================================================
|
||||
# Architect: Hamza Ayed - Founding Tech Architect & Sovereign Mobility Strategist
|
||||
# Stack: PostGIS 15, Martin Tile Server, GraphHopper, Redis, NestJS, React
|
||||
# ==============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ANSI Color Codes
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
BACKUP_BASE_DIR="${1:-$ROOT_DIR/backups}"
|
||||
BACKUP_DIR="$BACKUP_BASE_DIR/sovereign_backup_$TIMESTAMP"
|
||||
ARCHIVE_NAME="sovereign_map_full_backup_$TIMESTAMP.tar.gz"
|
||||
|
||||
echo -e "${CYAN}==================================================================${NC}"
|
||||
echo -e "${CYAN} 🛡️ منصة خرائط سيرو السيادية | إطلاق النسخ الاحتياطي الشامل${NC}"
|
||||
echo -e "${CYAN}==================================================================${NC}"
|
||||
echo -e "📁 مسار المشروع (Project Root): ${YELLOW}$ROOT_DIR${NC}"
|
||||
echo -e "📦 مجلد النسخة الاحتياطية: ${YELLOW}$BACKUP_DIR${NC}"
|
||||
echo -e "⏰ توقيت البدء: ${YELLOW}$(date)${NC}"
|
||||
echo -e "------------------------------------------------------------------"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
mkdir -p "$BACKUP_DIR/database"
|
||||
mkdir -p "$BACKUP_DIR/configs"
|
||||
mkdir -p "$BACKUP_DIR/geodata"
|
||||
mkdir -p "$BACKUP_DIR/styles"
|
||||
|
||||
# 1. PostgreSQL + PostGIS Full Database Dump
|
||||
echo -e "\n${BLUE}[1/5] 🐘 جاري تصدير قاعدة بيانات PostGIS المكانية بالكامل...${NC}"
|
||||
DB_CONTAINER="map-db"
|
||||
|
||||
if docker ps --format '{{.Names}}' | grep -q "^${DB_CONTAINER}$"; then
|
||||
echo -e " ✅ تم العثور على الحاوية: ${GREEN}$DB_CONTAINER${NC}"
|
||||
|
||||
# Custom compressed dump (-Fc: Fast, compressed, supports selective restore)
|
||||
echo -e " ⏳ جاري إنشاء Custom Dump المضغوط (موصى به للاستعادة)..."
|
||||
docker exec "$DB_CONTAINER" pg_dump -U mapuser -d mapdb -Fc > "$BACKUP_DIR/database/mapdb_custom.dump"
|
||||
|
||||
# Plain text SQL schema & functions (for auditing and quick inspection)
|
||||
echo -e " ⏳ جاري تصدير مخطط الجداول والدوال المكانية (Schema-only SQL)..."
|
||||
docker exec "$DB_CONTAINER" pg_dump -U mapuser -d mapdb --schema-only > "$BACKUP_DIR/database/mapdb_schema.sql"
|
||||
|
||||
DUMP_SIZE=$(du -h "$BACKUP_DIR/database/mapdb_custom.dump" | cut -f1)
|
||||
echo -e " ✅ تم تصدير قاعدة البيانات بنجاح! الحجم: ${GREEN}$DUMP_SIZE${NC}"
|
||||
else
|
||||
echo -e " ${YELLOW}⚠️ تحذير: حاوية $DB_CONTAINER ليست قيد التشغيل حالياً!${NC}"
|
||||
echo -e " سيتم تخطي تفريغ الداتابيز المباشر. إذا كان لديك dump سابق سيتم تضمينه."
|
||||
fi
|
||||
|
||||
# 2. Redis State Backup
|
||||
echo -e "\n${BLUE}[2/5] ⚡ جاري حفظ حالة الذاكرة اللحظية والتخزين المؤقت (Redis)...${NC}"
|
||||
REDIS_CONTAINER="map-redis"
|
||||
if docker ps --format '{{.Names}}' | grep -q "^${REDIS_CONTAINER}$"; then
|
||||
echo -e " ⏳ تفعيل BGSAVE في Redis..."
|
||||
docker exec "$REDIS_CONTAINER" redis-cli bgsave || true
|
||||
sleep 2
|
||||
# Copy dump.rdb if accessible
|
||||
docker cp "$REDIS_CONTAINER:/data/dump.rdb" "$BACKUP_DIR/database/redis_dump.rdb" 2>/dev/null || echo " ℹ️ لم يتوفر dump.rdb فوري (ذاكرة كاش خفيفة)."
|
||||
echo -e " ✅ اكتملت إجراءات Redis."
|
||||
fi
|
||||
|
||||
# 3. Critical Configuration Files
|
||||
echo -e "\n${BLUE}[3/5] ⚙️ جاري نسخ ملفات التكوين والبيئة والهندسة المعمارية...${NC}"
|
||||
cp -f "$ROOT_DIR/.env" "$BACKUP_DIR/configs/.env.backup" 2>/dev/null || echo " ⚠️ لم يُعثر على .env"
|
||||
cp -f "$ROOT_DIR/docker-compose.yml" "$BACKUP_DIR/configs/" 2>/dev/null || true
|
||||
cp -f "$ROOT_DIR/docker-compose.map2.yml" "$BACKUP_DIR/configs/" 2>/dev/null || true
|
||||
cp -rf "$ROOT_DIR/infrastructure/docker" "$BACKUP_DIR/configs/" 2>/dev/null || true
|
||||
|
||||
# Styles (Vector & Raster styles)
|
||||
cp -f "$ROOT_DIR"/style*.json "$BACKUP_DIR/styles/" 2>/dev/null || true
|
||||
echo -e " ✅ تم حفظ ملفات التكوين وأنماط الخرائط (Styles)."
|
||||
|
||||
# 4. Sovereign Geodata, Offline Tiles & Routing Graphs
|
||||
echo -e "\n${BLUE}[4/5] 🗺️ جاري أرشفة البيانات الجغرافية السيادية والأوفلاين...${NC}"
|
||||
if [ -d "$ROOT_DIR/data/mbtiles" ]; then
|
||||
echo -e " ⏳ جاري نسخ مربعات الارتفاع والصور الفضائية المحلية (MBTiles)..."
|
||||
cp -rf "$ROOT_DIR/data/mbtiles" "$BACKUP_DIR/geodata/"
|
||||
fi
|
||||
|
||||
if [ -f "$ROOT_DIR/infrastructure/osm-data/jordan-latest.osm.pbf" ]; then
|
||||
echo -e " ⏳ جاري نسخ حزمة خريطة الأردن المفتوحة (OSM PBF)..."
|
||||
cp -f "$ROOT_DIR/infrastructure/osm-data/jordan-latest.osm.pbf" "$BACKUP_DIR/geodata/"
|
||||
fi
|
||||
|
||||
if [ -d "$ROOT_DIR/infrastructure/osm-data/routing-packages" ]; then
|
||||
echo -e " ⏳ جاري نسخ حزم توجيه Valhalla المحسوبة محلياً..."
|
||||
cp -rf "$ROOT_DIR/infrastructure/osm-data/routing-packages" "$BACKUP_DIR/geodata/"
|
||||
fi
|
||||
|
||||
# Add restore script directly into the backup folder for self-contained recovery
|
||||
cp -f "$ROOT_DIR/infrastructure/scripts/restore_sovereign_map.sh" "$BACKUP_DIR/restore_sovereign_map.sh" 2>/dev/null || true
|
||||
chmod +x "$BACKUP_DIR/restore_sovereign_map.sh" 2>/dev/null || true
|
||||
|
||||
# 5. Manifest & Checksums
|
||||
echo -e "\n${BLUE}[5/5] 🔒 جاري توليد بصمات التحقق والنزاهة الرقمية (SHA256)...${NC}"
|
||||
cat <<EOF > "$BACKUP_DIR/MANIFEST.txt"
|
||||
Sovereign Map SaaS Platform Backup Manifest
|
||||
===========================================
|
||||
Timestamp: $TIMESTAMP
|
||||
Backup Date: $(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
Hostname: $(hostname)
|
||||
PostgreSQL Version: 15-3.3 (PostGIS)
|
||||
Project Root: $ROOT_DIR
|
||||
Architect: Hamza Ayed (Siro / Intaleq)
|
||||
EOF
|
||||
|
||||
cd "$BACKUP_DIR"
|
||||
find . -type f ! -name "checksums.sha256" -exec sha256sum {} + > "$BACKUP_DIR/checksums.sha256" 2>/dev/null || \
|
||||
find . -type f ! -name "checksums.sha256" -exec shasum -a 256 {} + > "$BACKUP_DIR/checksums.sha256"
|
||||
|
||||
# Package into a unified compressed tarball
|
||||
echo -e " 📦 ضغط الحزمة الكاملة في ملف أرشيف موحد..."
|
||||
cd "$BACKUP_BASE_DIR"
|
||||
tar -czf "$ARCHIVE_NAME" "sovereign_backup_$TIMESTAMP"
|
||||
|
||||
TOTAL_ARCHIVE_SIZE=$(du -h "$BACKUP_BASE_DIR/$ARCHIVE_NAME" | cut -f1)
|
||||
|
||||
echo -e "\n${GREEN}==================================================================${NC}"
|
||||
echo -e "${GREEN} 🎉 تم إنجاز النسخة الاحتياطية بنجاح تام!${NC}"
|
||||
echo -e "${GREEN}==================================================================${NC}"
|
||||
echo -e "📁 مجلد النسخة المفكوكة: ${CYAN}$BACKUP_DIR${NC}"
|
||||
echo -e "📦 ملف الأرشيف المضغوط: ${YELLOW}$BACKUP_BASE_DIR/$ARCHIVE_NAME${NC} (${GREEN}$TOTAL_ARCHIVE_SIZE${NC})"
|
||||
echo -e "🔒 بصمات التحقق: ${CYAN}$BACKUP_DIR/checksums.sha256${NC}"
|
||||
echo -e "------------------------------------------------------------------"
|
||||
echo -e "${YELLOW}🚀 أمر النقل المباشر إلى السيرفر الجديد (One-Liner Transfer Command):${NC}"
|
||||
echo -e "${CYAN}rsync -avzP -e \"ssh -i ~/.ssh/your-key\" \"$BACKUP_BASE_DIR/$ARCHIVE_NAME\" root@<TARGET_SERVER_IP>:/home/hamzadoctor/backups/${NC}"
|
||||
echo -e "${GREEN}==================================================================${NC}"
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env bash
|
||||
# ==============================================================================
|
||||
# Sovereign Map Platform - Full Autonomous Restoration System
|
||||
# منصة الخرائط السيادية - نظام الاستعادة الشامل وإعادة التشغيل على سيرفر جديد
|
||||
# ==============================================================================
|
||||
# Architect: Hamza Ayed - Founding Tech Architect & Sovereign Mobility Strategist
|
||||
# ==============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ANSI Color Codes
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
echo -e "${CYAN}==================================================================${NC}"
|
||||
echo -e "${CYAN} 🛡️ منصة خرائط سيرو السيادية | معالج استعادة النظام والسيرفر${NC}"
|
||||
echo -e "${CYAN}==================================================================${NC}"
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
echo -e "${RED}❌ خطأ: يرجى تحديد مسار ملف النسخة الاحتياطية (.tar.gz) أو مجلد النسخة.${NC}"
|
||||
echo -e "الاستخدام: $0 <path_to_backup.tar.gz | path_to_backup_dir>"
|
||||
echo -e "مثال: $0 /home/hamzadoctor/backups/sovereign_map_full_backup_20260926_120000.tar.gz"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INPUT_PATH="$1"
|
||||
TEMP_RESTORE_DIR=""
|
||||
|
||||
# Determine whether input is a tarball or a directory
|
||||
if [ -f "$INPUT_PATH" ] && [[ "$INPUT_PATH" == *.tar.gz ]]; then
|
||||
echo -e "📦 تم التعرف على ملف أرشيف مضغوط: ${YELLOW}$INPUT_PATH${NC}"
|
||||
TEMP_RESTORE_DIR=$(mktemp -d /tmp/sovereign_restore_XXXXXX)
|
||||
echo -e "⏳ جاري فك ضغط الأرشيف إلى: ${CYAN}$TEMP_RESTORE_DIR${NC}..."
|
||||
tar -xzf "$INPUT_PATH" -C "$TEMP_RESTORE_DIR"
|
||||
# Find the unpacked inner folder
|
||||
RESTORE_SRC=$(find "$TEMP_RESTORE_DIR" -maxdepth 1 -mindepth 1 -type d | head -n 1)
|
||||
if [ -z "$RESTORE_SRC" ]; then
|
||||
RESTORE_SRC="$TEMP_RESTORE_DIR"
|
||||
fi
|
||||
elif [ -d "$INPUT_PATH" ]; then
|
||||
echo -e "📁 تم التعرف على مجلد نسخة احتياطية: ${YELLOW}$INPUT_PATH${NC}"
|
||||
RESTORE_SRC="$INPUT_PATH"
|
||||
else
|
||||
echo -e "${RED}❌ المسار المحدد غير صالح: $INPUT_PATH${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "🔍 فحص مكونات النسخة الاحتياطية في: ${GREEN}$RESTORE_SRC${NC}"
|
||||
|
||||
# Check for database dump
|
||||
DUMP_FILE="$RESTORE_SRC/database/mapdb_custom.dump"
|
||||
SQL_FILE="$RESTORE_SRC/database/mapdb_schema.sql"
|
||||
|
||||
if [ ! -f "$DUMP_FILE" ] && [ ! -f "$SQL_FILE" ]; then
|
||||
echo -e "${RED}❌ لم يتم العثور على ملفات قاعدة البيانات (mapdb_custom.dump أو mapdb_schema.sql)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 1: Ensure Docker is active
|
||||
echo -e "\n${BLUE}[1/6] 🐳 فحص بيئة Docker و Docker Compose...${NC}"
|
||||
if ! command -v docker &> /dev/null; then
|
||||
echo -e "${RED}❌ Docker غير مثبت على هذا السيرفر! يرجى تثبيته أولاً.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DOCKER_COMPOSE_CMD=""
|
||||
if docker compose version &> /dev/null; then
|
||||
DOCKER_COMPOSE_CMD="docker compose"
|
||||
elif command -v docker-compose &> /dev/null; then
|
||||
DOCKER_COMPOSE_CMD="docker-compose"
|
||||
else
|
||||
echo -e "${RED}❌ docker compose غير متاح!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo -e " ✅ محرك الحاويات جاهز: ${GREEN}$($DOCKER_COMPOSE_CMD version --short)${NC}"
|
||||
|
||||
# Step 2: Check & prepare .env file
|
||||
echo -e "\n${BLUE}[2/6] ⚙️ تهيئة متغيرات البيئة (.env)...${NC}"
|
||||
cd "$ROOT_DIR"
|
||||
if [ ! -f "$ROOT_DIR/.env" ] && [ -f "$RESTORE_SRC/configs/.env.backup" ]; then
|
||||
echo -e " 📥 استعادة ملف .env من النسخة الاحتياطية..."
|
||||
cp "$RESTORE_SRC/configs/.env.backup" "$ROOT_DIR/.env"
|
||||
elif [ -f "$ROOT_DIR/.env" ]; then
|
||||
echo -e " ✅ ملف .env موجود مسبقاً في مسار المشروع."
|
||||
else
|
||||
echo -e " ${YELLOW}⚠️ لم يوجد .env! يرجى إنشاء ملف .env بالاعتماد على .env.example${NC}"
|
||||
fi
|
||||
|
||||
# Step 3: Restore offline MBTiles & OSM packages
|
||||
echo -e "\n${BLUE}[3/6] 🗺️ استعادة المربعات السيادية وحزم التوجيه (Offline Datasets)...${NC}"
|
||||
if [ -d "$RESTORE_SRC/geodata/mbtiles" ]; then
|
||||
mkdir -p "$ROOT_DIR/data/mbtiles"
|
||||
echo -e " 📥 استعادة ملفات الارتفاع والأقمار الصناعية (MBTiles)..."
|
||||
cp -rf "$RESTORE_SRC/geodata/mbtiles/"* "$ROOT_DIR/data/mbtiles/" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [ -f "$RESTORE_SRC/geodata/jordan-latest.osm.pbf" ]; then
|
||||
mkdir -p "$ROOT_DIR/infrastructure/osm-data"
|
||||
echo -e " 📥 استعادة ملف خريطة الأردن الخام (OSM PBF)..."
|
||||
cp -f "$RESTORE_SRC/geodata/jordan-latest.osm.pbf" "$ROOT_DIR/infrastructure/osm-data/"
|
||||
fi
|
||||
|
||||
if [ -d "$RESTORE_SRC/geodata/routing-packages" ]; then
|
||||
mkdir -p "$ROOT_DIR/infrastructure/osm-data/routing-packages"
|
||||
echo -e " 📥 استعادة حزم التوجيه المحلية (Valhalla routing)..."
|
||||
cp -rf "$RESTORE_SRC/geodata/routing-packages/"* "$ROOT_DIR/infrastructure/osm-data/routing-packages/" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Step 4: Boot Database & Redis Containers
|
||||
echo -e "\n${BLUE}[4/6] 🐘 تشغيل حاويات الداتابيز والكاش والتأكد من جاهزيتها...${NC}"
|
||||
$DOCKER_COMPOSE_CMD up -d db redis
|
||||
|
||||
echo -e " ⏳ في انتظار جاهزية PostgreSQL و PostGIS..."
|
||||
for i in {1..30}; do
|
||||
if docker exec map-db pg_isready -U mapuser -d mapdb &>/dev/null; then
|
||||
echo -e " ✅ قاعدة البيانات PostGIS جاهزة للحقن!"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
if [ $i -eq 30 ]; then
|
||||
echo -e "${RED}❌ فشل انتظار جاهزية قاعدة البيانات! تفقد سجلات docker logs map-db${NC}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Step 5: Inject PostGIS Database
|
||||
echo -e "\n${BLUE}[5/6] 📥 استعادة وحقن البيانات المكانية في PostGIS...${NC}"
|
||||
if [ -f "$DUMP_FILE" ]; then
|
||||
echo -e " ⏳ جاري الاستعادة من ملف الـ Custom Dump المسرّع..."
|
||||
# Ensure postgis extension exists
|
||||
docker exec -i map-db psql -U mapuser -d mapdb -c "CREATE EXTENSION IF NOT EXISTS postgis;"
|
||||
docker exec -i map-db psql -U mapuser -d mapdb -c "CREATE EXTENSION IF NOT EXISTS postgis_topology;"
|
||||
|
||||
# Restore dump (ignore non-fatal warnings for clean recreation)
|
||||
docker exec -i map-db pg_restore -U mapuser -d mapdb --clean --if-exists --no-owner --no-acl < "$DUMP_FILE" || true
|
||||
|
||||
# Run VACUUM ANALYZE to optimize geometry indices
|
||||
echo -e " ⚡ تحسين الفهارس المكانية (VACUUM ANALYZE)..."
|
||||
docker exec -i map-db psql -U mapuser -d mapdb -c "VACUUM ANALYZE;"
|
||||
echo -e " ✅ تم استعادة قاعدة البيانات وتحديث الفهارس بنجاح!"
|
||||
fi
|
||||
|
||||
# Step 6: Start Full Application Stack & Healthcheck
|
||||
echo -e "\n${BLUE}[6/6] 🚀 بناء وتشغيل كافة خدمات المنصة السيادية...${NC}"
|
||||
$DOCKER_COMPOSE_CMD up -d --build api martin web dashboard routing
|
||||
|
||||
echo -e "\n⏳ فحص صحة الخدمات (Health Checks)..."
|
||||
sleep 5
|
||||
|
||||
check_service() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
if curl -s -f -m 5 "$url" > /dev/null 2>&1; then
|
||||
echo -e " ✅ خدمة ${GREEN}$name${NC} تعمل بنجاح: $url"
|
||||
else
|
||||
echo -e " ℹ️ خدمة ${YELLOW}$name${NC} في طور الإقلاع أو تتطلب ثوانٍ إضافية: $url"
|
||||
fi
|
||||
}
|
||||
|
||||
check_service "Backend API (NestJS)" "http://localhost:3200/api/health"
|
||||
check_service "Martin Vector Tiles" "http://localhost:3202/health"
|
||||
check_service "GraphHopper Routing" "http://localhost:8989/health"
|
||||
check_service "Web Tactical Portal" "http://localhost:3201"
|
||||
check_service "Developer Dashboard" "http://localhost:3204"
|
||||
|
||||
# Cleanup temp unpack dir if created
|
||||
if [ -n "$TEMP_RESTORE_DIR" ] && [ -d "$TEMP_RESTORE_DIR" ]; then
|
||||
rm -rf "$TEMP_RESTORE_DIR"
|
||||
fi
|
||||
|
||||
echo -e "\n${GREEN}==================================================================${NC}"
|
||||
echo -e "${GREEN} 🎉 اكتملت عملية الاستعادة بنجاح والمنصة تعمل بكامل طاقتها!${NC}"
|
||||
echo -e "${GREEN}==================================================================${NC}"
|
||||
echo -e "🌐 البوابة التكتيكية والمناورة: http://localhost:3201/#maneuver"
|
||||
echo -e "📡 خادم الخرائط السيادي (Martin): http://localhost:3202"
|
||||
echo -e "⚡ محرك التوجيه (GraphHopper): http://localhost:8989"
|
||||
echo -e "💼 لوحة المطورين والإدارة: http://localhost:3204"
|
||||
echo -e "------------------------------------------------------------------"
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sovereign 3D DEM Tile Seeder for Jordan (سيرفر سيادي لتخزين تضاريس الأردن ثلاثية الأبعاد)
|
||||
Downloads Terrarium 3D DEM elevation tiles for the Jordan National Bounding Box
|
||||
and packages them into a sovereign MBTiles SQLite database and/or local tile directory.
|
||||
|
||||
Bounding Box (Jordan):
|
||||
West: 34.8°, South: 29.1°, East: 39.3°, North: 33.4°
|
||||
Zoom Levels: 0 to 14 (overscaled to z20 by MapLibre GPU)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
import sqlite3
|
||||
import argparse
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
# National Jordan Geographic Envelope
|
||||
JORDAN_BBOX = {
|
||||
'min_lon': 34.8,
|
||||
'min_lat': 29.1,
|
||||
'max_lon': 39.3,
|
||||
'max_lat': 33.4
|
||||
}
|
||||
|
||||
DEM_URL_TEMPLATE = "https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png"
|
||||
|
||||
def deg2num(lat_deg, lon_deg, zoom):
|
||||
lat_rad = math.radians(lat_deg)
|
||||
n = 2.0 ** zoom
|
||||
xtile = int((lon_deg + 180.0) / 360.0 * n)
|
||||
ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
|
||||
return (xtile, ytile)
|
||||
|
||||
def init_mbtiles(db_path, min_zoom, max_zoom):
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
cur.execute("PRAGMA synchronous = NORMAL")
|
||||
cur.execute("PRAGMA journal_mode = WAL")
|
||||
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS metadata (
|
||||
name TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS tiles (
|
||||
zoom_level INTEGER,
|
||||
tile_column INTEGER,
|
||||
tile_row INTEGER,
|
||||
tile_data BLOB,
|
||||
PRIMARY KEY (zoom_level, tile_column, tile_row)
|
||||
);
|
||||
""")
|
||||
|
||||
meta = [
|
||||
('name', 'Jordan Sovereign 3D Elevation DEM (Terrarium)'),
|
||||
('type', 'baselayer'),
|
||||
('version', '1.0'),
|
||||
('description', 'High-accuracy Sovereign 3D DEM Terrarium Tiles for the Hashemite Kingdom of Jordan'),
|
||||
('format', 'png'),
|
||||
('bounds', f"{JORDAN_BBOX['min_lon']},{JORDAN_BBOX['min_lat']},{JORDAN_BBOX['max_lon']},{JORDAN_BBOX['max_lat']}"),
|
||||
('minzoom', str(min_zoom)),
|
||||
('maxzoom', str(max_zoom)),
|
||||
('attribution', '© Intaleq Sovereign Spatial Engine | Mapzen Terrarium')
|
||||
]
|
||||
for k, v in meta:
|
||||
cur.execute("INSERT OR REPLACE INTO metadata (name, value) VALUES (?, ?)", (k, v))
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
def download_tile(z, x, y, retries=3):
|
||||
url = DEM_URL_TEMPLATE.format(z=z, x=x, y=y)
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={'User-Agent': 'Intaleq-Sovereign-Map-Seeder/1.0'}
|
||||
)
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
if response.status == 200:
|
||||
return (z, x, y, response.read(), None)
|
||||
except Exception as e:
|
||||
if attempt == retries - 1:
|
||||
return (z, x, y, None, str(e))
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
return (z, x, y, None, "Timeout")
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Jordan Sovereign 3D DEM Tile Seeder")
|
||||
parser.add_argument("--min-zoom", type=int, default=0, help="Minimum zoom level (default: 0)")
|
||||
parser.add_argument("--max-zoom", type=int, default=14, help="Maximum zoom level (default: 14)")
|
||||
parser.add_argument("--output-mbtiles", type=str, default="data/jordan-dem.mbtiles", help="Output MBTiles path")
|
||||
parser.add_argument("--output-dir", type=str, default="", help="Optional output directory for raw {z}/{x}/{y}.png files")
|
||||
parser.add_argument("--workers", type=int, default=16, help="Concurrent worker threads (default: 16)")
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(os.path.dirname(os.path.abspath(args.output_mbtiles)), exist_ok=True)
|
||||
if args.output_dir:
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
|
||||
print(f"🚀 Initializing Sovereign 3D DEM Tile Seeder for Jordan...")
|
||||
print(f"📍 Geographic Bounding Box: {JORDAN_BBOX}")
|
||||
print(f"🔍 Zoom range: {args.min_zoom} -> {args.max_zoom}")
|
||||
print(f"💾 Target MBTiles: {args.output_mbtiles}")
|
||||
|
||||
conn = init_mbtiles(args.output_mbtiles, args.min_zoom, args.max_zoom)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Pre-calculate tile list
|
||||
tiles_to_download = []
|
||||
for z in range(args.min_zoom, args.max_zoom + 1):
|
||||
x1, y2 = deg2num(JORDAN_BBOX['min_lat'], JORDAN_BBOX['min_lon'], z)
|
||||
x2, y1 = deg2num(JORDAN_BBOX['max_lat'], JORDAN_BBOX['max_lon'], z)
|
||||
min_x, max_x = min(x1, x2), max(x1, x2)
|
||||
min_y, max_y = min(y1, y2), max(y1, y2)
|
||||
for x in range(min_x, max_x + 1):
|
||||
for y in range(min_y, max_y + 1):
|
||||
# Check if tile already exists in MBTiles
|
||||
# Note: MBTiles uses TMS y-coordinates: tms_y = (2^z - 1) - y
|
||||
tms_y = (1 << z) - 1 - y
|
||||
cursor.execute("SELECT 1 FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?", (z, x, tms_y))
|
||||
if not cursor.fetchone():
|
||||
tiles_to_download.append((z, x, y))
|
||||
|
||||
total = len(tiles_to_download)
|
||||
print(f"📦 Total new tiles to fetch: {total:,}")
|
||||
if total == 0:
|
||||
print("✅ All tiles are already cached and present in the sovereign database!")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
batch = []
|
||||
start_time = time.time()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as executor:
|
||||
futures = {executor.submit(download_tile, z, x, y): (z, x, y) for z, x, y in tiles_to_download}
|
||||
|
||||
for future in as_completed(futures):
|
||||
z, x, y, data, err = future.result()
|
||||
if data:
|
||||
tms_y = (1 << z) - 1 - y
|
||||
batch.append((z, x, tms_y, data))
|
||||
downloaded += 1
|
||||
|
||||
# If raw dir specified, also save file
|
||||
if args.output_dir:
|
||||
tile_file_dir = os.path.join(args.output_dir, str(z), str(x))
|
||||
os.makedirs(tile_file_dir, exist_ok=True)
|
||||
with open(os.path.join(tile_file_dir, f"{y}.png"), "wb") as f:
|
||||
f.write(data)
|
||||
else:
|
||||
errors += 1
|
||||
|
||||
if len(batch) >= 200:
|
||||
cursor.executemany("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", batch)
|
||||
conn.commit()
|
||||
batch.clear()
|
||||
elapsed = time.time() - start_time
|
||||
speed = downloaded / elapsed if elapsed > 0 else 0
|
||||
pct = (downloaded + errors) / total * 100
|
||||
print(f"⏳ Progress: {downloaded:,}/{total:,} ({pct:.1f}%) | Speed: {speed:.1f} tiles/s | Errors: {errors}")
|
||||
|
||||
if batch:
|
||||
cursor.executemany("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", batch)
|
||||
conn.commit()
|
||||
|
||||
conn.close()
|
||||
elapsed = time.time() - start_time
|
||||
print(f"\n🎉 Finished! Downloaded: {downloaded:,} tiles in {elapsed:.1f}s. Errors: {errors}.")
|
||||
print(f"🛡️ Sovereign MBTiles stored at: {args.output_mbtiles}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sovereign Satellite Imagery Tile Seeder for Jordan (سيرفر سيادي لتخزين صور الأقمار الصناعية للأردن)
|
||||
Downloads high-resolution photographic satellite imagery tiles for Jordan's National Envelope
|
||||
and stores them into an offline sovereign MBTiles SQLite database.
|
||||
|
||||
Bounding Box (Jordan):
|
||||
West: 34.8°, South: 29.1°, East: 39.3°, North: 33.4°
|
||||
Zoom Levels: 0 to 15 (Strategic & Tactical military coverage)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
import sqlite3
|
||||
import argparse
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
JORDAN_BBOX = {
|
||||
'min_lon': 34.8,
|
||||
'min_lat': 29.1,
|
||||
'max_lon': 39.3,
|
||||
'max_lat': 33.4
|
||||
}
|
||||
|
||||
# Arcgis World Imagery Tile Template
|
||||
SATELLITE_URL_TEMPLATE = "https://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}"
|
||||
|
||||
def deg2num(lat_deg, lon_deg, zoom):
|
||||
lat_rad = math.radians(lat_deg)
|
||||
n = 2.0 ** zoom
|
||||
xtile = int((lon_deg + 180.0) / 360.0 * n)
|
||||
ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
|
||||
return (xtile, ytile)
|
||||
|
||||
def init_mbtiles(db_path, min_zoom, max_zoom):
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
cur.execute("PRAGMA synchronous = NORMAL")
|
||||
cur.execute("PRAGMA journal_mode = WAL")
|
||||
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS metadata (
|
||||
name TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS tiles (
|
||||
zoom_level INTEGER,
|
||||
tile_column INTEGER,
|
||||
tile_row INTEGER,
|
||||
tile_data BLOB,
|
||||
PRIMARY KEY (zoom_level, tile_column, tile_row)
|
||||
);
|
||||
""")
|
||||
|
||||
meta = [
|
||||
('name', 'Jordan Sovereign Satellite Imagery (الأقمار الصناعية السيادية للأردن)'),
|
||||
('type', 'baselayer'),
|
||||
('version', '1.0'),
|
||||
('description', 'High-resolution Sovereign Photographic Satellite Imagery for the Hashemite Kingdom of Jordan'),
|
||||
('format', 'jpg'),
|
||||
('bounds', f"{JORDAN_BBOX['min_lon']},{JORDAN_BBOX['min_lat']},{JORDAN_BBOX['max_lon']},{JORDAN_BBOX['max_lat']}"),
|
||||
('minzoom', str(min_zoom)),
|
||||
('maxzoom', str(max_zoom)),
|
||||
('attribution', '© Intaleq Sovereign Earth Observation Engine')
|
||||
]
|
||||
for k, v in meta:
|
||||
cur.execute("INSERT OR REPLACE INTO metadata (name, value) VALUES (?, ?)", (k, v))
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
def download_tile(z, x, y, retries=3):
|
||||
url = SATELLITE_URL_TEMPLATE.format(z=z, x=x, y=y)
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) IntaleqSovereignEngine/1.0',
|
||||
'Referer': 'https://map-saas.intaleqapp.com/'
|
||||
}
|
||||
)
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=12) as response:
|
||||
if response.status == 200:
|
||||
data = response.read()
|
||||
return (z, x, y, data, None)
|
||||
except Exception as e:
|
||||
if attempt == retries - 1:
|
||||
return (z, x, y, None, str(e))
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
return (z, x, y, None, "Timeout")
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Jordan Sovereign Satellite Imagery Tile Seeder")
|
||||
parser.add_argument("--min-zoom", type=int, default=0, help="Minimum zoom level (default: 0)")
|
||||
parser.add_argument("--max-zoom", type=int, default=14, help="Maximum zoom level (default: 14)")
|
||||
parser.add_argument("--output-mbtiles", type=str, default="data/jordan-satellite.mbtiles", help="Output MBTiles path")
|
||||
parser.add_argument("--output-dir", type=str, default="", help="Optional output directory for raw {z}/{x}/{y}.jpg files")
|
||||
parser.add_argument("--workers", type=int, default=16, help="Concurrent worker threads (default: 16)")
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(os.path.dirname(os.path.abspath(args.output_mbtiles)), exist_ok=True)
|
||||
if args.output_dir:
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
|
||||
print(f"🛰️ Initializing Sovereign Satellite Imagery Seeder for Jordan...")
|
||||
print(f"📍 Geographic Bounding Box: {JORDAN_BBOX}")
|
||||
print(f"🔍 Zoom range: {args.min_zoom} -> {args.max_zoom}")
|
||||
print(f"💾 Target MBTiles: {args.output_mbtiles}")
|
||||
|
||||
conn = init_mbtiles(args.output_mbtiles, args.min_zoom, args.max_zoom)
|
||||
cursor = conn.cursor()
|
||||
|
||||
tiles_to_download = []
|
||||
for z in range(args.min_zoom, args.max_zoom + 1):
|
||||
x1, y2 = deg2num(JORDAN_BBOX['min_lat'], JORDAN_BBOX['min_lon'], z)
|
||||
x2, y1 = deg2num(JORDAN_BBOX['max_lat'], JORDAN_BBOX['max_lon'], z)
|
||||
min_x, max_x = min(x1, x2), max(x1, x2)
|
||||
min_y, max_y = min(y1, y2), max(y1, y2)
|
||||
for x in range(min_x, max_x + 1):
|
||||
for y in range(min_y, max_y + 1):
|
||||
tms_y = (1 << z) - 1 - y
|
||||
cursor.execute("SELECT 1 FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?", (z, x, tms_y))
|
||||
if not cursor.fetchone():
|
||||
tiles_to_download.append((z, x, y))
|
||||
|
||||
total = len(tiles_to_download)
|
||||
print(f"📦 Total new satellite tiles to fetch: {total:,}")
|
||||
if total == 0:
|
||||
print("✅ All satellite tiles are already cached and present in the sovereign database!")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
batch = []
|
||||
start_time = time.time()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as executor:
|
||||
futures = {executor.submit(download_tile, z, x, y): (z, x, y) for z, x, y in tiles_to_download}
|
||||
|
||||
for future in as_completed(futures):
|
||||
z, x, y, data, err = future.result()
|
||||
if data:
|
||||
tms_y = (1 << z) - 1 - y
|
||||
batch.append((z, x, tms_y, data))
|
||||
downloaded += 1
|
||||
|
||||
if args.output_dir:
|
||||
tile_file_dir = os.path.join(args.output_dir, str(z), str(x))
|
||||
os.makedirs(tile_file_dir, exist_ok=True)
|
||||
with open(os.path.join(tile_file_dir, f"{y}.jpg"), "wb") as f:
|
||||
f.write(data)
|
||||
else:
|
||||
errors += 1
|
||||
|
||||
if len(batch) >= 200:
|
||||
cursor.executemany("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", batch)
|
||||
conn.commit()
|
||||
batch.clear()
|
||||
elapsed = time.time() - start_time
|
||||
speed = downloaded / elapsed if elapsed > 0 else 0
|
||||
pct = (downloaded + errors) / total * 100
|
||||
print(f"⏳ Progress: {downloaded:,}/{total:,} ({pct:.1f}%) | Speed: {speed:.1f} tiles/s | Errors: {errors}")
|
||||
|
||||
if batch:
|
||||
cursor.executemany("INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", batch)
|
||||
conn.commit()
|
||||
|
||||
conn.close()
|
||||
elapsed = time.time() - start_time
|
||||
print(f"\n🎉 Finished! Downloaded: {downloaded:,} satellite tiles in {elapsed:.1f}s. Errors: {errors}.")
|
||||
print(f"🛡️ Sovereign Satellite MBTiles stored at: {args.output_mbtiles}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user