feat: add Jordan 3D topographic platform, Siro Map cinematic orbit studio, sun-shadow simulation, and sovereign intelligence security gating

2026-09-19 12:34:26 +03:00
parent 91e9e71ce4
commit 0bf13824c9
180 changed files with 245224 additions and 704 deletions
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Generate SQL migration file to ingest Iraq checkpoints CSV into places_iraq
+and refresh the unified_search_index.
+"""
+import csv
+import sys
+import os
+
+def clean_str(val):
+    if not val:
+        return 'NULL'
+    s = str(val).strip().replace("'", "''")
+    return f"'{s}'"
+
+def main():
+    csv_path = 'data/iraq_checkpoints_70593.csv'
+    out_sql_path = 'infrastructure/sql/11_ingest_iraq_checkpoints.sql'
+
+    if not os.path.exists(csv_path):
+        print(f"Error: {csv_path} not found.")
+        sys.exit(1)
+
+    print(f"Reading {csv_path}...")
+
+    total_read = 0
+    valid_records = []
+
+    with open(csv_path, 'r', encoding='utf-8-sig') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            total_read += 1
+            try:
+                lat = float(row['latitude'])
+                lng = float(row['longitude'])
+                name = row.get('name', '').strip()
+                if not name:
+                    continue
+                # Valid bounding box for Iraq: 28.5 to 38.5 Lat, 38.5 to 49.0 Lng
+                if not (28.0 <= lat <= 39.0 and 38.0 <= lng <= 50.0):
+                    continue
+
+                category = row.get('category_queried', '').strip() or 'مكان عام'
+                sector = row.get('sector', '').strip()
+                gov = row.get('governorate', '').strip() or 'العراق'
+                area = row.get('area', '').strip()
+
+                address = f"{area}, {gov}, العراق" if area else f"{gov}, العراق"
+                desc = sector if sector else 'نقطة اهتمام في العراق'
+
+                # Compute popularity score from reviews and rating
+                reviews = 0
+                try:
+                    reviews = int(float(row.get('reviews_count', 0)))
+                except Exception:
+                    reviews = 0
+
+                rating = 0.0
+                try:
+                    rating = float(row.get('rating', 0))
+                except Exception:
+                    rating = 0.0
+
+                pop_score = min(100, int(reviews * 0.2 + rating * 5))
+
+                valid_records.append((name, lat, lng, category, gov, area, address, desc, pop_score))
+            except Exception:
+                continue
+
+    print(f"Total rows read: {total_read}, Valid for ingestion: {len(valid_records)}")
+
+    with open(out_sql_path, 'w', encoding='utf-8') as out:
+        out.write("-- ============================================================================\n")
+        out.write("-- MIGRATION 11: INGEST IRAQ CHECKPOINTS DATASET (70,000+ PLACES)\n")
+        out.write("-- Ingests into places_iraq and refreshes unified_search_index\n")
+        out.write("-- ============================================================================\n\n")
+
+        out.write("BEGIN;\n\n")
+        out.write("-- Clean previous checkpoint import if needed to avoid duplicates\n")
+        out.write("DELETE FROM places_iraq WHERE source = 'checkpoint_70593';\n\n")
+
+        batch_size = 1000
+        for i in range(0, len(valid_records), batch_size):
+            batch = valid_records[i:i + batch_size]
+            out.write("INSERT INTO places_iraq (\n")
+            out.write("    name, name_ar, latitude, longitude, category, city, neighbourhood,\n")
+            out.write("    address, description, popularity_score, source, location\n")
+            out.write(") VALUES\n")
+
+            value_lines = []
+            for item in batch:
+                name, lat, lng, category, gov, area, address, desc, pop_score = item
+                c_name = clean_str(name)
+                c_cat = clean_str(category)
+                c_city = clean_str(gov)
+                c_area = clean_str(area)
+                c_addr = clean_str(address)
+                c_desc = clean_str(desc)
+
+                line = (f"  ({c_name}, {c_name}, {lat:.7f}, {lng:.7f}, {c_cat}, {c_city}, {c_area}, "
+                        f"{c_addr}, {c_desc}, {pop_score}, 'checkpoint_70593', "
+                        f"ST_SetSRID(ST_MakePoint({lng:.7f}, {lat:.7f}), 4326))")
+                value_lines.append(line)
+
+            out.write(",\n".join(value_lines))
+            out.write(";\n\n")
+
+        out.write("COMMIT;\n\n")
+
+        out.write("-- Refresh Unified Search Index to instantly index all new Iraqi places\n")
+        out.write("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;\n")
+
+    print(f"✅ Generated SQL file: {out_sql_path} ({os.path.getsize(out_sql_path) / (1024*1024):.2f} MB)")
+
+if __name__ == '__main__':
+    main()