feat: add Jordan places migration scripts and enhance map controller with style-load safety and camera controls

This commit is contained in:
Hamza-Ayed
2026-09-23 20:56:34 +03:00
parent dc9a736e26
commit 8dbb336e0a
6 changed files with 35867 additions and 16 deletions
@@ -56,6 +56,7 @@ class NavigationCubit extends Cubit<NavigationState> {
Timer? _movementInterpolationTimer; Timer? _movementInterpolationTimer;
LatLng? _currentDisplayPosition; LatLng? _currentDisplayPosition;
double _currentDisplayHeading = 0.0; double _currentDisplayHeading = 0.0;
double _currentCameraBearing = 0.0;
LatLng? _animStartPosition; LatLng? _animStartPosition;
LatLng? _animTargetPosition; LatLng? _animTargetPosition;
double _animStartHeading = 0.0; double _animStartHeading = 0.0;
@@ -64,6 +65,13 @@ class NavigationCubit extends Cubit<NavigationState> {
static const int _animTotalSteps = 25; // 25 steps * 40ms = 1000ms duration static const int _animTotalSteps = 25; // 25 steps * 40ms = 1000ms duration
static const Duration _animTickDuration = Duration(milliseconds: 40); static const Duration _animTickDuration = Duration(milliseconds: 40);
void updateCameraBearing(double bearing) {
_currentCameraBearing = bearing;
if (_currentDisplayPosition != null) {
_updateCarMarker(_currentDisplayPosition!, _currentDisplayHeading);
}
}
Future<void> _init() async { Future<void> _init() async {
print("🚀 [NavigationCubit] Initializing NavigationCubit..."); print("🚀 [NavigationCubit] Initializing NavigationCubit...");
CarPlatformBridge.ensureInitialized(); CarPlatformBridge.ensureInitialized();
@@ -493,20 +501,48 @@ class NavigationCubit extends Cubit<NavigationState> {
void _onNewLocationFix( void _onNewLocationFix(
LatLng newLoc, double heading, double speedKmH, double alt) { LatLng newLoc, double heading, double speedKmH, double alt) {
double resolvedHeading = heading;
// Calculate heading from real movement if GPS course is missing / 0.0
if (resolvedHeading <= 0.0 || resolvedHeading.isNaN) {
if (_currentDisplayPosition != null) {
final dist = locationService.calculateDistance(_currentDisplayPosition!, newLoc);
if (dist >= 1.5) {
final computed = locationService.calculateBearing(_currentDisplayPosition!, newLoc);
resolvedHeading = (computed % 360.0 + 360.0) % 360.0;
} else {
resolvedHeading = _currentDisplayHeading;
}
} else {
resolvedHeading = _currentDisplayHeading;
}
}
// In active navigation, if stopped or crawling, lock heading to route segment ahead
if (state.isNavigating && speedKmH < 4.0 && state.currentRoute != null) {
final coords = state.currentRoute!.coordinates;
if (_lastTraveledIndexInFullRoute + 1 < coords.length) {
resolvedHeading = _calculateBearing(
coords[_lastTraveledIndexInFullRoute],
coords[_lastTraveledIndexInFullRoute + 1],
);
}
}
emit(state.copyWith( emit(state.copyWith(
myLocation: newLoc, myLocation: newLoc,
altitude: alt, altitude: alt,
heading: heading, heading: resolvedHeading,
speed: speedKmH, speed: speedKmH,
)); ));
// Smooth movement interpolation for the vehicle marker and 3D camera // Smooth movement interpolation for the vehicle marker and 3D camera
if (_currentDisplayPosition == null) { if (_currentDisplayPosition == null) {
_currentDisplayPosition = newLoc; _currentDisplayPosition = newLoc;
_currentDisplayHeading = heading; _currentDisplayHeading = resolvedHeading;
_updateCarMarker(newLoc, heading); _updateCarMarker(newLoc, resolvedHeading);
} else { } else {
_startMovementInterpolation(newLoc, heading); _startMovementInterpolation(newLoc, resolvedHeading);
} }
// Proactively move camera on first acquired GPS lock (only when style is loaded) // Proactively move camera on first acquired GPS lock (only when style is loaded)
@@ -514,9 +550,10 @@ class NavigationCubit extends Cubit<NavigationState> {
mapController != null && mapController != null &&
_isMapStyleLoaded) { _isMapStyleLoaded) {
_hasInitiallyCenteredCamera = true; _hasInitiallyCenteredCamera = true;
_currentCameraBearing = resolvedHeading;
mapController!.animateCamera( mapController!.animateCamera(
CameraUpdate.newCameraPosition( CameraUpdate.newCameraPosition(
CameraPosition(target: newLoc, zoom: 16.5, bearing: heading), CameraPosition(target: newLoc, zoom: 16.5, bearing: resolvedHeading),
), ),
); );
} }
@@ -527,11 +564,11 @@ class NavigationCubit extends Cubit<NavigationState> {
latitude: newLoc.latitude, latitude: newLoc.latitude,
longitude: newLoc.longitude, longitude: newLoc.longitude,
speedKmH: speedKmH, speedKmH: speedKmH,
heading: heading, heading: resolvedHeading,
elevation: alt, elevation: alt,
remainingDistance: state.remainingDistance, remainingDistance: state.remainingDistance,
); );
_processActiveNavigationTick(newLoc, speedKmH, heading); _processActiveNavigationTick(newLoc, speedKmH, resolvedHeading);
} }
} }
@@ -624,6 +661,8 @@ class NavigationCubit extends Cubit<NavigationState> {
} }
} }
_currentCameraBearing = effectiveBearing;
// 3. Shift camera target ahead along heading vector // 3. Shift camera target ahead along heading vector
// This positions the car in the bottom ~28% of the viewport with 45° tilt! // This positions the car in the bottom ~28% of the viewport with 45° tilt!
final cameraTarget = _computeOffset(pos, lookAheadMeters, effectiveBearing); final cameraTarget = _computeOffset(pos, lookAheadMeters, effectiveBearing);
@@ -672,13 +711,21 @@ class NavigationCubit extends Cubit<NavigationState> {
); );
} }
Future<void> _updateCarMarker(LatLng position, double bearing) async { Future<void> _updateCarMarker(LatLng position, double worldHeading) async {
if (mapController == null || !_isMapStyleLoaded) return; if (mapController == null || !_isMapStyleLoaded) return;
try { try {
// MapLibre symbols have viewport-aligned rotation.
// (worldHeading - _currentCameraBearing) keeps the vehicle oriented correctly:
// In active navigation with locked camera, (worldHeading - _currentCameraBearing) = 0° (facing UP/forward along the road).
// During turns, it tilts smoothly into the corner as the car turns.
// In free map exploration, it points towards true geographic direction relative to rotated map.
final double screenRotation =
(worldHeading - _currentCameraBearing + 360.0) % 360.0;
await mapController!.setUserMarker(Marker( await mapController!.setUserMarker(Marker(
markerId: const MarkerId('current_user_car'), markerId: const MarkerId('current_user_car'),
position: position, position: position,
rotation: bearing, rotation: screenRotation,
anchor: const Offset(0.5, 0.5), anchor: const Offset(0.5, 0.5),
flat: true, flat: true,
icon: InlqBitmap.fromStyleImage('current_vehicle_icon', icon: InlqBitmap.fromStyleImage('current_vehicle_icon',
@@ -709,6 +756,7 @@ class NavigationCubit extends Cubit<NavigationState> {
state.distanceToNextStep, state.distanceToNextStep,
); );
} else { } else {
_currentCameraBearing = state.heading;
mapController!.animateCamera( mapController!.animateCamera(
CameraUpdate.newCameraPosition( CameraUpdate.newCameraPosition(
CameraPosition( CameraPosition(
@@ -1705,6 +1705,7 @@ class _MapViewState extends State<MapView> {
zoomControlsEnabled: false, zoomControlsEnabled: false,
compassEnabled: false, compassEnabled: false,
onCameraMove: (pos) { onCameraMove: (pos) {
cubit.updateCameraBearing(pos.bearing);
if (state.isSelectingLocationOnMap) { if (state.isSelectingLocationOnMap) {
cubit.updatePickedLocation(pos.target); cubit.updatePickedLocation(pos.target);
} }
File diff suppressed because it is too large Load Diff
@@ -47,6 +47,7 @@ class IntaleqMapController {
/// [setUserMarker]. Reset to null on style reload (native symbol is /// [setUserMarker]. Reset to null on style reload (native symbol is
/// destroyed) so the next update re-adds it. /// destroyed) so the next update re-adds it.
mgl.Symbol? _userSymbol; mgl.Symbol? _userSymbol;
Marker? _pendingUserMarker;
bool _userSymbolBusy = false; bool _userSymbolBusy = false;
bool _isStyleLoaded = false; bool _isStyleLoaded = false;
CameraUpdate? _pendingCameraUpdate; CameraUpdate? _pendingCameraUpdate;
@@ -369,15 +370,21 @@ class IntaleqMapController {
/// dropped, and the next tick supplies fresh coordinates). The puck is /// dropped, and the next tick supplies fresh coordinates). The puck is
/// re-created automatically after a style reload. /// re-created automatically after a style reload.
Future<void> setUserMarker(Marker marker) async { Future<void> setUserMarker(Marker marker) async {
if (!_isStyleLoaded || _userSymbolBusy) return; if (!_isStyleLoaded) return;
_pendingUserMarker = marker;
if (_userSymbolBusy) return;
_userSymbolBusy = true; _userSymbolBusy = true;
try { try {
await _loadBitmapIfNeeded(marker.icon); while (_pendingUserMarker != null) {
final symbol = _userSymbol; final m = _pendingUserMarker!;
if (symbol == null) { _pendingUserMarker = null;
_userSymbol = await _raw.addSymbol(marker.toSymbolOptions()); await _loadBitmapIfNeeded(m.icon);
} else { final symbol = _userSymbol;
await _raw.updateSymbol(symbol, marker.toSymbolOptions()); if (symbol == null) {
_userSymbol = await _raw.addSymbol(m.toSymbolOptions());
} else {
await _raw.updateSymbol(symbol, m.toSymbolOptions());
}
} }
} catch (_) { } catch (_) {
} finally { } finally {
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""
Generate SQL migration file to ingest Jordan (Amman, Zarqa, Irbid) checkpoints CSV into places_jordan
and refresh the unified_search_index.
"""
import csv
import sys
import os
import math
from collections import defaultdict
def haversine(lat1, lon1, lat2, lon2):
R = 6371000
phi1, phi2 = math.radians(lat1), math.radians(lat2)
dphi = math.radians(lat2 - lat1)
dlam = math.radians(lon2 - lon1)
a = math.sin(dphi/2)**2 + math.cos(phi1)*math.cos(phi2)*math.sin(dlam/2)**2
return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
def normalize_text(t):
if not t: return ''
s = t.strip()
s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
return ' '.join(s.split()).lower()
def clean_str(val):
if not val:
return 'NULL'
s = str(val).strip().replace("'", "''")
return f"'{s}'"
def calculate_popularity(row):
reviews = 0
try:
reviews = int(float(row.get('reviews_count') or 0))
except Exception:
reviews = 0
rating = 0.0
try:
rating = float(row.get('rating') or 0)
except Exception:
rating = 0.0
score = int(reviews * 0.2 + rating * 5)
cat = row.get('category_queried', '')
if any(k in cat for k in ['مستشفى', 'جامعة', 'كلية', 'مركز صحي', 'طوارئ', 'دفاع مدني']):
score = max(score, 90)
elif any(k in cat for k in ['صيدلية', 'بنك', 'صراف آلي', 'سوبر ماركت', 'محطة']):
score = max(score, 60)
elif any(k in cat for k in ['مدرسة', 'روضة', 'عيادة', 'بلدية', 'دائرة']):
score = max(score, 50)
return min(100, max(10, score))
def main():
csv_path = 'data/checkpoint_35242_records.csv'
out_sql_path = 'infrastructure/sql/18_ingest_amman_zarqa_checkpoint.sql'
if not os.path.exists(csv_path):
print(f"Error: {csv_path} not found.")
sys.exit(1)
print(f"Reading {csv_path}...")
records = []
with open(csv_path, 'r', encoding='utf-8-sig') as f:
reader = csv.DictReader(f)
for row in reader:
name = row.get('name', '').strip()
if not name: continue
try:
lat = float(row['latitude'])
lng = float(row['longitude'])
except Exception:
continue
if not (29.15 <= lat <= 33.45 and 34.85 <= lng <= 39.35):
continue
category = row.get('category_queried', '').strip() or 'مكان عام'
sector = row.get('sector', '').strip()
gov = row.get('governorate', '').strip() or 'الأردن'
area = row.get('area', '').strip()
maps_url = row.get('maps_url', '').strip()
address = f"{area}, {gov}, الأردن" if area else f"{gov}, الأردن"
desc = sector if sector else 'نقطة اهتمام في الأردن'
pop_score = calculate_popularity(row)
records.append({
'name': name,
'norm_name': normalize_text(name),
'lat': lat,
'lng': lng,
'category': category,
'city': gov,
'neighbourhood': area,
'address': address,
'description': desc,
'popularity_score': pop_score,
'maps_url': maps_url
})
# Exact dedup
seen_exact = {}
seen_urls = {}
for r in records:
k = (r['norm_name'], round(r['lat'], 5), round(r['lng'], 5))
u = r['maps_url']
if k in seen_exact or (u and u in seen_urls):
continue
seen_exact[k] = r
if u: seen_urls[u] = r
step1 = list(seen_exact.values())
# Spatial near-duplicate dedup (<50m)
by_norm = defaultdict(list)
for r in step1:
by_norm[r['norm_name']].append(r)
final_records = []
for norm_name, items in by_norm.items():
if len(items) == 1:
final_records.append(items[0])
else:
items.sort(key=lambda x: x['popularity_score'], reverse=True)
kept = []
for candidate in items:
is_dup = False
for existing in kept:
if haversine(candidate['lat'], candidate['lng'], existing['lat'], existing['lng']) < 50:
is_dup = True
break
if not is_dup:
kept.append(candidate)
final_records.extend(kept)
print(f"Total valid unique records for SQL ingestion: {len(final_records)}")
source_tag = 'checkpoint_35242_amman_zarqa'
with open(out_sql_path, 'w', encoding='utf-8') as out:
out.write("-- ============================================================================\n")
out.write("-- MIGRATION 18: INGEST AMMAN & ZARQA CHECKPOINT DATASET (35,000+ PLACES)\n")
out.write("-- Ingests into places_jordan and refreshes unified_search_index\n")
out.write("-- ============================================================================\n\n")
out.write("BEGIN;\n\n")
out.write(f"-- Remove previous imports with same source to ensure idempotency\n")
out.write(f"DELETE FROM places_jordan WHERE source = '{source_tag}';\n\n")
out.write("ALTER TABLE places_jordan DISABLE TRIGGER trg_sync_place_location_jordan;\n\n")
batch_size = 1000
for i in range(0, len(final_records), batch_size):
batch = final_records[i:i + batch_size]
out.write("INSERT INTO places_jordan (\n")
out.write(" name, name_ar, latitude, longitude, category, city, neighbourhood,\n")
out.write(" address, description, popularity_score, source, location\n")
out.write(") VALUES\n")
value_lines = []
for r in batch:
c_name = clean_str(r['name'])
c_cat = clean_str(r['category'])
c_city = clean_str(r['city'])
c_area = clean_str(r['neighbourhood'])
c_addr = clean_str(r['address'])
c_desc = clean_str(r['description'])
lat = r['lat']
lng = r['lng']
pop = r['popularity_score']
line = (f" ({c_name}, {c_name}, {lat:.7f}, {lng:.7f}, {c_cat}, {c_city}, {c_area}, "
f"{c_addr}, {c_desc}, {pop}, '{source_tag}', "
f"ST_SetSRID(ST_MakePoint({lng:.7f}, {lat:.7f}), 4326))")
value_lines.append(line)
out.write(",\n".join(value_lines))
out.write(";\n\n")
out.write("ALTER TABLE places_jordan ENABLE TRIGGER trg_sync_place_location_jordan;\n\n")
out.write(f"UPDATE places_jordan SET governorate_id = 137 WHERE source = '{source_tag}' AND city = 'عمان';\n")
out.write(f"UPDATE places_jordan SET governorate_id = 5 WHERE source = '{source_tag}' AND city = 'الزرقاء';\n")
out.write(f"UPDATE places_jordan SET governorate_id = 184 WHERE source = '{source_tag}' AND city = 'إربد';\n\n")
out.write("COMMIT;\n\n")
out.write("-- Refresh Unified Search Index to instantly index all new Jordanian places\n")
out.write("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;\n\n")
out.write("ANALYZE places_jordan;\n")
print(f"✅ Generated SQL file: {out_sql_path} ({os.path.getsize(out_sql_path) / (1024*1024):.2f} MB)")
if __name__ == '__main__':
main()
+358
View File
@@ -0,0 +1,358 @@
#!/usr/bin/env python3
"""
Migrate and deduplicate Jordan (Amman, Zarqa, Irbid) Places Dataset into places_jordan table on PostgreSQL.
Handles Arabic normalization, spatial bounding box filtering (Jordan bounds),
exact deduplication, spatial near-duplicate filtering (< 50m), popularity scoring,
and concurrent refresh of unified_search_index.
"""
import os
import sys
import csv
import gzip
import math
import time
import argparse
from collections import defaultdict, Counter
def haversine(lat1, lon1, lat2, lon2):
R = 6371000 # meters
phi1 = math.radians(lat1)
phi2 = math.radians(lat2)
delta_phi = math.radians(lat2 - lat1)
delta_lambda = math.radians(lon2 - lon1)
a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
def normalize_text(t):
if not t:
return ''
s = t.strip()
# Normalize Arabic alef, teh marbuta, etc.
s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
return ' '.join(s.split()).lower()
def clean_sql_str(val):
if val is None:
return 'NULL'
s = str(val).strip().replace("'", "''")
return f"'{s}'"
def calculate_popularity(row):
# Reviews and rating
reviews = 0
try:
reviews = int(float(row.get('reviews_count') or 0))
except Exception:
reviews = 0
rating = 0.0
try:
rating = float(row.get('rating') or 0)
except Exception:
rating = 0.0
score = int(reviews * 0.2 + rating * 5)
# Heuristic boost for critical POIs
cat = row.get('category_queried', '')
if any(k in cat for k in ['مستشفى', 'جامعة', 'كلية', 'مركز صحي', 'طوارئ', 'دفاع مدني']):
score = max(score, 90)
elif any(k in cat for k in ['صيدلية', 'بنك', 'صراف آلي', 'سوبر ماركت', 'محطة']):
score = max(score, 60)
elif any(k in cat for k in ['مدرسة', 'روضة', 'عيادة', 'بلدية', 'دائرة']):
score = max(score, 50)
return min(100, max(10, score))
def main():
parser = argparse.ArgumentParser(description="Migrate Jordan places dataset with deduplication.")
parser.add_argument("--file", default="", help="Path to CSV file.")
parser.add_argument("--db-host", default="127.0.0.1", help="Database host")
parser.add_argument("--db-port", type=int, default=5432, help="Database port")
parser.add_argument("--db-user", default="mapuser", help="Database user")
parser.add_argument("--db-pass", default="mappass", help="Database password")
parser.add_argument("--db-name", default="mapdb", help="Database name")
parser.add_argument("--dry-run", action="store_true", help="Perform validation and deduplication only without inserting.")
args = parser.parse_args()
# Find file
file_path = args.file
if not file_path:
candidates = [
"data/checkpoint_35242_records.csv",
"/home/hamzadoctor/app/data/checkpoint_35242_records.csv",
"checkpoint_35242_records.csv"
]
for c in candidates:
if os.path.exists(c):
file_path = c
break
if not file_path or not os.path.exists(file_path):
print(f"❌ Error: Dataset file not found: {file_path}")
sys.exit(1)
print(f"📂 Processing dataset: {file_path}")
open_fn = gzip.open if file_path.endswith('.gz') else open
mode = 'rt' if file_path.endswith('.gz') else 'r'
total_read = 0
empty_name = 0
out_of_bounds = 0
valid_records = []
gov_counter = Counter()
t0 = time.time()
with open_fn(file_path, mode, encoding='utf-8-sig') as f:
reader = csv.DictReader(f)
for row in reader:
total_read += 1
name = row.get('name', '').strip()
if not name:
empty_name += 1
continue
try:
lat = float(row['latitude'])
lng = float(row['longitude'])
except Exception:
out_of_bounds += 1
continue
# Jordan bounding box check (29.15 to 33.45 Lat, 34.85 to 39.35 Lng)
if not (29.15 <= lat <= 33.45 and 34.85 <= lng <= 39.35):
out_of_bounds += 1
continue
category = row.get('category_queried', '').strip() or 'مكان عام'
sector = row.get('sector', '').strip()
gov = row.get('governorate', '').strip() or 'الأردن'
area = row.get('area', '').strip()
maps_url = row.get('maps_url', '').strip()
address = f"{area}, {gov}, الأردن" if area else f"{gov}, الأردن"
desc = sector if sector else 'نقطة اهتمام في الأردن'
pop_score = calculate_popularity(row)
gov_counter[gov] += 1
valid_records.append({
'name': name,
'norm_name': normalize_text(name),
'lat': lat,
'lng': lng,
'category': category,
'city': gov,
'neighbourhood': area,
'address': address,
'description': desc,
'popularity_score': pop_score,
'maps_url': maps_url,
'sector': sector
})
print(f"📊 Initial parse complete in {time.time() - t0:.2f}s:")
print(f" - Total rows in CSV: {total_read:,}")
print(f" - Out of Jordan bounds / invalid coords: {out_of_bounds:,}")
print(f" - Valid in Jordan bounds: {len(valid_records):,}")
print(f" - Governorate distribution: {dict(gov_counter)}")
# Deduplication Step 1: Exact (norm_name, round(lat, 5), round(lng, 5)) and maps_url
seen_exact = {}
seen_urls = {}
exact_dups = 0
for r in valid_records:
k = (r['norm_name'], round(r['lat'], 5), round(r['lng'], 5))
url = r['maps_url']
if k in seen_exact:
exact_dups += 1
if r['popularity_score'] > seen_exact[k]['popularity_score']:
seen_exact[k] = r
continue
if url and url in seen_urls:
exact_dups += 1
if r['popularity_score'] > seen_urls[url]['popularity_score']:
seen_urls[url] = r
continue
seen_exact[k] = r
if url:
seen_urls[url] = r
dedup_step1 = list(seen_exact.values())
print(f"🔍 Step 1 Deduplication (Exact coords / URL):")
print(f" - Removed {exact_dups} duplicate records.")
print(f" - Remaining: {len(dedup_step1):,}")
# Deduplication Step 2: Spatial near-duplicates (< 50m with identical normalized name)
by_norm_name = defaultdict(list)
for r in dedup_step1:
by_norm_name[r['norm_name']].append(r)
final_records = []
near_dups_filtered = 0
for norm_name, items in by_norm_name.items():
if len(items) == 1:
final_records.append(items[0])
else:
items.sort(key=lambda x: x['popularity_score'], reverse=True)
kept = []
for candidate in items:
is_dup = False
for existing in kept:
d = haversine(candidate['lat'], candidate['lng'], existing['lat'], existing['lng'])
if d < 50:
is_dup = True
near_dups_filtered += 1
break
if not is_dup:
kept.append(candidate)
final_records.extend(kept)
print(f"🎯 Step 2 Deduplication (Spatial near-duplicates < 50m):")
print(f" - Filtered out {near_dups_filtered} near-duplicates.")
print(f" - Final unique clean places to migrate: {len(final_records):,}")
final_govs = Counter([r['city'] for r in final_records])
print(f" - Final Governorates: {dict(final_govs)}")
if args.dry_run:
print("💡 Dry run complete. No database changes made.")
return
# Database Migration
import pg8000.native
print(f"\n🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
con = pg8000.native.Connection(
user=args.db_user,
password=args.db_pass,
host=args.db_host,
port=args.db_port,
database=args.db_name
)
t_db = time.time()
source_tag = 'checkpoint_35242_amman_zarqa'
print(f"🗑️ Removing previous '{source_tag}' imports from places_jordan...")
con.run("BEGIN;")
con.run(f"DELETE FROM places_jordan WHERE source = '{source_tag}';")
# Temporarily disable trigger for high-speed bulk ingestion
print("⚡ Disabling location trigger for high-speed ingestion...")
con.run("ALTER TABLE places_jordan DISABLE TRIGGER trg_sync_place_location_jordan;")
batch_size = 2000
total_inserted = 0
print(f"🚀 Inserting {len(final_records):,} places in batches of {batch_size}...")
for i in range(0, len(final_records), batch_size):
batch = final_records[i:i + batch_size]
values = []
for r in batch:
c_name = clean_sql_str(r['name'])
c_cat = clean_sql_str(r['category'])
c_city = clean_sql_str(r['city'])
c_area = clean_sql_str(r['neighbourhood'])
c_addr = clean_sql_str(r['address'])
c_desc = clean_sql_str(r['description'])
lat = r['lat']
lng = r['lng']
pop = r['popularity_score']
line = (
f"({c_name}, {c_name}, {lat:.7f}, {lng:.7f}, {c_cat}, {c_city}, {c_area}, "
f"{c_addr}, {c_desc}, {pop}, '{source_tag}', "
f"ST_SetSRID(ST_MakePoint({lng:.7f}, {lat:.7f}), 4326))"
)
values.append(line)
sql = (
"INSERT INTO places_jordan ("
" name, name_ar, latitude, longitude, category, city, neighbourhood,"
" address, description, popularity_score, source, location"
") VALUES " + ",\n".join(values) + ";"
)
con.run(sql)
total_inserted += len(batch)
print(f" -> Inserted {total_inserted:,} / {len(final_records):,} places...")
# Re-enable trigger
print("⚡ Re-enabling location trigger...")
con.run("ALTER TABLE places_jordan ENABLE TRIGGER trg_sync_place_location_jordan;")
# Batch update governorate_id for Amman and Zarqa based on city name
print("🏛️ Updating administrative links for new places...")
con.run(f"""
UPDATE places_jordan
SET governorate_id = 137
WHERE source = '{source_tag}' AND city = 'عمان';
""")
con.run(f"""
UPDATE places_jordan
SET governorate_id = 5
WHERE source = '{source_tag}' AND city = 'الزرقاء';
""")
con.run(f"""
UPDATE places_jordan
SET governorate_id = 184
WHERE source = '{source_tag}' AND city = 'إربد';
""")
con.run("COMMIT;")
print(f"✅ Ingestion committed successfully in {time.time() - t_db:.2f}s!")
# Refresh materialized view
print("🔄 Refreshing materialized view unified_search_index concurrently...")
t_mv = time.time()
con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
print(f"✅ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")
# Analyze table
print("⚡ Running ANALYZE on places_jordan...")
con.run("ANALYZE places_jordan;")
# Verification
print("\n🔍 Verification & Statistics:")
count_res = con.run("SELECT count(*) FROM places_jordan;")[0][0]
new_count = con.run(f"SELECT count(*) FROM places_jordan WHERE source = '{source_tag}';")[0][0]
gov_stats = con.run(f"""
SELECT city, count(*)
FROM places_jordan
WHERE source = '{source_tag}'
GROUP BY city
ORDER BY count(*) DESC;
""")
print(f" - Total places in places_jordan: {count_res:,}")
print(f" - Newly added from Amman & Zarqa checkpoint: {new_count:,}")
print(" - Governorates breakdown:")
for gov, cnt in gov_stats:
print(f" * {gov}: {cnt:,} places")
# Sample Geocoding search test
test_queries = ['صيدلية في عمان', 'الجبيهة', 'الزرقاء الجديدة', 'وسط البلد عمان', 'الشميساني']
print("\n🧪 Testing search on unified_search_index:")
for q in test_queries:
results = con.run(f"""
SELECT name, category, city, latitude, longitude
FROM places_jordan
WHERE (name ILIKE '%{q}%' OR neighbourhood ILIKE '%{q}%' OR address ILIKE '%{q}%')
AND source = '{source_tag}'
LIMIT 2;
""")
if results:
first = results[0]
print(f" ✓ Query '{q}': Found '{first[0]}' ({first[1]} - {first[2]}) @ {first[3]}, {first[4]}")
else:
print(f" - Query '{q}': No direct checkpoint match, checking overall...")
con.close()
print("\n🎉 Jordan dataset migration completed successfully!")
if __name__ == '__main__':
main()