524 lines
19 KiB
Python
524 lines
19 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Comprehensive Iraq Places Enrichment & Deduplication Script.
|
||
Enriches `places_iraq` table by harvesting and deduplicating:
|
||
1. Overture Maps Places (`overture_place`)
|
||
2. OpenStreetMap Named Points (`planet_osm_point`)
|
||
3. OpenStreetMap Named Polygons (`planet_osm_polygon`)
|
||
4. Administrative boundaries & Governorates linking
|
||
5. Concurrently refreshes `unified_search_index`
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import math
|
||
import time
|
||
import argparse
|
||
from collections import defaultdict
|
||
|
||
def haversine(lat1, lon1, lat2, lon2):
|
||
R = 6371000 # meters
|
||
phi1 = math.radians(lat1)
|
||
phi2 = math.radians(lat2)
|
||
delta_phi = math.radians(lat2 - lat1)
|
||
delta_lambda = math.radians(lon2 - lon1)
|
||
a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
|
||
return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
|
||
|
||
def normalize_text(t):
|
||
if not t:
|
||
return ''
|
||
s = t.strip()
|
||
s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
|
||
return ' '.join(s.split()).lower()
|
||
|
||
def clean_sql_str(val):
|
||
if val is None:
|
||
return 'NULL'
|
||
s = str(val).strip().replace("'", "''")
|
||
return f"'{s}'"
|
||
|
||
# Category Translations
|
||
OVERTURE_CAT_MAP = {
|
||
'restaurant': 'مطعم',
|
||
'cafe': 'مقهى وكافيه',
|
||
'coffee_shop': 'مقهى وكوفي شوب',
|
||
'casual_eatery': 'وجبات سريعة ومأكولات خفيفة',
|
||
'bakery': 'مخبز ومعجنات',
|
||
'food_and_beverage_store': 'بقالة ومواد غذائية',
|
||
'supermarket': 'سوبرماركت وهايبرماركت',
|
||
'fashion_and_apparel_store': 'متجر ألبسة وأزياء',
|
||
'shoe_store': 'متجر أحذية',
|
||
'jewelry_store': 'مجوهرات وحلي',
|
||
'electronics_store': 'متجر إلكترونيات وأجهزة',
|
||
'mobile_phone_store': 'متجر هواتف ذكية',
|
||
'hardware_home_and_garden_store': 'متجر مواد بناء ومستلزمات منزلية',
|
||
'home_service': 'خدمات منزلية وصيانة',
|
||
'personal_or_beauty_service': 'صالون ومركز تجميل',
|
||
'hair_salon': 'صالون حلاقة وتزيين',
|
||
'hospital': 'مستشفى / مركز طبي',
|
||
'dental_clinic': 'عيادة طب أسنان',
|
||
'specialized_health_care': 'عيادة استشارية متخصصة',
|
||
'pharmacy': 'صيدلية',
|
||
'hotel': 'فندق وإقامة',
|
||
'place_of_learning': 'مؤسسة تعليمية',
|
||
'school': 'مدرسة',
|
||
'college_university': 'كلية / جامعة',
|
||
'historic_site': 'موقع تاريخي وأثري',
|
||
'religious_organization': 'دار عبادة ومؤسسة دينية',
|
||
'mosque': 'مسجد / جامع',
|
||
'real_estate_service': 'مكتب عقاري',
|
||
'professional_service': 'خدمات مهنية واستشارية',
|
||
'travel_service': 'مكتب سياحة وسفر',
|
||
'corporate_or_business_office': 'مكتب شركة ومؤسسة أعمال',
|
||
'car_repair': 'صيانة وتصليح سيارات',
|
||
'car_dealership': 'معرض ووكالة سيارات',
|
||
'fuel_station': 'محطة وقود',
|
||
'bank': 'مصرف / بنك',
|
||
'government_office': 'دائرة حكومية ورسمية',
|
||
'law_firm': 'مكتب محاماة واستشارات قانونية',
|
||
'sports_club': 'نادي ومجمع رياضي',
|
||
'park': 'منتزه وحديقة عامة',
|
||
}
|
||
|
||
OSM_CAT_MAP = {
|
||
'village': 'قرية / تجمع سكاني',
|
||
'town': 'بلدة / قضاء',
|
||
'city': 'مدينة',
|
||
'suburb': 'حي / ضاحية',
|
||
'neighbourhood': 'حي سكني',
|
||
'hamlet': 'قرية صغيرة',
|
||
'locality': 'منطقة / موضع',
|
||
'school': 'مدرسة / تعليم',
|
||
'college': 'كلية',
|
||
'university': 'جامعة',
|
||
'kindergarten': 'روضة أطفال',
|
||
'supermarket': 'سوبرماركت ومواد غذائية',
|
||
'convenience': 'محل بقالة',
|
||
'bakery': 'مخبز',
|
||
'butcher': 'ملحمة وقصابة',
|
||
'mall': 'مركز تسوق ومول',
|
||
'place_of_worship': 'مسجد / دار عبادة',
|
||
'mosque': 'مسجد / جامع',
|
||
'clinic': 'مركز صحي وعيادة',
|
||
'pharmacy': 'صيدلية',
|
||
'hospital': 'مستشفى',
|
||
'doctors': 'عيادة طبيب',
|
||
'dentist': 'عيادة أسنان',
|
||
'restaurant': 'مطعم',
|
||
'cafe': 'مقهى وكافيه',
|
||
'fast_food': 'وجبات سريعة',
|
||
'fuel': 'محطة وقود',
|
||
'bank': 'مصرف / بنك',
|
||
'atm': 'صراف آلي',
|
||
'bureau_de_change': 'مكتب صرافة وتحويل مالي',
|
||
'police': 'مركز شرطة وأمن',
|
||
'fire_station': 'دفاع مدني / إطفاء',
|
||
'post_office': 'مكتب بريد',
|
||
'attraction': 'معلم سياحي',
|
||
'hotel': 'فندق',
|
||
'guest_house': 'نُزل واستضافة',
|
||
'museum': 'متحف / تراث',
|
||
'park': 'حديقة ومنتزه',
|
||
'stadium': 'ملعب / مجمع رياضي',
|
||
'sports_centre': 'مركز رياضي',
|
||
'car_repair': 'تصليح سيارات',
|
||
'car_wash': 'مغسلة سيارات',
|
||
'company': 'شركة ومؤسسة',
|
||
'government': 'دائرة حكومية',
|
||
'mobile_phone': 'متجر هواتف ونقالات',
|
||
'clothes': 'متجر ألبسة',
|
||
'shoes': 'متجر أحذية',
|
||
'hairdresser': 'صالون حلاقة',
|
||
'optician': 'بصريات ونظارات',
|
||
'hardware': 'مواد بناء وتجهيزات',
|
||
}
|
||
|
||
def map_overture_category(cat_str):
|
||
if not cat_str:
|
||
return 'مكان ونقطة اهتمام'
|
||
clean = cat_str.strip().lower()
|
||
return OVERTURE_CAT_MAP.get(clean, 'مكان عام وتجاري')
|
||
|
||
def map_osm_category(val):
|
||
if not val:
|
||
return 'نقطة اهتمام'
|
||
clean = str(val).strip().lower()
|
||
return OSM_CAT_MAP.get(clean, 'مكان عام')
|
||
|
||
def get_grid_cell(lat, lon, cell_size=0.01):
|
||
# ~1.1km cell size
|
||
return (int(lat / cell_size), int(lon / cell_size))
|
||
|
||
def is_duplicate(name_norm, lat, lon, grid, max_dist_m=60.0):
|
||
# 60 meters approx threshold in degrees
|
||
d_lat_thresh = 0.0006
|
||
d_lon_thresh = 0.0007
|
||
c_x, c_y = get_grid_cell(lat, lon)
|
||
for dx in (-1, 0, 1):
|
||
for dy in (-1, 0, 1):
|
||
cell = (c_x + dx, c_y + dy)
|
||
if cell in grid:
|
||
for existing_name, ex_lat, ex_lon in grid[cell]:
|
||
if abs(lat - ex_lat) > d_lat_thresh or abs(lon - ex_lon) > d_lon_thresh:
|
||
continue
|
||
dist = haversine(lat, lon, ex_lat, ex_lon)
|
||
if dist <= max_dist_m:
|
||
# Check name similarity
|
||
if (name_norm == existing_name or
|
||
name_norm in existing_name or
|
||
existing_name in name_norm):
|
||
return True
|
||
return False
|
||
|
||
def add_to_grid(name_norm, lat, lon, grid):
|
||
cell = get_grid_cell(lat, lon)
|
||
grid[cell].append((name_norm, lat, lon))
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Enrich Iraq places comprehensively from Overture & OSM.")
|
||
parser.add_argument("--db-host", default="127.0.0.1", help="Database host")
|
||
parser.add_argument("--db-port", type=int, default=5432, help="Database port")
|
||
parser.add_argument("--db-user", default="mapuser", help="Database user")
|
||
parser.add_argument("--db-pass", default="mappass", help="Database password")
|
||
parser.add_argument("--db-name", default="mapdb", help="Database name")
|
||
parser.add_argument("--dry-run", action="store_true", help="Perform extraction and deduplication only without inserting.")
|
||
args = parser.parse_args()
|
||
|
||
import pg8000.native
|
||
|
||
print("=" * 70)
|
||
print("🇮🇶 SIRO Maps — Comprehensive Iraq Places Enrichment & Deduplication")
|
||
print("=" * 70)
|
||
|
||
print(f"🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
|
||
con = pg8000.native.Connection(
|
||
user=args.db_user,
|
||
password=args.db_pass,
|
||
host=args.db_host,
|
||
port=args.db_port,
|
||
database=args.db_name
|
||
)
|
||
|
||
t0 = time.time()
|
||
grid = defaultdict(list)
|
||
|
||
# 1. Load existing places into spatial hash grid
|
||
print("\n📦 Step 1: Loading existing places_iraq into spatial index for deduplication...")
|
||
existing = con.run("""
|
||
SELECT id, name, latitude, longitude
|
||
FROM places_iraq
|
||
WHERE latitude IS NOT NULL AND longitude IS NOT NULL;
|
||
""")
|
||
print(f" -> Loaded {len(existing):,} existing places.")
|
||
for row in existing:
|
||
pid, name, lat, lon = row
|
||
lat = float(lat)
|
||
lon = float(lon)
|
||
norm_name = normalize_text(name)
|
||
add_to_grid(norm_name, lat, lon, grid)
|
||
|
||
print(f" ✓ Spatial grid built with {sum(len(v) for v in grid.values()):,} items.")
|
||
|
||
# 2. Harvest from overture_place
|
||
print("\n🗺️ Step 2: Harvesting from Overture Maps Places (overture_place)...")
|
||
overture_rows = con.run("""
|
||
SELECT
|
||
name_primary,
|
||
basic_category,
|
||
addresses,
|
||
confidence,
|
||
ST_Y(ST_Centroid(location::geometry)) as lat,
|
||
ST_X(ST_Centroid(location::geometry)) as lon
|
||
FROM overture_place
|
||
WHERE name_primary IS NOT NULL
|
||
AND TRIM(name_primary) != ''
|
||
AND ST_Within(location::geometry, ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326));
|
||
""")
|
||
print(f" -> Found {len(overture_rows):,} candidates in overture_place for Iraq.")
|
||
|
||
overture_to_insert = []
|
||
overture_dups = 0
|
||
|
||
for r in overture_rows:
|
||
name = str(r[0]).strip()
|
||
cat_str = r[1]
|
||
addr_list = r[2]
|
||
confidence = float(r[3] or 0.8)
|
||
lat = float(r[4])
|
||
lon = float(r[5])
|
||
|
||
norm_name = normalize_text(name)
|
||
if len(norm_name) < 2:
|
||
continue
|
||
|
||
if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
|
||
overture_dups += 1
|
||
continue
|
||
|
||
# Extract address details if available
|
||
city = 'العراق'
|
||
addr_text = None
|
||
if addr_list and isinstance(addr_list, list) and len(addr_list) > 0:
|
||
a = addr_list[0]
|
||
if isinstance(a, dict):
|
||
city = a.get('locality') or a.get('region') or 'العراق'
|
||
addr_text = a.get('freeform')
|
||
|
||
category = map_overture_category(cat_str)
|
||
pop_score = min(100, int(confidence * 60 + 20))
|
||
|
||
item = {
|
||
'name': name,
|
||
'name_ar': name,
|
||
'name_en': name,
|
||
'category': category,
|
||
'city': city,
|
||
'address': addr_text or f"{city}, العراق",
|
||
'lat': lat,
|
||
'lon': lon,
|
||
'source': 'overture_maps',
|
||
'popularity_score': pop_score
|
||
}
|
||
overture_to_insert.append(item)
|
||
add_to_grid(norm_name, lat, lon, grid)
|
||
|
||
print(f" ✓ Overture Processing Complete:")
|
||
print(f" - Duplicates Filtered: {overture_dups:,}")
|
||
print(f" - Net New Places: {len(overture_to_insert):,}")
|
||
|
||
# 3. Harvest from planet_osm_point
|
||
print("\n📍 Step 3: Harvesting from OpenStreetMap Named Points (planet_osm_point)...")
|
||
osm_point_rows = con.run("""
|
||
SELECT
|
||
name,
|
||
COALESCE(amenity, shop, tourism, historic, place, leisure, office, building, highway, "natural") as cat,
|
||
ST_Y(ST_Transform(way, 4326)) as lat,
|
||
ST_X(ST_Transform(way, 4326)) as lon
|
||
FROM planet_osm_point
|
||
WHERE name IS NOT NULL
|
||
AND TRIM(name) != ''
|
||
AND way && ST_Transform(ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326), 3857);
|
||
""")
|
||
print(f" -> Found {len(osm_point_rows):,} candidates in planet_osm_point for Iraq.")
|
||
|
||
osm_point_to_insert = []
|
||
osm_point_dups = 0
|
||
|
||
for r in osm_point_rows:
|
||
name = str(r[0]).strip()
|
||
cat_raw = r[1]
|
||
lat = float(r[2])
|
||
lon = float(r[3])
|
||
|
||
# Bounds check
|
||
if not (28.8 <= lat <= 37.5 and 38.8 <= lon <= 48.8):
|
||
continue
|
||
|
||
norm_name = normalize_text(name)
|
||
if len(norm_name) < 2:
|
||
continue
|
||
|
||
if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
|
||
osm_point_dups += 1
|
||
continue
|
||
|
||
category = map_osm_category(cat_raw)
|
||
pop_score = 65 if cat_raw in ('village', 'town', 'city', 'hospital', 'university', 'mosque') else 40
|
||
|
||
item = {
|
||
'name': name,
|
||
'name_ar': name,
|
||
'name_en': name,
|
||
'category': category,
|
||
'city': 'العراق',
|
||
'address': f"{category}, العراق",
|
||
'lat': lat,
|
||
'lon': lon,
|
||
'source': 'osm_point',
|
||
'popularity_score': pop_score
|
||
}
|
||
osm_point_to_insert.append(item)
|
||
add_to_grid(norm_name, lat, lon, grid)
|
||
|
||
print(f" ✓ OSM Points Processing Complete:")
|
||
print(f" - Duplicates Filtered: {osm_point_dups:,}")
|
||
print(f" - Net New Places: {len(osm_point_to_insert):,}")
|
||
|
||
# 4. Harvest from planet_osm_polygon (POIs)
|
||
print("\n🏛️ Step 4: Harvesting from OpenStreetMap Named Polygons (planet_osm_polygon)...")
|
||
osm_poly_rows = con.run("""
|
||
SELECT
|
||
name,
|
||
COALESCE(amenity, shop, tourism, historic, place, leisure, building, landuse) as cat,
|
||
ST_Y(ST_Centroid(ST_Transform(way, 4326))) as lat,
|
||
ST_X(ST_Centroid(ST_Transform(way, 4326))) as lon
|
||
FROM planet_osm_polygon
|
||
WHERE name IS NOT NULL
|
||
AND TRIM(name) != ''
|
||
AND (amenity IS NOT NULL OR shop IS NOT NULL OR tourism IS NOT NULL OR historic IS NOT NULL
|
||
OR place IS NOT NULL OR leisure IS NOT NULL OR landuse IN ('commercial', 'retail', 'industrial', 'cemetery', 'religious'))
|
||
AND way && ST_Transform(ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326), 3857);
|
||
""")
|
||
print(f" -> Found {len(osm_poly_rows):,} candidates in planet_osm_polygon for Iraq.")
|
||
|
||
osm_poly_to_insert = []
|
||
osm_poly_dups = 0
|
||
|
||
for r in osm_poly_rows:
|
||
name = str(r[0]).strip()
|
||
cat_raw = r[1]
|
||
lat = float(r[2])
|
||
lon = float(r[3])
|
||
|
||
if not (28.8 <= lat <= 37.5 and 38.8 <= lon <= 48.8):
|
||
continue
|
||
|
||
norm_name = normalize_text(name)
|
||
if len(norm_name) < 2:
|
||
continue
|
||
|
||
if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
|
||
osm_poly_dups += 1
|
||
continue
|
||
|
||
category = map_osm_category(cat_raw)
|
||
pop_score = 70 if cat_raw in ('hospital', 'university', 'mall', 'stadium', 'attraction') else 45
|
||
|
||
item = {
|
||
'name': name,
|
||
'name_ar': name,
|
||
'name_en': name,
|
||
'category': category,
|
||
'city': 'العراق',
|
||
'address': f"{category}, العراق",
|
||
'lat': lat,
|
||
'lon': lon,
|
||
'source': 'osm_polygon',
|
||
'popularity_score': pop_score
|
||
}
|
||
osm_poly_to_insert.append(item)
|
||
add_to_grid(norm_name, lat, lon, grid)
|
||
|
||
print(f" ✓ OSM Polygons Processing Complete:")
|
||
print(f" - Duplicates Filtered: {osm_poly_dups:,}")
|
||
print(f" - Net New Places: {len(osm_poly_to_insert):,}")
|
||
|
||
# Total new places to insert
|
||
all_new_places = overture_to_insert + osm_point_to_insert + osm_poly_to_insert
|
||
print(f"\n🌟 Total Net New Places to Insert: {len(all_new_places):,}")
|
||
|
||
if args.dry_run:
|
||
print("💡 Dry run requested. Exiting without database insertion.")
|
||
return
|
||
|
||
# 5. Database Batch Insertion
|
||
print(f"\n🚀 Step 5: Inserting {len(all_new_places):,} places into places_iraq...")
|
||
t_insert = time.time()
|
||
batch_size = 2500
|
||
total_inserted = 0
|
||
|
||
con.run("BEGIN;")
|
||
for i in range(0, len(all_new_places), batch_size):
|
||
batch = all_new_places[i:i + batch_size]
|
||
values = []
|
||
for r in batch:
|
||
c_name = clean_sql_str(r['name'])
|
||
c_name_ar = clean_sql_str(r['name_ar'])
|
||
c_name_en = clean_sql_str(r['name_en'])
|
||
c_cat = clean_sql_str(r['category'])
|
||
c_city = clean_sql_str(r['city'])
|
||
c_addr = clean_sql_str(r['address'])
|
||
c_src = clean_sql_str(r['source'])
|
||
lat = r['lat']
|
||
lon = r['lon']
|
||
pop = r['popularity_score']
|
||
|
||
line = (
|
||
f"({c_name}, {c_name_ar}, {c_name_en}, {lat:.7f}, {lon:.7f}, {c_cat}, "
|
||
f"{c_city}, {c_addr}, {c_src}, {pop}, "
|
||
f"ST_SetSRID(ST_MakePoint({lon:.7f}, {lat:.7f}), 4326))"
|
||
)
|
||
values.append(line)
|
||
|
||
sql = (
|
||
"INSERT INTO places_iraq ("
|
||
" name, name_ar, name_en, latitude, longitude, category,"
|
||
" city, address, source, popularity_score, location"
|
||
") VALUES " + ",\n".join(values) + ";"
|
||
)
|
||
con.run(sql)
|
||
total_inserted += len(batch)
|
||
print(f" -> Progress: {total_inserted:,} / {len(all_new_places):,} places inserted...")
|
||
|
||
con.run("COMMIT;")
|
||
print(f"✅ Ingestion committed successfully in {time.time() - t_insert:.2f}s!")
|
||
|
||
# 6. Link Administrative Hierarchy (Governorates & Districts)
|
||
print("\n🏛️ Step 6: Linking administrative hierarchy (Governorates & Districts)...")
|
||
t_admin = time.time()
|
||
con.run("""
|
||
UPDATE places_iraq p
|
||
SET
|
||
governorate_id = ab.id,
|
||
city = COALESCE(ab.name_ar, ab.name, p.city)
|
||
FROM admin_boundaries ab
|
||
WHERE ab.country_code = 'IQ'
|
||
AND ab.admin_level = 4
|
||
AND (p.governorate_id IS NULL OR p.city = 'العراق')
|
||
AND ST_Within(p.location::geometry, ab.geom::geometry);
|
||
""")
|
||
print(f" ✓ Linked governorates in {time.time() - t_admin:.2f}s.")
|
||
|
||
# 7. Refresh Materialized View
|
||
print("\n🔄 Step 7: Refreshing materialized view unified_search_index concurrently...")
|
||
t_mv = time.time()
|
||
try:
|
||
con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
|
||
print(f" ✓ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")
|
||
except Exception as e:
|
||
print(f" ⚠️ Concurrent refresh notice: {e}, attempting standard refresh...")
|
||
con.run("REFRESH MATERIALIZED VIEW unified_search_index;")
|
||
print(f" ✓ unified_search_index refreshed!")
|
||
|
||
# 8. Analyze
|
||
print("\n⚡ Step 8: Optimizing database query planner with ANALYZE...")
|
||
con.run("ANALYZE places_iraq;")
|
||
con.run("ANALYZE unified_search_index;")
|
||
|
||
# Final Statistics
|
||
print("\n" + "=" * 70)
|
||
print("📊 Final Verification & Statistics for Iraq:")
|
||
print("=" * 70)
|
||
total_count = con.run("SELECT count(*) FROM places_iraq;")[0][0]
|
||
print(f" 🎯 Total Places in places_iraq: {total_count:,}")
|
||
|
||
by_source = con.run("""
|
||
SELECT source, count(*)
|
||
FROM places_iraq
|
||
GROUP BY source
|
||
ORDER BY count(*) DESC;
|
||
""")
|
||
print(" 📈 Breakdown by Source:")
|
||
for src, cnt in by_source:
|
||
print(f" - {src}: {cnt:,}")
|
||
|
||
by_gov = con.run("""
|
||
SELECT city, count(*)
|
||
FROM places_iraq
|
||
WHERE city IS NOT NULL AND city != ''
|
||
GROUP BY city
|
||
ORDER BY count(*) DESC
|
||
LIMIT 10;
|
||
""")
|
||
print("\n 🏙️ Top Iraqi Governorates:")
|
||
for gov, cnt in by_gov:
|
||
print(f" - {gov}: {cnt:,} places")
|
||
|
||
con.close()
|
||
print(f"\n🎉 Total script execution time: {time.time() - t0:.2f}s")
|
||
print("✨ Iraq geospatial database is now vibrantly enriched!")
|
||
|
||
if __name__ == '__main__':
|
||
main()
|