#!/usr/bin/env python3
"""
Comprehensive Iraq Places Enrichment & Deduplication Script.
Enriches `places_iraq` table by harvesting and deduplicating:
1. Overture Maps Places (`overture_place`)
2. OpenStreetMap Named Points (`planet_osm_point`)
3. OpenStreetMap Named Polygons (`planet_osm_polygon`)
4. Administrative boundaries & Governorates linking
5. Concurrently refreshes `unified_search_index`
"""

import os
import sys
import math
import time
import argparse
from collections import defaultdict

def haversine(lat1, lon1, lat2, lon2):
    R = 6371000  # meters
    phi1 = math.radians(lat1)
    phi2 = math.radians(lat2)
    delta_phi = math.radians(lat2 - lat1)
    delta_lambda = math.radians(lon2 - lon1)
    a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
    return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))

def normalize_text(t):
    if not t:
        return ''
    s = t.strip()
    s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
    return ' '.join(s.split()).lower()

def clean_sql_str(val):
    if val is None:
        return 'NULL'
    s = str(val).strip().replace("'", "''")
    return f"'{s}'"

# Category Translations
OVERTURE_CAT_MAP = {
    'restaurant': 'مطعم',
    'cafe': 'مقهى وكافيه',
    'coffee_shop': 'مقهى وكوفي شوب',
    'casual_eatery': 'وجبات سريعة ومأكولات خفيفة',
    'bakery': 'مخبز ومعجنات',
    'food_and_beverage_store': 'بقالة ومواد غذائية',
    'supermarket': 'سوبرماركت وهايبرماركت',
    'fashion_and_apparel_store': 'متجر ألبسة وأزياء',
    'shoe_store': 'متجر أحذية',
    'jewelry_store': 'مجوهرات وحلي',
    'electronics_store': 'متجر إلكترونيات وأجهزة',
    'mobile_phone_store': 'متجر هواتف ذكية',
    'hardware_home_and_garden_store': 'متجر مواد بناء ومستلزمات منزلية',
    'home_service': 'خدمات منزلية وصيانة',
    'personal_or_beauty_service': 'صالون ومركز تجميل',
    'hair_salon': 'صالون حلاقة وتزيين',
    'hospital': 'مستشفى / مركز طبي',
    'dental_clinic': 'عيادة طب أسنان',
    'specialized_health_care': 'عيادة استشارية متخصصة',
    'pharmacy': 'صيدلية',
    'hotel': 'فندق وإقامة',
    'place_of_learning': 'مؤسسة تعليمية',
    'school': 'مدرسة',
    'college_university': 'كلية / جامعة',
    'historic_site': 'موقع تاريخي وأثري',
    'religious_organization': 'دار عبادة ومؤسسة دينية',
    'mosque': 'مسجد / جامع',
    'real_estate_service': 'مكتب عقاري',
    'professional_service': 'خدمات مهنية واستشارية',
    'travel_service': 'مكتب سياحة وسفر',
    'corporate_or_business_office': 'مكتب شركة ومؤسسة أعمال',
    'car_repair': 'صيانة وتصليح سيارات',
    'car_dealership': 'معرض ووكالة سيارات',
    'fuel_station': 'محطة وقود',
    'bank': 'مصرف / بنك',
    'government_office': 'دائرة حكومية ورسمية',
    'law_firm': 'مكتب محاماة واستشارات قانونية',
    'sports_club': 'نادي ومجمع رياضي',
    'park': 'منتزه وحديقة عامة',
}

OSM_CAT_MAP = {
    'village': 'قرية / تجمع سكاني',
    'town': 'بلدة / قضاء',
    'city': 'مدينة',
    'suburb': 'حي / ضاحية',
    'neighbourhood': 'حي سكني',
    'hamlet': 'قرية صغيرة',
    'locality': 'منطقة / موضع',
    'school': 'مدرسة / تعليم',
    'college': 'كلية',
    'university': 'جامعة',
    'kindergarten': 'روضة أطفال',
    'supermarket': 'سوبرماركت ومواد غذائية',
    'convenience': 'محل بقالة',
    'bakery': 'مخبز',
    'butcher': 'ملحمة وقصابة',
    'mall': 'مركز تسوق ومول',
    'place_of_worship': 'مسجد / دار عبادة',
    'mosque': 'مسجد / جامع',
    'clinic': 'مركز صحي وعيادة',
    'pharmacy': 'صيدلية',
    'hospital': 'مستشفى',
    'doctors': 'عيادة طبيب',
    'dentist': 'عيادة أسنان',
    'restaurant': 'مطعم',
    'cafe': 'مقهى وكافيه',
    'fast_food': 'وجبات سريعة',
    'fuel': 'محطة وقود',
    'bank': 'مصرف / بنك',
    'atm': 'صراف آلي',
    'bureau_de_change': 'مكتب صرافة وتحويل مالي',
    'police': 'مركز شرطة وأمن',
    'fire_station': 'دفاع مدني / إطفاء',
    'post_office': 'مكتب بريد',
    'attraction': 'معلم سياحي',
    'hotel': 'فندق',
    'guest_house': 'نُزل واستضافة',
    'museum': 'متحف / تراث',
    'park': 'حديقة ومنتزه',
    'stadium': 'ملعب / مجمع رياضي',
    'sports_centre': 'مركز رياضي',
    'car_repair': 'تصليح سيارات',
    'car_wash': 'مغسلة سيارات',
    'company': 'شركة ومؤسسة',
    'government': 'دائرة حكومية',
    'mobile_phone': 'متجر هواتف ونقالات',
    'clothes': 'متجر ألبسة',
    'shoes': 'متجر أحذية',
    'hairdresser': 'صالون حلاقة',
    'optician': 'بصريات ونظارات',
    'hardware': 'مواد بناء وتجهيزات',
}

def map_overture_category(cat_str):
    if not cat_str:
        return 'مكان ونقطة اهتمام'
    clean = cat_str.strip().lower()
    return OVERTURE_CAT_MAP.get(clean, 'مكان عام وتجاري')

def map_osm_category(val):
    if not val:
        return 'نقطة اهتمام'
    clean = str(val).strip().lower()
    return OSM_CAT_MAP.get(clean, 'مكان عام')

def get_grid_cell(lat, lon, cell_size=0.01):
    # ~1.1km cell size
    return (int(lat / cell_size), int(lon / cell_size))

def is_duplicate(name_norm, lat, lon, grid, max_dist_m=60.0):
    # 60 meters approx threshold in degrees
    d_lat_thresh = 0.0006
    d_lon_thresh = 0.0007
    c_x, c_y = get_grid_cell(lat, lon)
    for dx in (-1, 0, 1):
        for dy in (-1, 0, 1):
            cell = (c_x + dx, c_y + dy)
            if cell in grid:
                for existing_name, ex_lat, ex_lon in grid[cell]:
                    if abs(lat - ex_lat) > d_lat_thresh or abs(lon - ex_lon) > d_lon_thresh:
                        continue
                    dist = haversine(lat, lon, ex_lat, ex_lon)
                    if dist <= max_dist_m:
                        # Check name similarity
                        if (name_norm == existing_name or 
                            name_norm in existing_name or 
                            existing_name in name_norm):
                            return True
    return False

def add_to_grid(name_norm, lat, lon, grid):
    cell = get_grid_cell(lat, lon)
    grid[cell].append((name_norm, lat, lon))

def main():
    parser = argparse.ArgumentParser(description="Enrich Iraq places comprehensively from Overture & OSM.")
    parser.add_argument("--db-host", default="127.0.0.1", help="Database host")
    parser.add_argument("--db-port", type=int, default=5432, help="Database port")
    parser.add_argument("--db-user", default="mapuser", help="Database user")
    parser.add_argument("--db-pass", default="mappass", help="Database password")
    parser.add_argument("--db-name", default="mapdb", help="Database name")
    parser.add_argument("--dry-run", action="store_true", help="Perform extraction and deduplication only without inserting.")
    args = parser.parse_args()

    import pg8000.native

    print("=" * 70)
    print("🇮🇶 SIRO Maps — Comprehensive Iraq Places Enrichment & Deduplication")
    print("=" * 70)

    print(f"🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
    con = pg8000.native.Connection(
        user=args.db_user,
        password=args.db_pass,
        host=args.db_host,
        port=args.db_port,
        database=args.db_name
    )

    t0 = time.time()
    grid = defaultdict(list)

    # 1. Load existing places into spatial hash grid
    print("\n📦 Step 1: Loading existing places_iraq into spatial index for deduplication...")
    existing = con.run("""
        SELECT id, name, latitude, longitude 
        FROM places_iraq 
        WHERE latitude IS NOT NULL AND longitude IS NOT NULL;
    """)
    print(f"   -> Loaded {len(existing):,} existing places.")
    for row in existing:
        pid, name, lat, lon = row
        lat = float(lat)
        lon = float(lon)
        norm_name = normalize_text(name)
        add_to_grid(norm_name, lat, lon, grid)

    print(f"   ✓ Spatial grid built with {sum(len(v) for v in grid.values()):,} items.")

    # 2. Harvest from overture_place
    print("\n🗺️ Step 2: Harvesting from Overture Maps Places (overture_place)...")
    overture_rows = con.run("""
        SELECT 
            name_primary,
            basic_category,
            addresses,
            confidence,
            ST_Y(ST_Centroid(location::geometry)) as lat,
            ST_X(ST_Centroid(location::geometry)) as lon
        FROM overture_place
        WHERE name_primary IS NOT NULL 
          AND TRIM(name_primary) != ''
          AND ST_Within(location::geometry, ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326));
    """)
    print(f"   -> Found {len(overture_rows):,} candidates in overture_place for Iraq.")

    overture_to_insert = []
    overture_dups = 0

    for r in overture_rows:
        name = str(r[0]).strip()
        cat_str = r[1]
        addr_list = r[2]
        confidence = float(r[3] or 0.8)
        lat = float(r[4])
        lon = float(r[5])

        norm_name = normalize_text(name)
        if len(norm_name) < 2:
            continue

        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
            overture_dups += 1
            continue

        # Extract address details if available
        city = 'العراق'
        addr_text = None
        if addr_list and isinstance(addr_list, list) and len(addr_list) > 0:
            a = addr_list[0]
            if isinstance(a, dict):
                city = a.get('locality') or a.get('region') or 'العراق'
                addr_text = a.get('freeform')

        category = map_overture_category(cat_str)
        pop_score = min(100, int(confidence * 60 + 20))

        item = {
            'name': name,
            'name_ar': name,
            'name_en': name,
            'category': category,
            'city': city,
            'address': addr_text or f"{city}, العراق",
            'lat': lat,
            'lon': lon,
            'source': 'overture_maps',
            'popularity_score': pop_score
        }
        overture_to_insert.append(item)
        add_to_grid(norm_name, lat, lon, grid)

    print(f"   ✓ Overture Processing Complete:")
    print(f"      - Duplicates Filtered: {overture_dups:,}")
    print(f"      - Net New Places: {len(overture_to_insert):,}")

    # 3. Harvest from planet_osm_point
    print("\n📍 Step 3: Harvesting from OpenStreetMap Named Points (planet_osm_point)...")
    osm_point_rows = con.run("""
        SELECT 
            name,
            COALESCE(amenity, shop, tourism, historic, place, leisure, office, building, highway, "natural") as cat,
            ST_Y(ST_Transform(way, 4326)) as lat,
            ST_X(ST_Transform(way, 4326)) as lon
        FROM planet_osm_point
        WHERE name IS NOT NULL 
          AND TRIM(name) != ''
          AND way && ST_Transform(ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326), 3857);
    """)
    print(f"   -> Found {len(osm_point_rows):,} candidates in planet_osm_point for Iraq.")

    osm_point_to_insert = []
    osm_point_dups = 0

    for r in osm_point_rows:
        name = str(r[0]).strip()
        cat_raw = r[1]
        lat = float(r[2])
        lon = float(r[3])

        # Bounds check
        if not (28.8 <= lat <= 37.5 and 38.8 <= lon <= 48.8):
            continue

        norm_name = normalize_text(name)
        if len(norm_name) < 2:
            continue

        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
            osm_point_dups += 1
            continue

        category = map_osm_category(cat_raw)
        pop_score = 65 if cat_raw in ('village', 'town', 'city', 'hospital', 'university', 'mosque') else 40

        item = {
            'name': name,
            'name_ar': name,
            'name_en': name,
            'category': category,
            'city': 'العراق',
            'address': f"{category}, العراق",
            'lat': lat,
            'lon': lon,
            'source': 'osm_point',
            'popularity_score': pop_score
        }
        osm_point_to_insert.append(item)
        add_to_grid(norm_name, lat, lon, grid)

    print(f"   ✓ OSM Points Processing Complete:")
    print(f"      - Duplicates Filtered: {osm_point_dups:,}")
    print(f"      - Net New Places: {len(osm_point_to_insert):,}")

    # 4. Harvest from planet_osm_polygon (POIs)
    print("\n🏛️ Step 4: Harvesting from OpenStreetMap Named Polygons (planet_osm_polygon)...")
    osm_poly_rows = con.run("""
        SELECT 
            name,
            COALESCE(amenity, shop, tourism, historic, place, leisure, building, landuse) as cat,
            ST_Y(ST_Centroid(ST_Transform(way, 4326))) as lat,
            ST_X(ST_Centroid(ST_Transform(way, 4326))) as lon
        FROM planet_osm_polygon
        WHERE name IS NOT NULL 
          AND TRIM(name) != ''
          AND (amenity IS NOT NULL OR shop IS NOT NULL OR tourism IS NOT NULL OR historic IS NOT NULL 
               OR place IS NOT NULL OR leisure IS NOT NULL OR landuse IN ('commercial', 'retail', 'industrial', 'cemetery', 'religious'))
          AND way && ST_Transform(ST_MakeEnvelope(38.8, 28.8, 48.8, 37.5, 4326), 3857);
    """)
    print(f"   -> Found {len(osm_poly_rows):,} candidates in planet_osm_polygon for Iraq.")

    osm_poly_to_insert = []
    osm_poly_dups = 0

    for r in osm_poly_rows:
        name = str(r[0]).strip()
        cat_raw = r[1]
        lat = float(r[2])
        lon = float(r[3])

        if not (28.8 <= lat <= 37.5 and 38.8 <= lon <= 48.8):
            continue

        norm_name = normalize_text(name)
        if len(norm_name) < 2:
            continue

        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
            osm_poly_dups += 1
            continue

        category = map_osm_category(cat_raw)
        pop_score = 70 if cat_raw in ('hospital', 'university', 'mall', 'stadium', 'attraction') else 45

        item = {
            'name': name,
            'name_ar': name,
            'name_en': name,
            'category': category,
            'city': 'العراق',
            'address': f"{category}, العراق",
            'lat': lat,
            'lon': lon,
            'source': 'osm_polygon',
            'popularity_score': pop_score
        }
        osm_poly_to_insert.append(item)
        add_to_grid(norm_name, lat, lon, grid)

    print(f"   ✓ OSM Polygons Processing Complete:")
    print(f"      - Duplicates Filtered: {osm_poly_dups:,}")
    print(f"      - Net New Places: {len(osm_poly_to_insert):,}")

    # Total new places to insert
    all_new_places = overture_to_insert + osm_point_to_insert + osm_poly_to_insert
    print(f"\n🌟 Total Net New Places to Insert: {len(all_new_places):,}")

    if args.dry_run:
        print("💡 Dry run requested. Exiting without database insertion.")
        return

    # 5. Database Batch Insertion
    print(f"\n🚀 Step 5: Inserting {len(all_new_places):,} places into places_iraq...")
    t_insert = time.time()
    batch_size = 2500
    total_inserted = 0

    con.run("BEGIN;")
    for i in range(0, len(all_new_places), batch_size):
        batch = all_new_places[i:i + batch_size]
        values = []
        for r in batch:
            c_name = clean_sql_str(r['name'])
            c_name_ar = clean_sql_str(r['name_ar'])
            c_name_en = clean_sql_str(r['name_en'])
            c_cat = clean_sql_str(r['category'])
            c_city = clean_sql_str(r['city'])
            c_addr = clean_sql_str(r['address'])
            c_src = clean_sql_str(r['source'])
            lat = r['lat']
            lon = r['lon']
            pop = r['popularity_score']

            line = (
                f"({c_name}, {c_name_ar}, {c_name_en}, {lat:.7f}, {lon:.7f}, {c_cat}, "
                f"{c_city}, {c_addr}, {c_src}, {pop}, "
                f"ST_SetSRID(ST_MakePoint({lon:.7f}, {lat:.7f}), 4326))"
            )
            values.append(line)

        sql = (
            "INSERT INTO places_iraq ("
            "    name, name_ar, name_en, latitude, longitude, category,"
            "    city, address, source, popularity_score, location"
            ") VALUES " + ",\n".join(values) + ";"
        )
        con.run(sql)
        total_inserted += len(batch)
        print(f"   -> Progress: {total_inserted:,} / {len(all_new_places):,} places inserted...")

    con.run("COMMIT;")
    print(f"✅ Ingestion committed successfully in {time.time() - t_insert:.2f}s!")

    # 6. Link Administrative Hierarchy (Governorates & Districts)
    print("\n🏛️ Step 6: Linking administrative hierarchy (Governorates & Districts)...")
    t_admin = time.time()
    con.run("""
        UPDATE places_iraq p
        SET 
            governorate_id = ab.id,
            city = COALESCE(ab.name_ar, ab.name, p.city)
        FROM admin_boundaries ab
        WHERE ab.country_code = 'IQ' 
          AND ab.admin_level = 4
          AND (p.governorate_id IS NULL OR p.city = 'العراق')
          AND ST_Within(p.location::geometry, ab.geom::geometry);
    """)
    print(f"   ✓ Linked governorates in {time.time() - t_admin:.2f}s.")

    # 7. Refresh Materialized View
    print("\n🔄 Step 7: Refreshing materialized view unified_search_index concurrently...")
    t_mv = time.time()
    try:
        con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
        print(f"   ✓ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")
    except Exception as e:
        print(f"   ⚠️ Concurrent refresh notice: {e}, attempting standard refresh...")
        con.run("REFRESH MATERIALIZED VIEW unified_search_index;")
        print(f"   ✓ unified_search_index refreshed!")

    # 8. Analyze
    print("\n⚡ Step 8: Optimizing database query planner with ANALYZE...")
    con.run("ANALYZE places_iraq;")
    con.run("ANALYZE unified_search_index;")

    # Final Statistics
    print("\n" + "=" * 70)
    print("📊 Final Verification & Statistics for Iraq:")
    print("=" * 70)
    total_count = con.run("SELECT count(*) FROM places_iraq;")[0][0]
    print(f"   🎯 Total Places in places_iraq: {total_count:,}")

    by_source = con.run("""
        SELECT source, count(*) 
        FROM places_iraq 
        GROUP BY source 
        ORDER BY count(*) DESC;
    """)
    print("   📈 Breakdown by Source:")
    for src, cnt in by_source:
        print(f"      - {src}: {cnt:,}")

    by_gov = con.run("""
        SELECT city, count(*) 
        FROM places_iraq 
        WHERE city IS NOT NULL AND city != ''
        GROUP BY city 
        ORDER BY count(*) DESC 
        LIMIT 10;
    """)
    print("\n   🏙️ Top Iraqi Governorates:")
    for gov, cnt in by_gov:
        print(f"      - {gov}: {cnt:,} places")

    con.close()
    print(f"\n🎉 Total script execution time: {time.time() - t0:.2f}s")
    print("✨ Iraq geospatial database is now vibrantly enriched!")

if __name__ == '__main__':
    main()