#!/usr/bin/env python3
"""
Enrich Egypt Places Dataset (`places_egypt`) from:
1. OpenStreetMap Named Points (`planet_osm_point`)
2. OpenStreetMap Named Polygons (`planet_osm_polygon`)
3. Overture Maps Places (`overture_place`)
4. Deduplicate using spatial grid (<60m) against existing places_egypt
5. Refresh `unified_search_index`
"""

import os
import sys
import math
import time
import argparse
from collections import defaultdict

def haversine(lat1, lon1, lat2, lon2):
    R = 6371000  # meters
    phi1 = math.radians(lat1)
    phi2 = math.radians(lat2)
    delta_phi = math.radians(lat2 - lat1)
    delta_lambda = math.radians(lon2 - lon1)
    a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
    return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))

def normalize_text(t):
    if not t:
        return ''
    s = t.strip()
    s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
    return ' '.join(s.split()).lower()

def clean_sql_str(val):
    if val is None:
        return 'NULL'
    s = str(val).strip().replace("'", "''")
    return f"'{s}'"

OSM_CAT_MAP = {
    'village': 'قرية / تجمع سكاني',
    'town': 'بلدة / مركز',
    'city': 'مدينة',
    'suburb': 'حي / ضاحية',
    'neighbourhood': 'منطقة سكنية',
    'hamlet': 'نجع / كفر',
    'locality': 'موضع / منطقة',
    'school': 'مدرسة / تعليم',
    'college': 'كلية',
    'university': 'جامعة',
    'kindergarten': 'روضة أطفال',
    'supermarket': 'سوبرماركت وهايبرماركت',
    'convenience': 'محل بقالة',
    'bakery': 'مخبز وفرن',
    'butcher': 'جزارة وقصابة',
    'mall': 'مركز تسوق ومول',
    'place_of_worship': 'مسجد / دار عبادة',
    'mosque': 'جامع / مسجد',
    'clinic': 'مركز طبي وعيادة',
    'pharmacy': 'صيدلية',
    'hospital': 'مستشفى',
    'doctors': 'عيادات تخصصية',
    'dentist': 'عيادة أسنان',
    'restaurant': 'مطعم',
    'cafe': 'مقهى وكافيه',
    'fast_food': 'مأكولات سريعة',
    'fuel': 'محطة وقود وبنزين',
    'bank': 'مصرف / بنك',
    'atm': 'ماكينة صراف آلي ATM',
    'bureau_de_change': 'مكتب صرافة',
    'police': 'قسم شرطة ونقطة أمنية',
    'fire_station': 'وحدة مطافئ / دفاع مدني',
    'post_office': 'مكتب بريد',
    'attraction': 'معلم سياحي وتاريخي',
    'hotel': 'فندق وإقامة',
    'guest_house': 'نُزل واستضافة',
    'museum': 'متحف وآثار',
    'park': 'حديقة ومنتزه عام',
    'stadium': 'استاد وملعب رياضي',
    'sports_centre': 'مركز ونادي رياضي',
    'car_repair': 'ورشة صيانة سيارات',
    'car_wash': 'مغسلة سيارات',
    'company': 'شركة ومقر أعمال',
    'government': 'جهة حكومية ورسمية',
    'mobile_phone': 'متجر هواتف ومحمول',
    'clothes': 'متجر ملابس وأزياء',
    'shoes': 'متجر أحذية',
    'hairdresser': 'صالون حلاقة وتجميل',
    'optician': 'بصريات ونظارات',
    'hardware': 'أدوات بناء وتجهيزات',
}

OVERTURE_CAT_MAP = {
    'restaurant': 'مطعم',
    'cafe': 'مقهى وكافيه',
    'coffee_shop': 'مقهى وكوفي شوب',
    'casual_eatery': 'مأكولات سريعة وخفيفة',
    'bakery': 'مخبز ومعجنات',
    'food_and_beverage_store': 'بقالة ومواد غذائية',
    'supermarket': 'سوبرماركت وهايبرماركت',
    'fashion_and_apparel_store': 'متجر ملابس وأزياء',
    'shoe_store': 'متجر أحذية',
    'jewelry_store': 'مجوهرات وحلي',
    'electronics_store': 'متجر إلكترونيات وأجهزة',
    'mobile_phone_store': 'متجر هواتف ذكية',
    'hardware_home_and_garden_store': 'مواد بناء ومستلزمات منزلية',
    'home_service': 'خدمات منزلية وصيانة',
    'personal_or_beauty_service': 'صالون ومركز تجميل',
    'hospital': 'مستشفى / مركز طبي',
    'dental_clinic': 'عيادة أسنان',
    'specialized_health_care': 'عيادة تخصصية',
    'pharmacy': 'صيدلية',
    'hotel': 'فندق وإقامة',
    'place_of_learning': 'منشأة تعليمية',
    'school': 'مدرسة',
    'college_university': 'جامعة / كلية',
    'historic_site': 'موقع تاريخي وأثري',
    'religious_organization': 'مؤسسة دينية',
    'mosque': 'مسجد / جامع',
    'real_estate_service': 'مكتب عقاري',
    'professional_service': 'خدمات مهنية واستشارية',
    'travel_service': 'مكتب سياحة وسفر',
    'corporate_or_business_office': 'شركة ومؤسسة أعمال',
    'car_repair': 'صيانة وتصليح سيارات',
    'fuel_station': 'محطة وقود',
    'bank': 'مصرف / بنك',
    'government_office': 'مصلحة وهيئة حكومية',
}

def map_category(cat_str, source_type='osm'):
    if not cat_str:
        return 'مكان ونقطة اهتمام'
    clean = str(cat_str).strip().lower()
    if source_type == 'overture':
        return OVERTURE_CAT_MAP.get(clean, 'مكان عام وتجاري')
    return OSM_CAT_MAP.get(clean, 'مكان عام')

def get_grid_cell(lat, lon, cell_size=0.01):
    return (int(lat / cell_size), int(lon / cell_size))

def is_duplicate(name_norm, lat, lon, grid, max_dist_m=60.0):
    d_lat_thresh = 0.0006
    d_lon_thresh = 0.0007
    c_x, c_y = get_grid_cell(lat, lon)
    for dx in (-1, 0, 1):
        for dy in (-1, 0, 1):
            cell = (c_x + dx, c_y + dy)
            if cell in grid:
                for existing_name, ex_lat, ex_lon in grid[cell]:
                    if abs(lat - ex_lat) > d_lat_thresh or abs(lon - ex_lon) > d_lon_thresh:
                        continue
                    dist = haversine(lat, lon, ex_lat, ex_lon)
                    if dist <= max_dist_m:
                        if (name_norm == existing_name or 
                            name_norm in existing_name or 
                            existing_name in name_norm):
                            return True
    return False

def add_to_grid(name_norm, lat, lon, grid):
    cell = get_grid_cell(lat, lon)
    grid[cell].append((name_norm, lat, lon))

def main():
    parser = argparse.ArgumentParser(description="Enrich Egypt places from OSM & Overture.")
    parser.add_argument("--db-host", default="127.0.0.1")
    parser.add_argument("--db-port", type=int, default=5432)
    parser.add_argument("--db-user", default="mapuser")
    parser.add_argument("--db-pass", default="mappass")
    parser.add_argument("--db-name", default="mapdb")
    parser.add_argument("--dry-run", action="store_true")
    args = parser.parse_args()

    import pg8000.native

    print("=" * 70)
    print("🇪🇬 SIRO Maps — Comprehensive Egypt Places Enrichment & Deduplication")
    print("=" * 70)

    print(f"🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
    con = pg8000.native.Connection(
        user=args.db_user,
        password=args.db_pass,
        host=args.db_host,
        port=args.db_port,
        database=args.db_name
    )

    t0 = time.time()
    grid = defaultdict(list)

    # 1. Load existing places_egypt
    print("\n📦 Step 1: Loading existing places_egypt into spatial index...")
    existing = con.run("""
        SELECT id, name, latitude, longitude 
        FROM places_egypt 
        WHERE latitude IS NOT NULL AND longitude IS NOT NULL;
    """)
    print(f"   -> Loaded {len(existing):,} existing places.")
    for row in existing:
        pid, name, lat, lon = row
        lat = float(lat)
        lon = float(lon)
        norm_name = normalize_text(name)
        add_to_grid(norm_name, lat, lon, grid)

    print(f"   ✓ Spatial grid built with {sum(len(v) for v in grid.values()):,} items.")

    # Egypt Bounding Box: 21.8 <= lat <= 31.8, 24.7 <= lon <= 36.9
    EG_MIN_LON, EG_MIN_LAT = 24.7, 21.8
    EG_MAX_LON, EG_MAX_LAT = 36.9, 31.8

    # 2. Harvest from planet_osm_point in Egypt
    print("\n📍 Step 2: Harvesting from OpenStreetMap Named Points (planet_osm_point)...")
    osm_pts = con.run(f"""
        SELECT 
            name,
            COALESCE(amenity, shop, tourism, historic, place, leisure, office, building, highway, "natural") as cat,
            ST_Y(ST_Transform(way, 4326)) as lat,
            ST_X(ST_Transform(way, 4326)) as lon
        FROM planet_osm_point
        WHERE name IS NOT NULL 
          AND TRIM(name) != ''
          AND way && ST_Transform(ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326), 3857);
    """)
    print(f"   -> Found {len(osm_pts):,} candidates in planet_osm_point for Egypt.")

    osm_pts_to_insert = []
    osm_pts_dups = 0

    for r in osm_pts:
        name = str(r[0]).strip()
        cat_raw = r[1]
        lat = float(r[2])
        lon = float(r[3])

        if not (EG_MIN_LAT <= lat <= EG_MAX_LAT and EG_MIN_LON <= lon <= EG_MAX_LON):
            continue

        norm_name = normalize_text(name)
        if len(norm_name) < 2:
            continue

        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
            osm_pts_dups += 1
            continue

        category = map_category(cat_raw, 'osm')
        pop_score = 65 if cat_raw in ('village', 'town', 'city', 'hospital', 'university', 'mosque') else 40

        item = {
            'name': name,
            'name_ar': name,
            'name_en': name,
            'category': category,
            'city': 'مصر',
            'address': f"{category}, مصر",
            'lat': lat,
            'lon': lon,
            'source': 'osm_point',
            'popularity_score': pop_score
        }
        osm_pts_to_insert.append(item)
        add_to_grid(norm_name, lat, lon, grid)

    print(f"   ✓ OSM Points Processing Complete:")
    print(f"      - Duplicates Filtered: {osm_pts_dups:,}")
    print(f"      - Net New Places: {len(osm_pts_to_insert):,}")

    # 3. Harvest from planet_osm_polygon in Egypt
    print("\n🏛️ Step 3: Harvesting from OpenStreetMap Named Polygons (planet_osm_polygon)...")
    osm_polys = con.run(f"""
        SELECT 
            name,
            COALESCE(amenity, shop, tourism, historic, place, leisure, building, landuse) as cat,
            ST_Y(ST_Centroid(ST_Transform(way, 4326))) as lat,
            ST_X(ST_Centroid(ST_Transform(way, 4326))) as lon
        FROM planet_osm_polygon
        WHERE name IS NOT NULL 
          AND TRIM(name) != ''
          AND (amenity IS NOT NULL OR shop IS NOT NULL OR tourism IS NOT NULL OR historic IS NOT NULL 
               OR place IS NOT NULL OR leisure IS NOT NULL OR landuse IN ('commercial', 'retail', 'industrial', 'cemetery', 'religious'))
          AND way && ST_Transform(ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326), 3857);
    """)
    print(f"   -> Found {len(osm_polys):,} candidates in planet_osm_polygon for Egypt.")

    osm_polys_to_insert = []
    osm_polys_dups = 0

    for r in osm_polys:
        name = str(r[0]).strip()
        cat_raw = r[1]
        lat = float(r[2])
        lon = float(r[3])

        if not (EG_MIN_LAT <= lat <= EG_MAX_LAT and EG_MIN_LON <= lon <= EG_MAX_LON):
            continue

        norm_name = normalize_text(name)
        if len(norm_name) < 2:
            continue

        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
            osm_polys_dups += 1
            continue

        category = map_category(cat_raw, 'osm')
        pop_score = 70 if cat_raw in ('hospital', 'university', 'mall', 'stadium', 'attraction') else 45

        item = {
            'name': name,
            'name_ar': name,
            'name_en': name,
            'category': category,
            'city': 'مصر',
            'address': f"{category}, مصر",
            'lat': lat,
            'lon': lon,
            'source': 'osm_polygon',
            'popularity_score': pop_score
        }
        osm_polys_to_insert.append(item)
        add_to_grid(norm_name, lat, lon, grid)

    print(f"   ✓ OSM Polygons Processing Complete:")
    print(f"      - Duplicates Filtered: {osm_polys_dups:,}")
    print(f"      - Net New Places: {len(osm_polys_to_insert):,}")

    # 4. Harvest from overture_place in Egypt
    print("\n🗺️ Step 4: Harvesting from Overture Maps Places (overture_place)...")
    ovt_rows = con.run(f"""
        SELECT 
            name_primary,
            basic_category,
            addresses,
            confidence,
            ST_Y(ST_Centroid(location::geometry)) as lat,
            ST_X(ST_Centroid(location::geometry)) as lon
        FROM overture_place
        WHERE name_primary IS NOT NULL 
          AND TRIM(name_primary) != ''
          AND ST_Within(location::geometry, ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326));
    """)
    print(f"   -> Found {len(ovt_rows):,} candidates in overture_place for Egypt.")

    ovt_to_insert = []
    ovt_dups = 0

    for r in ovt_rows:
        name = str(r[0]).strip()
        cat_str = r[1]
        addr_list = r[2]
        confidence = float(r[3] or 0.8)
        lat = float(r[4])
        lon = float(r[5])

        norm_name = normalize_text(name)
        if len(norm_name) < 2:
            continue

        if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
            ovt_dups += 1
            continue

        city = 'مصر'
        addr_text = None
        if addr_list and isinstance(addr_list, list) and len(addr_list) > 0:
            a = addr_list[0]
            if isinstance(a, dict):
                city = a.get('locality') or a.get('region') or 'مصر'
                addr_text = a.get('freeform')

        category = map_category(cat_str, 'overture')
        pop_score = min(100, int(confidence * 60 + 20))

        item = {
            'name': name,
            'name_ar': name,
            'name_en': name,
            'category': category,
            'city': city,
            'address': addr_text or f"{city}, مصر",
            'lat': lat,
            'lon': lon,
            'source': 'overture_maps',
            'popularity_score': pop_score
        }
        ovt_to_insert.append(item)
        add_to_grid(norm_name, lat, lon, grid)

    print(f"   ✓ Overture Processing Complete:")
    print(f"      - Duplicates Filtered: {ovt_dups:,}")
    print(f"      - Net New Places: {len(ovt_to_insert):,}")

    all_new = osm_pts_to_insert + osm_polys_to_insert + ovt_to_insert
    print(f"\n🌟 Total Net New Places to Insert into places_egypt: {len(all_new):,}")

    if args.dry_run:
        print("💡 Dry run complete. No database changes made.")
        return

    # 5. Insert into places_egypt
    print(f"\n🚀 Step 5: Inserting {len(all_new):,} places into places_egypt...")
    t_insert = time.time()
    batch_size = 2500
    total_inserted = 0

    con.run("BEGIN;")
    for i in range(0, len(all_new), batch_size):
        batch = all_new[i:i + batch_size]
        values = []
        for r in batch:
            c_name = clean_sql_str(r['name'])
            c_name_ar = clean_sql_str(r['name_ar'])
            c_name_en = clean_sql_str(r['name_en'])
            c_cat = clean_sql_str(r['category'])
            c_city = clean_sql_str(r['city'])
            c_addr = clean_sql_str(r['address'])
            c_src = clean_sql_str(r['source'])
            lat = r['lat']
            lon = r['lon']
            pop = r['popularity_score']

            line = (
                f"({c_name}, {c_name_ar}, {c_name_en}, {lat:.7f}, {lon:.7f}, {c_cat}, "
                f"{c_city}, {c_addr}, {c_src}, {pop}, "
                f"ST_SetSRID(ST_MakePoint({lon:.7f}, {lat:.7f}), 4326))"
            )
            values.append(line)

        sql = (
            "INSERT INTO places_egypt ("
            "    name, name_ar, name_en, latitude, longitude, category,"
            "    city, address, source, popularity_score, location"
            ") VALUES " + ",\n".join(values) + ";"
        )
        con.run(sql)
        total_inserted += len(batch)
        print(f"   -> Progress: {total_inserted:,} / {len(all_new):,} places inserted...")

    con.run("COMMIT;")
    print(f"✅ Ingestion committed successfully in {time.time() - t_insert:.2f}s!")

    # 6. Refresh Materialized View
    print("\n🔄 Step 6: Refreshing materialized view unified_search_index concurrently...")
    t_mv = time.time()
    try:
        con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
        print(f"   ✓ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")
    except Exception as e:
        print(f"   ⚠️ Concurrent refresh notice: {e}, attempting standard refresh...")
        con.run("REFRESH MATERIALIZED VIEW unified_search_index;")
        print(f"   ✓ unified_search_index refreshed!")

    con.run("ANALYZE places_egypt;")
    con.run("ANALYZE unified_search_index;")

    total_count = con.run("SELECT count(*) FROM places_egypt;")[0][0]
    print(f"\n🎯 Total Places in places_egypt: {total_count:,}")
    con.close()
    print(f"🎉 Egypt places enrichment completed in {time.time() - t0:.2f}s!")

if __name__ == '__main__':
    main()