#!/usr/bin/env python3
"""
Migrate and deduplicate Jordan (Amman, Zarqa, Irbid) Places Dataset into places_jordan table on PostgreSQL.
Handles Arabic normalization, spatial bounding box filtering (Jordan bounds),
exact deduplication, spatial near-duplicate filtering (< 50m), popularity scoring,
and concurrent refresh of unified_search_index.
"""

import os
import sys
import csv
import gzip
import math
import time
import argparse
from collections import defaultdict, Counter

def haversine(lat1, lon1, lat2, lon2):
    R = 6371000  # meters
    phi1 = math.radians(lat1)
    phi2 = math.radians(lat2)
    delta_phi = math.radians(lat2 - lat1)
    delta_lambda = math.radians(lon2 - lon1)
    a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
    return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))

def normalize_text(t):
    if not t:
        return ''
    s = t.strip()
    # Normalize Arabic alef, teh marbuta, etc.
    s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
    return ' '.join(s.split()).lower()

def clean_sql_str(val):
    if val is None:
        return 'NULL'
    s = str(val).strip().replace("'", "''")
    return f"'{s}'"

def calculate_popularity(row):
    # Reviews and rating
    reviews = 0
    try:
        reviews = int(float(row.get('reviews_count') or 0))
    except Exception:
        reviews = 0

    rating = 0.0
    try:
        rating = float(row.get('rating') or 0)
    except Exception:
        rating = 0.0

    score = int(reviews * 0.2 + rating * 5)

    # Heuristic boost for critical POIs
    cat = row.get('category_queried', '')
    if any(k in cat for k in ['مستشفى', 'جامعة', 'كلية', 'مركز صحي', 'طوارئ', 'دفاع مدني']):
        score = max(score, 90)
    elif any(k in cat for k in ['صيدلية', 'بنك', 'صراف آلي', 'سوبر ماركت', 'محطة']):
        score = max(score, 60)
    elif any(k in cat for k in ['مدرسة', 'روضة', 'عيادة', 'بلدية', 'دائرة']):
        score = max(score, 50)

    return min(100, max(10, score))

def main():
    parser = argparse.ArgumentParser(description="Migrate Jordan places dataset with deduplication.")
    parser.add_argument("--file", default="", help="Path to CSV file.")
    parser.add_argument("--db-host", default="127.0.0.1", help="Database host")
    parser.add_argument("--db-port", type=int, default=5432, help="Database port")
    parser.add_argument("--db-user", default="mapuser", help="Database user")
    parser.add_argument("--db-pass", default="mappass", help="Database password")
    parser.add_argument("--db-name", default="mapdb", help="Database name")
    parser.add_argument("--dry-run", action="store_true", help="Perform validation and deduplication only without inserting.")
    args = parser.parse_args()

    # Find file
    file_path = args.file
    if not file_path:
        candidates = [
            "data/checkpoint_35242_records.csv",
            "/home/hamzadoctor/app/data/checkpoint_35242_records.csv",
            "checkpoint_35242_records.csv"
        ]
        for c in candidates:
            if os.path.exists(c):
                file_path = c
                break

    if not file_path or not os.path.exists(file_path):
        print(f"❌ Error: Dataset file not found: {file_path}")
        sys.exit(1)

    print(f"📂 Processing dataset: {file_path}")
    open_fn = gzip.open if file_path.endswith('.gz') else open
    mode = 'rt' if file_path.endswith('.gz') else 'r'

    total_read = 0
    empty_name = 0
    out_of_bounds = 0
    valid_records = []
    gov_counter = Counter()

    t0 = time.time()
    with open_fn(file_path, mode, encoding='utf-8-sig') as f:
        reader = csv.DictReader(f)
        for row in reader:
            total_read += 1
            name = row.get('name', '').strip()
            if not name:
                empty_name += 1
                continue

            try:
                lat = float(row['latitude'])
                lng = float(row['longitude'])
            except Exception:
                out_of_bounds += 1
                continue

            # Jordan bounding box check (29.15 to 33.45 Lat, 34.85 to 39.35 Lng)
            if not (29.15 <= lat <= 33.45 and 34.85 <= lng <= 39.35):
                out_of_bounds += 1
                continue

            category = row.get('category_queried', '').strip() or 'مكان عام'
            sector = row.get('sector', '').strip()
            gov = row.get('governorate', '').strip() or 'الأردن'
            area = row.get('area', '').strip()
            maps_url = row.get('maps_url', '').strip()

            address = f"{area}, {gov}, الأردن" if area else f"{gov}, الأردن"
            desc = sector if sector else 'نقطة اهتمام في الأردن'
            pop_score = calculate_popularity(row)

            gov_counter[gov] += 1

            valid_records.append({
                'name': name,
                'norm_name': normalize_text(name),
                'lat': lat,
                'lng': lng,
                'category': category,
                'city': gov,
                'neighbourhood': area,
                'address': address,
                'description': desc,
                'popularity_score': pop_score,
                'maps_url': maps_url,
                'sector': sector
            })

    print(f"📊 Initial parse complete in {time.time() - t0:.2f}s:")
    print(f"   - Total rows in CSV: {total_read:,}")
    print(f"   - Out of Jordan bounds / invalid coords: {out_of_bounds:,}")
    print(f"   - Valid in Jordan bounds: {len(valid_records):,}")
    print(f"   - Governorate distribution: {dict(gov_counter)}")

    # Deduplication Step 1: Exact (norm_name, round(lat, 5), round(lng, 5)) and maps_url
    seen_exact = {}
    seen_urls = {}
    exact_dups = 0

    for r in valid_records:
        k = (r['norm_name'], round(r['lat'], 5), round(r['lng'], 5))
        url = r['maps_url']

        if k in seen_exact:
            exact_dups += 1
            if r['popularity_score'] > seen_exact[k]['popularity_score']:
                seen_exact[k] = r
            continue

        if url and url in seen_urls:
            exact_dups += 1
            if r['popularity_score'] > seen_urls[url]['popularity_score']:
                seen_urls[url] = r
            continue

        seen_exact[k] = r
        if url:
            seen_urls[url] = r

    dedup_step1 = list(seen_exact.values())
    print(f"🔍 Step 1 Deduplication (Exact coords / URL):")
    print(f"   - Removed {exact_dups} duplicate records.")
    print(f"   - Remaining: {len(dedup_step1):,}")

    # Deduplication Step 2: Spatial near-duplicates (< 50m with identical normalized name)
    by_norm_name = defaultdict(list)
    for r in dedup_step1:
        by_norm_name[r['norm_name']].append(r)

    final_records = []
    near_dups_filtered = 0

    for norm_name, items in by_norm_name.items():
        if len(items) == 1:
            final_records.append(items[0])
        else:
            items.sort(key=lambda x: x['popularity_score'], reverse=True)
            kept = []
            for candidate in items:
                is_dup = False
                for existing in kept:
                    d = haversine(candidate['lat'], candidate['lng'], existing['lat'], existing['lng'])
                    if d < 50:
                        is_dup = True
                        near_dups_filtered += 1
                        break
                if not is_dup:
                    kept.append(candidate)
            final_records.extend(kept)

    print(f"🎯 Step 2 Deduplication (Spatial near-duplicates < 50m):")
    print(f"   - Filtered out {near_dups_filtered} near-duplicates.")
    print(f"   - Final unique clean places to migrate: {len(final_records):,}")

    final_govs = Counter([r['city'] for r in final_records])
    print(f"   - Final Governorates: {dict(final_govs)}")

    if args.dry_run:
        print("💡 Dry run complete. No database changes made.")
        return

    # Database Migration
    import pg8000.native
    print(f"\n🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
    con = pg8000.native.Connection(
        user=args.db_user,
        password=args.db_pass,
        host=args.db_host,
        port=args.db_port,
        database=args.db_name
    )

    t_db = time.time()
    source_tag = 'checkpoint_35242_amman_zarqa'

    print(f"🗑️ Removing previous '{source_tag}' imports from places_jordan...")
    con.run("BEGIN;")
    con.run(f"DELETE FROM places_jordan WHERE source = '{source_tag}';")

    # Temporarily disable trigger for high-speed bulk ingestion
    print("⚡ Disabling location trigger for high-speed ingestion...")
    con.run("ALTER TABLE places_jordan DISABLE TRIGGER trg_sync_place_location_jordan;")

    batch_size = 2000
    total_inserted = 0
    print(f"🚀 Inserting {len(final_records):,} places in batches of {batch_size}...")

    for i in range(0, len(final_records), batch_size):
        batch = final_records[i:i + batch_size]
        values = []
        for r in batch:
            c_name = clean_sql_str(r['name'])
            c_cat = clean_sql_str(r['category'])
            c_city = clean_sql_str(r['city'])
            c_area = clean_sql_str(r['neighbourhood'])
            c_addr = clean_sql_str(r['address'])
            c_desc = clean_sql_str(r['description'])
            lat = r['lat']
            lng = r['lng']
            pop = r['popularity_score']

            line = (
                f"({c_name}, {c_name}, {lat:.7f}, {lng:.7f}, {c_cat}, {c_city}, {c_area}, "
                f"{c_addr}, {c_desc}, {pop}, '{source_tag}', "
                f"ST_SetSRID(ST_MakePoint({lng:.7f}, {lat:.7f}), 4326))"
            )
            values.append(line)

        sql = (
            "INSERT INTO places_jordan ("
            "    name, name_ar, latitude, longitude, category, city, neighbourhood,"
            "    address, description, popularity_score, source, location"
            ") VALUES " + ",\n".join(values) + ";"
        )
        con.run(sql)
        total_inserted += len(batch)
        print(f"   -> Inserted {total_inserted:,} / {len(final_records):,} places...")

    # Re-enable trigger
    print("⚡ Re-enabling location trigger...")
    con.run("ALTER TABLE places_jordan ENABLE TRIGGER trg_sync_place_location_jordan;")

    # Batch update governorate_id for Amman and Zarqa based on city name
    print("🏛️ Updating administrative links for new places...")
    con.run(f"""
        UPDATE places_jordan 
        SET governorate_id = 137 
        WHERE source = '{source_tag}' AND city = 'عمان';
    """)
    con.run(f"""
        UPDATE places_jordan 
        SET governorate_id = 5 
        WHERE source = '{source_tag}' AND city = 'الزرقاء';
    """)
    con.run(f"""
        UPDATE places_jordan 
        SET governorate_id = 184 
        WHERE source = '{source_tag}' AND city = 'إربد';
    """)

    con.run("COMMIT;")
    print(f"✅ Ingestion committed successfully in {time.time() - t_db:.2f}s!")

    # Refresh materialized view
    print("🔄 Refreshing materialized view unified_search_index concurrently...")
    t_mv = time.time()
    con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
    print(f"✅ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")

    # Analyze table
    print("⚡ Running ANALYZE on places_jordan...")
    con.run("ANALYZE places_jordan;")

    # Verification
    print("\n🔍 Verification & Statistics:")
    count_res = con.run("SELECT count(*) FROM places_jordan;")[0][0]
    new_count = con.run(f"SELECT count(*) FROM places_jordan WHERE source = '{source_tag}';")[0][0]
    gov_stats = con.run(f"""
        SELECT city, count(*) 
        FROM places_jordan 
        WHERE source = '{source_tag}' 
        GROUP BY city 
        ORDER BY count(*) DESC;
    """)
    print(f"   - Total places in places_jordan: {count_res:,}")
    print(f"   - Newly added from Amman & Zarqa checkpoint: {new_count:,}")
    print("   - Governorates breakdown:")
    for gov, cnt in gov_stats:
        print(f"     * {gov}: {cnt:,} places")

    # Sample Geocoding search test
    test_queries = ['صيدلية في عمان', 'الجبيهة', 'الزرقاء الجديدة', 'وسط البلد عمان', 'الشميساني']
    print("\n🧪 Testing search on unified_search_index:")
    for q in test_queries:
        results = con.run(f"""
            SELECT name, category, city, latitude, longitude 
            FROM places_jordan 
            WHERE (name ILIKE '%{q}%' OR neighbourhood ILIKE '%{q}%' OR address ILIKE '%{q}%')
              AND source = '{source_tag}'
            LIMIT 2;
        """)
        if results:
            first = results[0]
            print(f"   ✓ Query '{q}': Found '{first[0]}' ({first[1]} - {first[2]}) @ {first[3]}, {first[4]}")
        else:
            print(f"   - Query '{q}': No direct checkpoint match, checking overall...")

    con.close()
    print("\n🎉 Jordan dataset migration completed successfully!")

if __name__ == '__main__':
    main()