117 lines
4.5 KiB
Python
117 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Generate SQL migration file to ingest Iraq checkpoints CSV into places_iraq
|
|
and refresh the unified_search_index.
|
|
"""
|
|
import csv
|
|
import sys
|
|
import os
|
|
|
|
def clean_str(val):
|
|
if not val:
|
|
return 'NULL'
|
|
s = str(val).strip().replace("'", "''")
|
|
return f"'{s}'"
|
|
|
|
def main():
|
|
csv_path = 'data/iraq_checkpoints_70593.csv'
|
|
out_sql_path = 'infrastructure/sql/11_ingest_iraq_checkpoints.sql'
|
|
|
|
if not os.path.exists(csv_path):
|
|
print(f"Error: {csv_path} not found.")
|
|
sys.exit(1)
|
|
|
|
print(f"Reading {csv_path}...")
|
|
|
|
total_read = 0
|
|
valid_records = []
|
|
|
|
with open(csv_path, 'r', encoding='utf-8-sig') as f:
|
|
reader = csv.DictReader(f)
|
|
for row in reader:
|
|
total_read += 1
|
|
try:
|
|
lat = float(row['latitude'])
|
|
lng = float(row['longitude'])
|
|
name = row.get('name', '').strip()
|
|
if not name:
|
|
continue
|
|
# Valid bounding box for Iraq: 28.5 to 38.5 Lat, 38.5 to 49.0 Lng
|
|
if not (28.0 <= lat <= 39.0 and 38.0 <= lng <= 50.0):
|
|
continue
|
|
|
|
category = row.get('category_queried', '').strip() or 'مكان عام'
|
|
sector = row.get('sector', '').strip()
|
|
gov = row.get('governorate', '').strip() or 'العراق'
|
|
area = row.get('area', '').strip()
|
|
|
|
address = f"{area}, {gov}, العراق" if area else f"{gov}, العراق"
|
|
desc = sector if sector else 'نقطة اهتمام في العراق'
|
|
|
|
# Compute popularity score from reviews and rating
|
|
reviews = 0
|
|
try:
|
|
reviews = int(float(row.get('reviews_count', 0)))
|
|
except Exception:
|
|
reviews = 0
|
|
|
|
rating = 0.0
|
|
try:
|
|
rating = float(row.get('rating', 0))
|
|
except Exception:
|
|
rating = 0.0
|
|
|
|
pop_score = min(100, int(reviews * 0.2 + rating * 5))
|
|
|
|
valid_records.append((name, lat, lng, category, gov, area, address, desc, pop_score))
|
|
except Exception:
|
|
continue
|
|
|
|
print(f"Total rows read: {total_read}, Valid for ingestion: {len(valid_records)}")
|
|
|
|
with open(out_sql_path, 'w', encoding='utf-8') as out:
|
|
out.write("-- ============================================================================\n")
|
|
out.write("-- MIGRATION 11: INGEST IRAQ CHECKPOINTS DATASET (70,000+ PLACES)\n")
|
|
out.write("-- Ingests into places_iraq and refreshes unified_search_index\n")
|
|
out.write("-- ============================================================================\n\n")
|
|
|
|
out.write("BEGIN;\n\n")
|
|
out.write("-- Clean previous checkpoint import if needed to avoid duplicates\n")
|
|
out.write("DELETE FROM places_iraq WHERE source = 'checkpoint_70593';\n\n")
|
|
|
|
batch_size = 1000
|
|
for i in range(0, len(valid_records), batch_size):
|
|
batch = valid_records[i:i + batch_size]
|
|
out.write("INSERT INTO places_iraq (\n")
|
|
out.write(" name, name_ar, latitude, longitude, category, city, neighbourhood,\n")
|
|
out.write(" address, description, popularity_score, source, location\n")
|
|
out.write(") VALUES\n")
|
|
|
|
value_lines = []
|
|
for item in batch:
|
|
name, lat, lng, category, gov, area, address, desc, pop_score = item
|
|
c_name = clean_str(name)
|
|
c_cat = clean_str(category)
|
|
c_city = clean_str(gov)
|
|
c_area = clean_str(area)
|
|
c_addr = clean_str(address)
|
|
c_desc = clean_str(desc)
|
|
|
|
line = (f" ({c_name}, {c_name}, {lat:.7f}, {lng:.7f}, {c_cat}, {c_city}, {c_area}, "
|
|
f"{c_addr}, {c_desc}, {pop_score}, 'checkpoint_70593', "
|
|
f"ST_SetSRID(ST_MakePoint({lng:.7f}, {lat:.7f}), 4326))")
|
|
value_lines.append(line)
|
|
|
|
out.write(",\n".join(value_lines))
|
|
out.write(";\n\n")
|
|
|
|
out.write("COMMIT;\n\n")
|
|
|
|
out.write("-- Refresh Unified Search Index to instantly index all new Iraqi places\n")
|
|
out.write("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;\n")
|
|
|
|
print(f"✅ Generated SQL file: {out_sql_path} ({os.path.getsize(out_sql_path) / (1024*1024):.2f} MB)")
|
|
|
|
if __name__ == '__main__':
|
|
main()
|