465 lines
17 KiB
Python
465 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Enrich Egypt Places Dataset (`places_egypt`) from:
|
||
1. OpenStreetMap Named Points (`planet_osm_point`)
|
||
2. OpenStreetMap Named Polygons (`planet_osm_polygon`)
|
||
3. Overture Maps Places (`overture_place`)
|
||
4. Deduplicate using spatial grid (<60m) against existing places_egypt
|
||
5. Refresh `unified_search_index`
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import math
|
||
import time
|
||
import argparse
|
||
from collections import defaultdict
|
||
|
||
def haversine(lat1, lon1, lat2, lon2):
|
||
R = 6371000 # meters
|
||
phi1 = math.radians(lat1)
|
||
phi2 = math.radians(lat2)
|
||
delta_phi = math.radians(lat2 - lat1)
|
||
delta_lambda = math.radians(lon2 - lon1)
|
||
a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
|
||
return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
|
||
|
||
def normalize_text(t):
|
||
if not t:
|
||
return ''
|
||
s = t.strip()
|
||
s = s.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا').replace('ة', 'ه').replace('ى', 'ي')
|
||
return ' '.join(s.split()).lower()
|
||
|
||
def clean_sql_str(val):
|
||
if val is None:
|
||
return 'NULL'
|
||
s = str(val).strip().replace("'", "''")
|
||
return f"'{s}'"
|
||
|
||
OSM_CAT_MAP = {
|
||
'village': 'قرية / تجمع سكاني',
|
||
'town': 'بلدة / مركز',
|
||
'city': 'مدينة',
|
||
'suburb': 'حي / ضاحية',
|
||
'neighbourhood': 'منطقة سكنية',
|
||
'hamlet': 'نجع / كفر',
|
||
'locality': 'موضع / منطقة',
|
||
'school': 'مدرسة / تعليم',
|
||
'college': 'كلية',
|
||
'university': 'جامعة',
|
||
'kindergarten': 'روضة أطفال',
|
||
'supermarket': 'سوبرماركت وهايبرماركت',
|
||
'convenience': 'محل بقالة',
|
||
'bakery': 'مخبز وفرن',
|
||
'butcher': 'جزارة وقصابة',
|
||
'mall': 'مركز تسوق ومول',
|
||
'place_of_worship': 'مسجد / دار عبادة',
|
||
'mosque': 'جامع / مسجد',
|
||
'clinic': 'مركز طبي وعيادة',
|
||
'pharmacy': 'صيدلية',
|
||
'hospital': 'مستشفى',
|
||
'doctors': 'عيادات تخصصية',
|
||
'dentist': 'عيادة أسنان',
|
||
'restaurant': 'مطعم',
|
||
'cafe': 'مقهى وكافيه',
|
||
'fast_food': 'مأكولات سريعة',
|
||
'fuel': 'محطة وقود وبنزين',
|
||
'bank': 'مصرف / بنك',
|
||
'atm': 'ماكينة صراف آلي ATM',
|
||
'bureau_de_change': 'مكتب صرافة',
|
||
'police': 'قسم شرطة ونقطة أمنية',
|
||
'fire_station': 'وحدة مطافئ / دفاع مدني',
|
||
'post_office': 'مكتب بريد',
|
||
'attraction': 'معلم سياحي وتاريخي',
|
||
'hotel': 'فندق وإقامة',
|
||
'guest_house': 'نُزل واستضافة',
|
||
'museum': 'متحف وآثار',
|
||
'park': 'حديقة ومنتزه عام',
|
||
'stadium': 'استاد وملعب رياضي',
|
||
'sports_centre': 'مركز ونادي رياضي',
|
||
'car_repair': 'ورشة صيانة سيارات',
|
||
'car_wash': 'مغسلة سيارات',
|
||
'company': 'شركة ومقر أعمال',
|
||
'government': 'جهة حكومية ورسمية',
|
||
'mobile_phone': 'متجر هواتف ومحمول',
|
||
'clothes': 'متجر ملابس وأزياء',
|
||
'shoes': 'متجر أحذية',
|
||
'hairdresser': 'صالون حلاقة وتجميل',
|
||
'optician': 'بصريات ونظارات',
|
||
'hardware': 'أدوات بناء وتجهيزات',
|
||
}
|
||
|
||
OVERTURE_CAT_MAP = {
|
||
'restaurant': 'مطعم',
|
||
'cafe': 'مقهى وكافيه',
|
||
'coffee_shop': 'مقهى وكوفي شوب',
|
||
'casual_eatery': 'مأكولات سريعة وخفيفة',
|
||
'bakery': 'مخبز ومعجنات',
|
||
'food_and_beverage_store': 'بقالة ومواد غذائية',
|
||
'supermarket': 'سوبرماركت وهايبرماركت',
|
||
'fashion_and_apparel_store': 'متجر ملابس وأزياء',
|
||
'shoe_store': 'متجر أحذية',
|
||
'jewelry_store': 'مجوهرات وحلي',
|
||
'electronics_store': 'متجر إلكترونيات وأجهزة',
|
||
'mobile_phone_store': 'متجر هواتف ذكية',
|
||
'hardware_home_and_garden_store': 'مواد بناء ومستلزمات منزلية',
|
||
'home_service': 'خدمات منزلية وصيانة',
|
||
'personal_or_beauty_service': 'صالون ومركز تجميل',
|
||
'hospital': 'مستشفى / مركز طبي',
|
||
'dental_clinic': 'عيادة أسنان',
|
||
'specialized_health_care': 'عيادة تخصصية',
|
||
'pharmacy': 'صيدلية',
|
||
'hotel': 'فندق وإقامة',
|
||
'place_of_learning': 'منشأة تعليمية',
|
||
'school': 'مدرسة',
|
||
'college_university': 'جامعة / كلية',
|
||
'historic_site': 'موقع تاريخي وأثري',
|
||
'religious_organization': 'مؤسسة دينية',
|
||
'mosque': 'مسجد / جامع',
|
||
'real_estate_service': 'مكتب عقاري',
|
||
'professional_service': 'خدمات مهنية واستشارية',
|
||
'travel_service': 'مكتب سياحة وسفر',
|
||
'corporate_or_business_office': 'شركة ومؤسسة أعمال',
|
||
'car_repair': 'صيانة وتصليح سيارات',
|
||
'fuel_station': 'محطة وقود',
|
||
'bank': 'مصرف / بنك',
|
||
'government_office': 'مصلحة وهيئة حكومية',
|
||
}
|
||
|
||
def map_category(cat_str, source_type='osm'):
|
||
if not cat_str:
|
||
return 'مكان ونقطة اهتمام'
|
||
clean = str(cat_str).strip().lower()
|
||
if source_type == 'overture':
|
||
return OVERTURE_CAT_MAP.get(clean, 'مكان عام وتجاري')
|
||
return OSM_CAT_MAP.get(clean, 'مكان عام')
|
||
|
||
def get_grid_cell(lat, lon, cell_size=0.01):
|
||
return (int(lat / cell_size), int(lon / cell_size))
|
||
|
||
def is_duplicate(name_norm, lat, lon, grid, max_dist_m=60.0):
|
||
d_lat_thresh = 0.0006
|
||
d_lon_thresh = 0.0007
|
||
c_x, c_y = get_grid_cell(lat, lon)
|
||
for dx in (-1, 0, 1):
|
||
for dy in (-1, 0, 1):
|
||
cell = (c_x + dx, c_y + dy)
|
||
if cell in grid:
|
||
for existing_name, ex_lat, ex_lon in grid[cell]:
|
||
if abs(lat - ex_lat) > d_lat_thresh or abs(lon - ex_lon) > d_lon_thresh:
|
||
continue
|
||
dist = haversine(lat, lon, ex_lat, ex_lon)
|
||
if dist <= max_dist_m:
|
||
if (name_norm == existing_name or
|
||
name_norm in existing_name or
|
||
existing_name in name_norm):
|
||
return True
|
||
return False
|
||
|
||
def add_to_grid(name_norm, lat, lon, grid):
|
||
cell = get_grid_cell(lat, lon)
|
||
grid[cell].append((name_norm, lat, lon))
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Enrich Egypt places from OSM & Overture.")
|
||
parser.add_argument("--db-host", default="127.0.0.1")
|
||
parser.add_argument("--db-port", type=int, default=5432)
|
||
parser.add_argument("--db-user", default="mapuser")
|
||
parser.add_argument("--db-pass", default="mappass")
|
||
parser.add_argument("--db-name", default="mapdb")
|
||
parser.add_argument("--dry-run", action="store_true")
|
||
args = parser.parse_args()
|
||
|
||
import pg8000.native
|
||
|
||
print("=" * 70)
|
||
print("🇪🇬 SIRO Maps — Comprehensive Egypt Places Enrichment & Deduplication")
|
||
print("=" * 70)
|
||
|
||
print(f"🔌 Connecting to PostgreSQL at {args.db_host}:{args.db_port} ({args.db_name})...")
|
||
con = pg8000.native.Connection(
|
||
user=args.db_user,
|
||
password=args.db_pass,
|
||
host=args.db_host,
|
||
port=args.db_port,
|
||
database=args.db_name
|
||
)
|
||
|
||
t0 = time.time()
|
||
grid = defaultdict(list)
|
||
|
||
# 1. Load existing places_egypt
|
||
print("\n📦 Step 1: Loading existing places_egypt into spatial index...")
|
||
existing = con.run("""
|
||
SELECT id, name, latitude, longitude
|
||
FROM places_egypt
|
||
WHERE latitude IS NOT NULL AND longitude IS NOT NULL;
|
||
""")
|
||
print(f" -> Loaded {len(existing):,} existing places.")
|
||
for row in existing:
|
||
pid, name, lat, lon = row
|
||
lat = float(lat)
|
||
lon = float(lon)
|
||
norm_name = normalize_text(name)
|
||
add_to_grid(norm_name, lat, lon, grid)
|
||
|
||
print(f" ✓ Spatial grid built with {sum(len(v) for v in grid.values()):,} items.")
|
||
|
||
# Egypt Bounding Box: 21.8 <= lat <= 31.8, 24.7 <= lon <= 36.9
|
||
EG_MIN_LON, EG_MIN_LAT = 24.7, 21.8
|
||
EG_MAX_LON, EG_MAX_LAT = 36.9, 31.8
|
||
|
||
# 2. Harvest from planet_osm_point in Egypt
|
||
print("\n📍 Step 2: Harvesting from OpenStreetMap Named Points (planet_osm_point)...")
|
||
osm_pts = con.run(f"""
|
||
SELECT
|
||
name,
|
||
COALESCE(amenity, shop, tourism, historic, place, leisure, office, building, highway, "natural") as cat,
|
||
ST_Y(ST_Transform(way, 4326)) as lat,
|
||
ST_X(ST_Transform(way, 4326)) as lon
|
||
FROM planet_osm_point
|
||
WHERE name IS NOT NULL
|
||
AND TRIM(name) != ''
|
||
AND way && ST_Transform(ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326), 3857);
|
||
""")
|
||
print(f" -> Found {len(osm_pts):,} candidates in planet_osm_point for Egypt.")
|
||
|
||
osm_pts_to_insert = []
|
||
osm_pts_dups = 0
|
||
|
||
for r in osm_pts:
|
||
name = str(r[0]).strip()
|
||
cat_raw = r[1]
|
||
lat = float(r[2])
|
||
lon = float(r[3])
|
||
|
||
if not (EG_MIN_LAT <= lat <= EG_MAX_LAT and EG_MIN_LON <= lon <= EG_MAX_LON):
|
||
continue
|
||
|
||
norm_name = normalize_text(name)
|
||
if len(norm_name) < 2:
|
||
continue
|
||
|
||
if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
|
||
osm_pts_dups += 1
|
||
continue
|
||
|
||
category = map_category(cat_raw, 'osm')
|
||
pop_score = 65 if cat_raw in ('village', 'town', 'city', 'hospital', 'university', 'mosque') else 40
|
||
|
||
item = {
|
||
'name': name,
|
||
'name_ar': name,
|
||
'name_en': name,
|
||
'category': category,
|
||
'city': 'مصر',
|
||
'address': f"{category}, مصر",
|
||
'lat': lat,
|
||
'lon': lon,
|
||
'source': 'osm_point',
|
||
'popularity_score': pop_score
|
||
}
|
||
osm_pts_to_insert.append(item)
|
||
add_to_grid(norm_name, lat, lon, grid)
|
||
|
||
print(f" ✓ OSM Points Processing Complete:")
|
||
print(f" - Duplicates Filtered: {osm_pts_dups:,}")
|
||
print(f" - Net New Places: {len(osm_pts_to_insert):,}")
|
||
|
||
# 3. Harvest from planet_osm_polygon in Egypt
|
||
print("\n🏛️ Step 3: Harvesting from OpenStreetMap Named Polygons (planet_osm_polygon)...")
|
||
osm_polys = con.run(f"""
|
||
SELECT
|
||
name,
|
||
COALESCE(amenity, shop, tourism, historic, place, leisure, building, landuse) as cat,
|
||
ST_Y(ST_Centroid(ST_Transform(way, 4326))) as lat,
|
||
ST_X(ST_Centroid(ST_Transform(way, 4326))) as lon
|
||
FROM planet_osm_polygon
|
||
WHERE name IS NOT NULL
|
||
AND TRIM(name) != ''
|
||
AND (amenity IS NOT NULL OR shop IS NOT NULL OR tourism IS NOT NULL OR historic IS NOT NULL
|
||
OR place IS NOT NULL OR leisure IS NOT NULL OR landuse IN ('commercial', 'retail', 'industrial', 'cemetery', 'religious'))
|
||
AND way && ST_Transform(ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326), 3857);
|
||
""")
|
||
print(f" -> Found {len(osm_polys):,} candidates in planet_osm_polygon for Egypt.")
|
||
|
||
osm_polys_to_insert = []
|
||
osm_polys_dups = 0
|
||
|
||
for r in osm_polys:
|
||
name = str(r[0]).strip()
|
||
cat_raw = r[1]
|
||
lat = float(r[2])
|
||
lon = float(r[3])
|
||
|
||
if not (EG_MIN_LAT <= lat <= EG_MAX_LAT and EG_MIN_LON <= lon <= EG_MAX_LON):
|
||
continue
|
||
|
||
norm_name = normalize_text(name)
|
||
if len(norm_name) < 2:
|
||
continue
|
||
|
||
if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
|
||
osm_polys_dups += 1
|
||
continue
|
||
|
||
category = map_category(cat_raw, 'osm')
|
||
pop_score = 70 if cat_raw in ('hospital', 'university', 'mall', 'stadium', 'attraction') else 45
|
||
|
||
item = {
|
||
'name': name,
|
||
'name_ar': name,
|
||
'name_en': name,
|
||
'category': category,
|
||
'city': 'مصر',
|
||
'address': f"{category}, مصر",
|
||
'lat': lat,
|
||
'lon': lon,
|
||
'source': 'osm_polygon',
|
||
'popularity_score': pop_score
|
||
}
|
||
osm_polys_to_insert.append(item)
|
||
add_to_grid(norm_name, lat, lon, grid)
|
||
|
||
print(f" ✓ OSM Polygons Processing Complete:")
|
||
print(f" - Duplicates Filtered: {osm_polys_dups:,}")
|
||
print(f" - Net New Places: {len(osm_polys_to_insert):,}")
|
||
|
||
# 4. Harvest from overture_place in Egypt
|
||
print("\n🗺️ Step 4: Harvesting from Overture Maps Places (overture_place)...")
|
||
ovt_rows = con.run(f"""
|
||
SELECT
|
||
name_primary,
|
||
basic_category,
|
||
addresses,
|
||
confidence,
|
||
ST_Y(ST_Centroid(location::geometry)) as lat,
|
||
ST_X(ST_Centroid(location::geometry)) as lon
|
||
FROM overture_place
|
||
WHERE name_primary IS NOT NULL
|
||
AND TRIM(name_primary) != ''
|
||
AND ST_Within(location::geometry, ST_MakeEnvelope({EG_MIN_LON}, {EG_MIN_LAT}, {EG_MAX_LON}, {EG_MAX_LAT}, 4326));
|
||
""")
|
||
print(f" -> Found {len(ovt_rows):,} candidates in overture_place for Egypt.")
|
||
|
||
ovt_to_insert = []
|
||
ovt_dups = 0
|
||
|
||
for r in ovt_rows:
|
||
name = str(r[0]).strip()
|
||
cat_str = r[1]
|
||
addr_list = r[2]
|
||
confidence = float(r[3] or 0.8)
|
||
lat = float(r[4])
|
||
lon = float(r[5])
|
||
|
||
norm_name = normalize_text(name)
|
||
if len(norm_name) < 2:
|
||
continue
|
||
|
||
if is_duplicate(norm_name, lat, lon, grid, max_dist_m=60.0):
|
||
ovt_dups += 1
|
||
continue
|
||
|
||
city = 'مصر'
|
||
addr_text = None
|
||
if addr_list and isinstance(addr_list, list) and len(addr_list) > 0:
|
||
a = addr_list[0]
|
||
if isinstance(a, dict):
|
||
city = a.get('locality') or a.get('region') or 'مصر'
|
||
addr_text = a.get('freeform')
|
||
|
||
category = map_category(cat_str, 'overture')
|
||
pop_score = min(100, int(confidence * 60 + 20))
|
||
|
||
item = {
|
||
'name': name,
|
||
'name_ar': name,
|
||
'name_en': name,
|
||
'category': category,
|
||
'city': city,
|
||
'address': addr_text or f"{city}, مصر",
|
||
'lat': lat,
|
||
'lon': lon,
|
||
'source': 'overture_maps',
|
||
'popularity_score': pop_score
|
||
}
|
||
ovt_to_insert.append(item)
|
||
add_to_grid(norm_name, lat, lon, grid)
|
||
|
||
print(f" ✓ Overture Processing Complete:")
|
||
print(f" - Duplicates Filtered: {ovt_dups:,}")
|
||
print(f" - Net New Places: {len(ovt_to_insert):,}")
|
||
|
||
all_new = osm_pts_to_insert + osm_polys_to_insert + ovt_to_insert
|
||
print(f"\n🌟 Total Net New Places to Insert into places_egypt: {len(all_new):,}")
|
||
|
||
if args.dry_run:
|
||
print("💡 Dry run complete. No database changes made.")
|
||
return
|
||
|
||
# 5. Insert into places_egypt
|
||
print(f"\n🚀 Step 5: Inserting {len(all_new):,} places into places_egypt...")
|
||
t_insert = time.time()
|
||
batch_size = 2500
|
||
total_inserted = 0
|
||
|
||
con.run("BEGIN;")
|
||
for i in range(0, len(all_new), batch_size):
|
||
batch = all_new[i:i + batch_size]
|
||
values = []
|
||
for r in batch:
|
||
c_name = clean_sql_str(r['name'])
|
||
c_name_ar = clean_sql_str(r['name_ar'])
|
||
c_name_en = clean_sql_str(r['name_en'])
|
||
c_cat = clean_sql_str(r['category'])
|
||
c_city = clean_sql_str(r['city'])
|
||
c_addr = clean_sql_str(r['address'])
|
||
c_src = clean_sql_str(r['source'])
|
||
lat = r['lat']
|
||
lon = r['lon']
|
||
pop = r['popularity_score']
|
||
|
||
line = (
|
||
f"({c_name}, {c_name_ar}, {c_name_en}, {lat:.7f}, {lon:.7f}, {c_cat}, "
|
||
f"{c_city}, {c_addr}, {c_src}, {pop}, "
|
||
f"ST_SetSRID(ST_MakePoint({lon:.7f}, {lat:.7f}), 4326))"
|
||
)
|
||
values.append(line)
|
||
|
||
sql = (
|
||
"INSERT INTO places_egypt ("
|
||
" name, name_ar, name_en, latitude, longitude, category,"
|
||
" city, address, source, popularity_score, location"
|
||
") VALUES " + ",\n".join(values) + ";"
|
||
)
|
||
con.run(sql)
|
||
total_inserted += len(batch)
|
||
print(f" -> Progress: {total_inserted:,} / {len(all_new):,} places inserted...")
|
||
|
||
con.run("COMMIT;")
|
||
print(f"✅ Ingestion committed successfully in {time.time() - t_insert:.2f}s!")
|
||
|
||
# 6. Refresh Materialized View
|
||
print("\n🔄 Step 6: Refreshing materialized view unified_search_index concurrently...")
|
||
t_mv = time.time()
|
||
try:
|
||
con.run("REFRESH MATERIALIZED VIEW CONCURRENTLY unified_search_index;")
|
||
print(f" ✓ unified_search_index refreshed in {time.time() - t_mv:.2f}s!")
|
||
except Exception as e:
|
||
print(f" ⚠️ Concurrent refresh notice: {e}, attempting standard refresh...")
|
||
con.run("REFRESH MATERIALIZED VIEW unified_search_index;")
|
||
print(f" ✓ unified_search_index refreshed!")
|
||
|
||
con.run("ANALYZE places_egypt;")
|
||
con.run("ANALYZE unified_search_index;")
|
||
|
||
total_count = con.run("SELECT count(*) FROM places_egypt;")[0][0]
|
||
print(f"\n🎯 Total Places in places_egypt: {total_count:,}")
|
||
con.close()
|
||
print(f"🎉 Egypt places enrichment completed in {time.time() - t0:.2f}s!")
|
||
|
||
if __name__ == '__main__':
|
||
main()
|