99 lines
3.9 KiB
Python
99 lines
3.9 KiB
Python
import csv
|
|
import os
|
|
|
|
def parse_jordan_csv(csv_file):
|
|
print(f"Parsing {csv_file}...")
|
|
records = []
|
|
if not os.path.exists(csv_file):
|
|
print(f"CSV file {csv_file} not found!")
|
|
return []
|
|
|
|
with open(csv_file, 'r', encoding='utf-8') as f:
|
|
reader = csv.DictReader(f)
|
|
for row in reader:
|
|
try:
|
|
# Column names: الاسم,latitude,longitude,الفئة الرئيسية,المحافظة,الرابط المباشر,تاريخ الرصد
|
|
name_ar = row['الاسم'].strip()
|
|
lat = float(row['latitude'])
|
|
lng = float(row['longitude'])
|
|
category = row['الفئة الرئيسية'].strip()
|
|
city = row['المحافظة'].strip()
|
|
link = row['الرابط المباشر'].strip()
|
|
created_at = row['تاريخ الرصد'].strip()
|
|
|
|
records.append({
|
|
'name_ar': name_ar,
|
|
'name': name_ar, # Use Arabic name for both as no English name provided
|
|
'lat': lat,
|
|
'lng': lng,
|
|
'category': category,
|
|
'city': city,
|
|
'link': link,
|
|
'created_at': created_at,
|
|
'source': 'csv'
|
|
})
|
|
except Exception as e:
|
|
# print(f"Error parsing row: {e}")
|
|
continue
|
|
return records
|
|
|
|
def merge_and_deduplicate(csv_records):
|
|
final_records = []
|
|
seen_points = set()
|
|
|
|
new_added = 0
|
|
for r in csv_records:
|
|
if not r['name_ar']: continue
|
|
# Creating a key for de-duplication based on name and rounded coordinates
|
|
key = (r['name_ar'].lower().strip(), round(r['lat'], 5), round(r['lng'], 5))
|
|
if key not in seen_points:
|
|
final_records.append(r)
|
|
seen_points.add(key)
|
|
new_added += 1
|
|
|
|
print(f"CSV records parsed: {len(csv_records)}")
|
|
print(f"Unique records: {new_added}")
|
|
return final_records
|
|
|
|
def write_final_sql(merged_records, output_file):
|
|
print(f"Writing final SQL to {output_file}...")
|
|
with open(output_file, 'w', encoding='utf-8') as f:
|
|
f.write("-- Jordan Location Data Combined\n")
|
|
f.write("TRUNCATE TABLE places_jordan RESTART IDENTITY;\n\n")
|
|
|
|
batch_size = 500
|
|
for i in range(0, len(merged_records), batch_size):
|
|
batch = merged_records[i:i+batch_size]
|
|
f.write('INSERT INTO "places_jordan" (name, name_ar, latitude, longitude, category, city, link, created_at, location) VALUES\n')
|
|
|
|
values_list = []
|
|
for r in batch:
|
|
name = r['name'].replace("'", "''")
|
|
name_ar = r['name_ar'].replace("'", "''")
|
|
category = r['category'].replace("'", "''")
|
|
city = r['city'].replace("'", "''")
|
|
link = r['link'].replace("'", "''")
|
|
created_at = r['created_at']
|
|
lat = r['lat']
|
|
lng = r['lng']
|
|
|
|
# ST_SetSRID(ST_MakePoint(lon, lat), 4326)
|
|
val = f"('{name}', '{name_ar}', {lat}, {lng}, '{category}', '{city}', '{link}', '{created_at}', ST_SetSRID(ST_MakePoint({lng}, {lat}), 4326))"
|
|
values_list.append(val)
|
|
|
|
f.write(",\n".join(values_list))
|
|
f.write(";\n\n")
|
|
|
|
if __name__ == "__main__":
|
|
csv_file = 'infrastructure/docker/postgis/jordan_data.csv'
|
|
output_file = 'jordan_combined_final.sql'
|
|
|
|
# If running on server, adjust path if needed
|
|
if not os.path.exists(csv_file):
|
|
csv_file = 'jordan_data.csv'
|
|
|
|
csv_records = parse_jordan_csv(csv_file)
|
|
merged = merge_and_deduplicate(csv_records)
|
|
write_final_sql(merged, output_file)
|
|
print(f"Generation complete! File: {output_file}")
|