2026-04-12-1

migrate_jordan.py

@@ -0,0 +1,98 @@
+import csv
+import os
+
+def parse_jordan_csv(csv_file):
+    print(f"Parsing {csv_file}...")
+    records = []
+    if not os.path.exists(csv_file):
+        print(f"CSV file {csv_file} not found!")
+        return []
+        
+    with open(csv_file, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            try:
+                # Column names: الاسم,latitude,longitude,الفئة الرئيسية,المحافظة,الرابط المباشر,تاريخ الرصد
+                name_ar = row['الاسم'].strip()
+                lat = float(row['latitude'])
+                lng = float(row['longitude'])
+                category = row['الفئة الرئيسية'].strip()
+                city = row['المحافظة'].strip()
+                link = row['الرابط المباشر'].strip()
+                created_at = row['تاريخ الرصد'].strip()
+                
+                records.append({
+                    'name_ar': name_ar,
+                    'name': name_ar, # Use Arabic name for both as no English name provided
+                    'lat': lat,
+                    'lng': lng,
+                    'category': category,
+                    'city': city,
+                    'link': link,
+                    'created_at': created_at,
+                    'source': 'csv'
+                })
+            except Exception as e:
+                # print(f"Error parsing row: {e}")
+                continue
+    return records
+
+def merge_and_deduplicate(csv_records):
+    final_records = []
+    seen_points = set()
+    
+    new_added = 0
+    for r in csv_records:
+        if not r['name_ar']: continue
+        # Creating a key for de-duplication based on name and rounded coordinates
+        key = (r['name_ar'].lower().strip(), round(r['lat'], 5), round(r['lng'], 5))
+        if key not in seen_points:
+            final_records.append(r)
+            seen_points.add(key)
+            new_added += 1
+            
+    print(f"CSV records parsed: {len(csv_records)}")
+    print(f"Unique records: {new_added}")
+    return final_records
+
+def write_final_sql(merged_records, output_file):
+    print(f"Writing final SQL to {output_file}...")
+    with open(output_file, 'w', encoding='utf-8') as f:
+        f.write("-- Jordan Location Data Combined\n")
+        f.write("TRUNCATE TABLE places_jordan RESTART IDENTITY;\n\n")
+        
+        batch_size = 500
+        for i in range(0, len(merged_records), batch_size):
+            batch = merged_records[i:i+batch_size]
+            f.write('INSERT INTO "places_jordan" (name, name_ar, latitude, longitude, category, city, link, created_at, location) VALUES\n')
+            
+            values_list = []
+            for r in batch:
+                name = r['name'].replace("'", "''")
+                name_ar = r['name_ar'].replace("'", "''")
+                category = r['category'].replace("'", "''")
+                city = r['city'].replace("'", "''")
+                link = r['link'].replace("'", "''")
+                created_at = r['created_at']
+                lat = r['lat']
+                lng = r['lng']
+                
+                # ST_SetSRID(ST_MakePoint(lon, lat), 4326)
+                val = f"('{name}', '{name_ar}', {lat}, {lng}, '{category}', '{city}', '{link}', '{created_at}', ST_SetSRID(ST_MakePoint({lng}, {lat}), 4326))"
+                values_list.append(val)
+            
+            f.write(",\n".join(values_list))
+            f.write(";\n\n")
+
+if __name__ == "__main__":
+    csv_file = 'infrastructure/docker/postgis/jordan_data.csv'
+    output_file = 'jordan_combined_final.sql'
+    
+    # If running on server, adjust path if needed
+    if not os.path.exists(csv_file):
+        csv_file = 'jordan_data.csv'
+        
+    csv_records = parse_jordan_csv(csv_file)
+    merged = merge_and_deduplicate(csv_records)
+    write_final_sql(merged, output_file)
+    print(f"Generation complete! File: {output_file}")