119 lines
3.7 KiB
JavaScript
119 lines
3.7 KiB
JavaScript
const fs = require('fs');
|
|
const readline = require('readline');
|
|
|
|
// CONFIGURATION
|
|
const inputFile = '/Users/hamzaaleghwairyeen/development/App/map-saas/infrastructure/docker/postgis/egypt_places.sql';
|
|
const outputFile = '/tmp/converted_pois_full.sql';
|
|
|
|
/**
|
|
* Parses a MySQL INSERT value row like (1, 2.3, 4.5, 'Name', 'Addr', 'Time')
|
|
* Respects single quotes and escaped characters.
|
|
*/
|
|
function parseRow(row) {
|
|
const parts = [];
|
|
let currentPart = '';
|
|
let inQuotes = false;
|
|
for (let i = 0; i < row.length; i++) {
|
|
const char = row[i];
|
|
if (char === "'" && (i === 0 || row[i - 1] !== "\\")) {
|
|
inQuotes = !inQuotes;
|
|
currentPart += char;
|
|
} else if (char === "," && !inQuotes) {
|
|
parts.push(currentPart.trim());
|
|
currentPart = '';
|
|
} else {
|
|
currentPart += char;
|
|
}
|
|
}
|
|
parts.push(currentPart.trim());
|
|
return parts;
|
|
}
|
|
|
|
async function processFile() {
|
|
console.log(`Starting conversion of ${inputFile}...`);
|
|
|
|
const fileStream = fs.createReadStream(inputFile);
|
|
const rl = readline.createInterface({
|
|
input: fileStream,
|
|
crlfDelay: Infinity
|
|
});
|
|
|
|
const uniquePlaces = new Map();
|
|
let totalRows = 0;
|
|
let inValues = false;
|
|
|
|
// Output stream
|
|
const outStream = fs.createWriteStream(outputFile);
|
|
outStream.write('TRUNCATE TABLE places_egypt;\n');
|
|
outStream.write('INSERT INTO places_egypt (id, latitude, longitude, name, name_ar, address, created_at, location) VALUES\n');
|
|
|
|
let firstRow = true;
|
|
|
|
for await (const line of rl) {
|
|
if (line.includes('INSERT INTO `places`')) {
|
|
inValues = true;
|
|
// Handle lines that start with INSERT and have values
|
|
const match = line.match(/VALUES\s*(.*)/);
|
|
if (match) processValues(match[1]);
|
|
continue;
|
|
}
|
|
|
|
if (inValues) {
|
|
if (line.trim().endsWith(';')) {
|
|
processValues(line.trim().slice(0, -1));
|
|
inValues = false;
|
|
} else {
|
|
processValues(line.trim());
|
|
}
|
|
}
|
|
}
|
|
|
|
function processValues(vals) {
|
|
// Split by ), ( but very carefully. Standard SQL dumps use ),\n(
|
|
// We'll clean leading ( and trailing , or )
|
|
const rows = vals.split(/\),\s*\(/);
|
|
|
|
rows.forEach(row => {
|
|
let cleanRow = row.replace(/^\(/, '').replace(/,$/, '').replace(/\)$/, '');
|
|
if (!cleanRow) return;
|
|
|
|
const parts = parseRow(cleanRow);
|
|
if (parts.length < 4) return;
|
|
|
|
const id = parts[0];
|
|
const lat = parts[1];
|
|
const lng = parts[2];
|
|
const name = parts[3]; // with quotes
|
|
const address = parts[4] || "''";
|
|
const createdAt = parts[5] || 'CURRENT_TIMESTAMP';
|
|
|
|
// DEDUPLICATION KEY: Name + Lat + Lng
|
|
const cleanName = name.replace(/'/g, '').trim();
|
|
const key = `${cleanName}|${lat}|${lng}`;
|
|
|
|
if (!uniquePlaces.has(key)) {
|
|
uniquePlaces.set(key, true);
|
|
totalRows++;
|
|
|
|
const location = `ST_SetSRID(ST_Point(${lng}, ${lat}), 4326)`;
|
|
const pgRow = `(${id}, ${lat}, ${lng}, ${name}, ${name}, ${address}, ${createdAt}, ${location})`;
|
|
|
|
if (!firstRow) {
|
|
outStream.write(',\n');
|
|
}
|
|
outStream.write(pgRow);
|
|
firstRow = false;
|
|
}
|
|
});
|
|
}
|
|
|
|
outStream.write(';\n');
|
|
outStream.end();
|
|
|
|
console.log(`✅ Conversion complete!`);
|
|
console.log(`📊 Total Unique Records: ${totalRows}`);
|
|
console.log(`📂 Output saved to: ${outputFile}`);
|
|
}
|
|
|
|
processFile().catch(console.error);
|