Hamza/maps-saas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement geocoding evaluation scripts and enhance routing service with traffic-aware path processing

Browse Source
This commit is contained in:
Hamza-Ayed
2026-07-16 14:31:07 +03:00
parent 3cf87999f4
commit 3773ab56a4
20 changed files with 885 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/geocoding/geocoding.service.ts
+226 -100
View File
@@ -65,8 +65,17 @@ export class GeocodingService {
async searchPlaces(query: string, lat?: number, lon?: number, radius: number = 20000, country?: string) {
try {
const cleanQuery = query.trim();
let cleanQuery = query.trim();
if (!cleanQuery) return { results: [] };
let relativePrefix = '';
const relativeQueryRegex = /^(قرب|بالقرب من|قريب من|عند|بجانب|جنب|حد|بجوار|مقابل|قبال|خلف|ورا|وراء)\s+(.+)$/i;
const match = cleanQuery.match(relativeQueryRegex);
if (match) {
relativePrefix = match[1];
cleanQuery = match[2];
}
const hasLocation = lat !== undefined && lon !== undefined;
const geoSegment = hasLocation ? `${lat!.toFixed(2)}_${lon!.toFixed(2)}` : 'global';
@@ -76,112 +85,110 @@ export class GeocodingService {
if (cached) return { results: cached, source: 'cache_hit' };
let targetRegion = country?.toLowerCase() || this.identifyRegion(lat, lon);
const primaryTables = targetRegion && ['syria', 'egypt', 'jordan'].includes(targetRegion)
? [`places_${targetRegion}`] : ['places_jordan', 'places_syria', 'places_egypt'];
// Normalize the search query using the DB function
const [normalizedQueryRes] = await this.osmPointsRepository.query(`SELECT normalize_arabic($1) as nq`, [cleanQuery]);
const normalizedQuery = normalizedQueryRes?.nq || cleanQuery.toLowerCase();
const queryPromises: Promise<any[]>[] = [];
let queryParams: any[] = [normalizedQuery];
let locationCondition = '';
if (hasLocation) {
locationCondition = `AND ST_DistanceSphere(location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $4`;
queryParams.push(lat, lon, radius);
}
primaryTables.forEach(tableName => {
const repo = this.getRepoByTableName(tableName);
queryPromises.push(repo.query(`
SELECT
p.id, p.name, p.name_ar, p.name_en, p.category,
n.name_ar as neighbourhood, d.name_ar as district, g.name_ar as governorate,
p.latitude, p.longitude, p.address, '${tableName.replace('places_', '')}' as region, 'user_place' as source,
CASE WHEN $2::float IS NOT NULL THEN ST_DistanceSphere(p.location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) ELSE 0 END as distance,
GREATEST(similarity(COALESCE(p.name_ar, ''), $1), similarity(COALESCE(p.name, ''), $1)) as relevance
FROM ${tableName} p
LEFT JOIN neighborhood_polygons n ON p.neighborhood_id = n.id
LEFT JOIN admin_boundaries d ON p.district_id = d.id
LEFT JOIN admin_boundaries g ON p.governorate_id = g.id
WHERE (p.name_ar % $1 OR p.name % $1)
AND ($2::float IS NULL OR ST_DistanceSphere(p.location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $4)
ORDER BY (p.name_ar <-> $1) ASC LIMIT 10
`, [cleanQuery, lat || null, lon || null, radius]));
});
let regionCondition = '';
if (targetRegion && ['syria', 'jordan', 'egypt'].includes(targetRegion)) {
regionCondition = `AND (region = '${targetRegion}' OR region = 'global')`;
}
queryPromises.push(this.osmPointsRepository.query(`
const sqlQuery = `
SELECT
o.osm_id as id, o.name, o.name_ar, o.name_en, COALESCE(o.amenity, o.shop, 'place') as category,
o.latitude, o.longitude, o.addr_street as address, 'osm_global' as source,
CASE WHEN $2::float IS NOT NULL THEN ST_DistanceSphere(o.geom, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) ELSE 0 END as distance,
GREATEST(similarity(COALESCE(o.name, ''), $1), similarity(COALESCE(o.name_ar, ''), $1)) as relevance
FROM osm_points_with_area o
WHERE (o.name % $1 OR o.name_ar % $1)
AND ($2::float IS NULL OR ST_DistanceSphere(o.geom, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $4)
ORDER BY (o.name <-> $1) ASC LIMIT 10
`, [cleanQuery, lat || null, lon || null, radius]));
queryPromises.push(this.osmPointsRepository.query(`
SELECT
id, name_ar as name, name_ar, name_en, 'admin' as category,
id, name, name_ar, category,
'' as neighbourhood, '' as district, '' as governorate,
ST_Y(ST_Centroid(geom))::text as latitude, ST_X(ST_Centroid(geom))::text as longitude, '' as address, 'admin_boundary' as region, 'admin' as source,
CASE WHEN $2::float IS NOT NULL THEN ST_DistanceSphere(geom, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) ELSE 0 END as distance,
similarity(COALESCE(name_ar, ''), $1) as relevance
FROM admin_boundaries
WHERE name_ar % $1
AND ($2::float IS NULL OR ST_DistanceSphere(geom, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $4)
ORDER BY (name_ar <-> $1) ASC LIMIT 5
`, [cleanQuery, lat || null, lon || null, radius]));
latitude, longitude, address, region, source, popularity_score,
${hasLocation ? 'ST_DistanceSphere(location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326))' : '0'} as distance,
similarity(normalized_name, $1) as relevance
FROM unified_search_index
WHERE normalized_name % $1
${locationCondition}
${regionCondition}
ORDER BY (normalized_name <-> $1) ASC
LIMIT 50
`;
queryPromises.push(this.osmPointsRepository.query(`
SELECT
id::text, COALESCE(names->>'primary', names->>'common', 'Street') as name, COALESCE(names->>'primary', names->>'common', 'Street') as name_ar, '' as name_en, 'street' as category,
'' as neighbourhood, '' as district, '' as governorate,
ST_Y(ST_Centroid(location))::text as latitude, ST_X(ST_Centroid(location))::text as longitude, '' as address, 'overture' as region, 'overture_global' as source,
CASE WHEN $2::float IS NOT NULL THEN ST_DistanceSphere(location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) ELSE 0 END as distance,
GREATEST(similarity(COALESCE(names->>'primary', ''), $1), similarity(COALESCE(names->>'common', ''), $1)) as relevance
FROM overture_segment
WHERE (names->>'primary' % $1 OR names->>'common' % $1)
AND ($2::float IS NULL OR ST_DistanceSphere(location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $4)
ORDER BY (COALESCE(names->>'primary', '') <-> $1) ASC LIMIT 10
`, [cleanQuery, lat || null, lon || null, radius]));
queryPromises.push(this.osmPointsRepository.query(`
SELECT
id::text, COALESCE(names->>'primary', names->>'common', 'Building') as name, COALESCE(names->>'primary', names->>'common', 'Building') as name_ar, '' as name_en, 'building' as category,
'' as neighbourhood, '' as district, '' as governorate,
ST_Y(ST_Centroid(location))::text as latitude, ST_X(ST_Centroid(location))::text as longitude, '' as address, 'overture' as region, 'overture_global' as source,
CASE WHEN $2::float IS NOT NULL THEN ST_DistanceSphere(location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) ELSE 0 END as distance,
GREATEST(similarity(COALESCE(names->>'primary', ''), $1), similarity(COALESCE(names->>'common', ''), $1)) as relevance
FROM overture_building
WHERE (names->>'primary' % $1 OR names->>'common' % $1)
AND ($2::float IS NULL OR ST_DistanceSphere(location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $4)
ORDER BY (COALESCE(names->>'primary', '') <-> $1) ASC LIMIT 10
`, [cleanQuery, lat || null, lon || null, radius]));
queryPromises.push(this.osmPointsRepository.query(`
SELECT
id::text, COALESCE(names->>'primary', names->>'common', 'Place') as name, COALESCE(names->>'primary', names->>'common', 'Place') as name_ar, '' as name_en, 'place' as category,
'' as neighbourhood, '' as district, '' as governorate,
ST_Y(ST_Centroid(location))::text as latitude, ST_X(ST_Centroid(location))::text as longitude, '' as address, 'overture' as region, 'overture_global' as source,
CASE WHEN $2::float IS NOT NULL THEN ST_DistanceSphere(location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) ELSE 0 END as distance,
GREATEST(similarity(COALESCE(names->>'primary', ''), $1), similarity(COALESCE(names->>'common', ''), $1)) as relevance
FROM overture_place
WHERE (names->>'primary' % $1 OR names->>'common' % $1)
AND ($2::float IS NULL OR ST_DistanceSphere(location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $4)
ORDER BY (COALESCE(names->>'primary', '') <-> $1) ASC LIMIT 10
`, [cleanQuery, lat || null, lon || null, radius]));
const executionResults = await Promise.race([
Promise.allSettled(queryPromises),
new Promise<any>((_, reject) => setTimeout(() => reject(new Error('QUERY_TIMEOUT')), this.DB_TIMEOUT_MS))
const allResults = await Promise.race([
this.osmPointsRepository.query(sqlQuery, queryParams),
new Promise<any[]>((_, reject) => setTimeout(() => reject(new Error('QUERY_TIMEOUT')), this.DB_TIMEOUT_MS))
]).catch(e => {
this.logger.warn(`Search optimization threshold hit: ${e.message}`);
return [] as any[];
});
let allResults: any[] = [];
if (Array.isArray(executionResults)) {
(executionResults as any[]).forEach(res => {
if (res.status === 'fulfilled' && res.value) allResults.push(...res.value);
});
const formatted = this.formatResults(allResults, hasLocation, relativePrefix);
// --- POI GATES (Clustering) ---
const placeIds = formatted.map(r => r.id);
if (placeIds.length > 0) {
try {
const gates = await this.osmPointsRepository.query(
`SELECT place_id, gate_name_ar, gate_name_en, latitude, longitude, is_main_gate
FROM place_gates
WHERE place_id = ANY($1)`,
[placeIds]
);
if (gates.length > 0) {
formatted.forEach(r => {
const placeGates = gates.filter((g: any) => g.place_id === r.id).map((g: any) => ({
name_ar: g.gate_name_ar,
name_en: g.gate_name_en,
latitude: parseFloat(g.latitude),
longitude: parseFloat(g.longitude),
is_main_gate: g.is_main_gate
}));
if (placeGates.length > 0) {
r.gates = placeGates;
}
});
}
} catch (err) {
this.logger.warn('Failed to fetch POI gates, table might not exist yet.');
}
}
// -----------------------------
const formatted = this.formatResults(allResults, hasLocation);
if (formatted.length > 0) {
await this.cacheManager.set(cacheKey, formatted, 3600000);
} else {
// Log zero-result query asynchronously
this.logFailedSearch(cleanQuery, normalizedQuery, targetRegion, lat, lon).catch(err => {
this.logger.error('Failed to log zero-result search:', err);
});
// --- DID YOU MEAN? (Safety Net) ---
try {
const whereClause = regionCondition ? `WHERE ${regionCondition.substring(4)}` : '';
const fallbackQuery = `
SELECT name_ar, name, (normalized_name <-> $1) as dist
FROM unified_search_index
${whereClause}
ORDER BY normalized_name <-> $1 ASC
LIMIT 1
`;
const suggestions = await this.osmPointsRepository.query(fallbackQuery, [normalizedQuery]);
if (suggestions.length > 0 && suggestions[0].dist < 0.6) {
return {
results: [],
did_you_mean: suggestions[0].name_ar || suggestions[0].name
};
}
} catch (err) {
this.logger.warn('Did You Mean fallback failed: ' + err.message);
}
// -----------------------------------
}
return { results: formatted };
@@ -191,14 +198,26 @@ export class GeocodingService {
}
}
private formatResults(results: any[], hasLocation: boolean) {
private formatResults(results: any[], hasLocation: boolean, relativePrefix: string = '') {
const seenStreets = new Set<string>();
// Normalize popularity to a 0-1 scale
const maxPopularity = Math.max(...results.map(r => r.popularity_score || 10), 100);
return results
.map(r => {
// Weighted scoring: 70% Name Similarity, 30% Geographic Proximity
// Weighted scoring:
// 50% Text Match (relevance)
// 30% Popularity
// 20% Geographic Proximity
const textScore = Number(r.relevance);
const popularityScore = (r.popularity_score || 10) / maxPopularity;
// Proximity bonus is 1.0 at 0m, decaying linearly to 0.0 at 10km.
const proximityBonus = hasLocation ? Math.max(0, 1 - (Number(r.distance) / 10000)) : 0;
const totalScore = (Number(r.relevance) * 0.7) + (proximityBonus * 0.3);
const totalScore = (textScore * 0.5) + (popularityScore * 0.3) + (proximityBonus * 0.2);
return { ...r, totalScore };
})
.sort((a, b) => b.totalScore - a.totalScore)
@@ -214,8 +233,12 @@ export class GeocodingService {
.map(r => {
const addressParts = [r.neighbourhood, r.district, r.governorate].filter(Boolean);
const full_address = addressParts.length > 0 ? addressParts.join('، ') : (r.address || '');
const nameAr = r.name_ar || r.name;
const displayName = relativePrefix ? `${relativePrefix} ${nameAr}` : nameAr;
return {
...r,
name: displayName,
name_ar: displayName,
latitude: parseFloat(r.latitude),
longitude: parseFloat(r.longitude),
distance_km: r.distance ? (Number(r.distance) / 1000).toFixed(2) : null,
@@ -232,6 +255,87 @@ export class GeocodingService {
return this.placesSyriaRepository;
}
/**
* Log zero-result queries to the failed_searches table for mining missing places.
*/
private async logFailedSearch(query: string, normalizedQuery: string, country?: string, lat?: number, lon?: number) {
const q = `
INSERT INTO failed_searches (query_text, normalized_query, country, latitude, longitude)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (normalized_query, COALESCE(country, 'global'))
DO UPDATE SET search_count = failed_searches.search_count + 1, last_seen_at = CURRENT_TIMESTAMP;
`;
await this.osmPointsRepository.query(q, [query, normalizedQuery, country || null, lat || null, lon || null]);
}
/**
* Fast Autocomplete using prefix matching
*/
async autocomplete(query: string, country?: string) {
try {
let cleanQuery = query.trim();
if (cleanQuery.length < 2) return { results: [] };
let relativePrefix = '';
const relativeQueryRegex = /^(قرب|بالقرب من|قريب من|عند|بجانب|جنب|حد|بجوار|مقابل|قبال|خلف|ورا|وراء)\s+(.+)$/i;
const match = cleanQuery.match(relativeQueryRegex);
if (match) {
relativePrefix = match[1];
cleanQuery = match[2];
if (cleanQuery.length < 2) return { results: [] };
}
const targetRegion = country?.toLowerCase();
const cacheKey = `geo_auto:${targetRegion || 'auto'}:${cleanQuery.toLowerCase()}`;
const cached: any = await this.cacheManager.get(cacheKey);
if (cached) return { results: cached, source: 'cache_hit' };
const [normalizedQueryRes] = await this.osmPointsRepository.query(`SELECT normalize_arabic($1) as nq`, [cleanQuery]);
const normalizedQuery = normalizedQueryRes?.nq || cleanQuery.toLowerCase();
let queryParams: any[] = [`${normalizedQuery}%`];
let regionCondition = '';
if (targetRegion && ['syria', 'jordan', 'egypt'].includes(targetRegion)) {
regionCondition = `AND (region = '${targetRegion}' OR region = 'global')`;
}
// Using the specialized btree index on varchar_pattern_ops
const sqlQuery = `
SELECT
id, name, name_ar, category, region, source, address
FROM unified_search_index
WHERE normalized_name LIKE $1
${regionCondition}
ORDER BY LENGTH(normalized_name) ASC
LIMIT 7
`;
const results = await this.osmPointsRepository.query(sqlQuery, queryParams);
const formatted = results.map(r => {
const nameAr = r.name_ar || r.name;
const displayName = relativePrefix ? `${relativePrefix} ${nameAr}` : nameAr;
return {
id: r.id,
name: displayName,
category: r.category,
region: r.region,
address: r.address
};
});
if (formatted.length > 0) {
await this.cacheManager.set(cacheKey, formatted, 3600000); // 1 hour
}
return { results: formatted };
} catch (e) {
this.logger.error('Autocomplete failed:', e);
return { results: [] };
}
}
async reverseGeocode(lat: number, lng: number) {
try {
const repo = this.getRepositoryForCoords(lat, lng);
@@ -307,11 +411,33 @@ export class GeocodingService {
return allResults
.sort((a, b) => Number(a.distance) - Number(b.distance))
.slice(0, 5)
.map(r => ({
...r,
latitude: parseFloat(r.latitude),
longitude: parseFloat(r.longitude)
}));
.map(r => {
const distance = Number(r.distance);
const fullAddressParts = [r.name_ar || r.name, r.address, r.neighbourhood, r.district, r.governorate].filter(Boolean);
let humanReadable = r.name_ar || r.name;
if (distance <= 20) {
// Very close: just the name
humanReadable = r.name_ar || r.name;
} else if (distance <= 70 && r.category !== 'street') {
// Close: "أمام [اسم المعلم]" (In front of)
const streetName = r.address ? `، ${r.address}` : '';
humanReadable = `أمام ${r.name_ar || r.name}${streetName}`;
} else if (r.category === 'street' || distance > 70) {
// Far or Street: "شارع كذا، الحي"
const streetPart = r.category === 'street' ? (r.name_ar || r.name) : (r.address || r.name_ar || r.name);
const districtPart = r.neighbourhood || r.district || '';
humanReadable = [streetPart, districtPart].filter(Boolean).join('، ');
}
return {
...r,
latitude: parseFloat(r.latitude),
longitude: parseFloat(r.longitude),
human_readable_address: humanReadable,
full_address: fullAddressParts.join('، ')
};
});
} catch (error) {
this.logger.error('Reverse geocoding error:', error);
return [];