Hamza/maps-saas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2026-04-14-2 full

Browse Source
This commit is contained in:
Hamza-Ayed
2026-04-14 13:28:19 +03:00
parent 581eda1ea8
commit 830b498485
8 changed files with 138 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/geocoding/administrative-linking.service.ts
+53 -62
View File
@@ -49,10 +49,10 @@ export class AdministrativeLinkingService {
}
/**
* Step 1: Sync Neighborhood Points from OSM for a given BBox
* Step 1: Sync Neighborhood Points from OSM for a given BBox and Country
*/
async syncOsmNeighborhoodPoints(bbox: string = '31.86,35.94,32.22,36.25') {
this.logger.log(`Syncing neighborhood points for bbox: ${bbox}`);
async syncOsmNeighborhoodPoints(bbox: string = '31.86,35.94,32.22,36.25', country?: string) {
this.logger.log(`Syncing neighborhood points for country: ${country || 'auto'}, bbox: ${bbox}`);
const query = `[out:json][timeout:900];node["place"~"neighbourhood|suburb|town"](${bbox});out;`;
try {
@@ -60,23 +60,33 @@ export class AdministrativeLinkingService {
const elements = data.elements;
this.logger.log(`Found ${elements.length} points in OSM.`);
// Determine country from BBox if not provided (simple heuristic)
let detectedCountry = country;
if (!detectedCountry) {
const firstLat = parseFloat(bbox.split(',')[0]);
if (firstLat > 32.3) detectedCountry = 'syria';
else if (firstLat < 31.0) detectedCountry = 'egypt';
else detectedCountry = 'jordan';
}
for (const e of elements) {
const nameAr = e.tags['name:ar'] || e.tags['name'];
const nameEn = e.tags['name:en'] || e.tags['name'];
const geometry = { type: 'Point', coordinates: [e.lon, e.lat] };
await this.dataSource.query(`
INSERT INTO neighborhood_points (osm_id, name_ar, name_en, place_type, geometry)
VALUES ($1, $2, $3, $4, ST_SetSRID(ST_GeomFromGeoJSON($5), 4326))
INSERT INTO neighborhood_points (osm_id, name_ar, name_en, place_type, geometry, country)
VALUES ($1, $2, $3, $4, ST_SetSRID(ST_GeomFromGeoJSON($5), 4326), $6)
ON CONFLICT (osm_id) DO UPDATE SET
name_ar = EXCLUDED.name_ar,
name_en = EXCLUDED.name_en,
place_type = EXCLUDED.place_type,
geometry = EXCLUDED.geometry;
`, [e.id, nameAr, nameEn, e.tags['place'], JSON.stringify(geometry)]);
geometry = EXCLUDED.geometry,
country = COALESCE(EXCLUDED.country, neighborhood_points.country);
`, [e.id, nameAr, nameEn, e.tags['place'], JSON.stringify(geometry), detectedCountry]);
}
// Link points to parent districts using containment or nearest neighbor
// Link points to parent districts
await this.dataSource.query(`
UPDATE neighborhood_points np
SET district_id = COALESCE(
@@ -94,21 +104,20 @@ export class AdministrativeLinkingService {
LIMIT 1
)
)
WHERE district_id IS NULL;
`);
WHERE district_id IS NULL AND (country = $1 OR $1 IS NULL);
`, [detectedCountry]);
// Diagnostics
const totalPoints = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_points`);
const linkedPoints = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_points WHERE district_id IS NOT NULL`);
const unlinkedPoints = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_points WHERE district_id IS NULL`);
const adminCount = await this.dataSource.query(`SELECT admin_level, count(*) as cnt FROM admin_boundaries GROUP BY admin_level ORDER BY admin_level`);
const countryPoints = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_points WHERE country = $1`, [detectedCountry]);
const diagnostics = {
detected_country: detectedCountry,
osm_fetched: elements.length,
country_points: parseInt(countryPoints[0].cnt),
total_points_in_db: parseInt(totalPoints[0].cnt),
linked_to_district: parseInt(linkedPoints[0].cnt),
unlinked: parseInt(unlinkedPoints[0].cnt),
admin_boundaries: adminCount.map(r => ({ level: r.admin_level, count: parseInt(r.cnt) })),
};
this.logger.log(`Step 1 diagnostics: ${JSON.stringify(diagnostics)}`);
return diagnostics;
@@ -119,28 +128,22 @@ export class AdministrativeLinkingService {
}
/**
* Step 2: Generate Voronoi Polygons for Neighborhoods
* Step 2: Generate Voronoi Polygons for Neighborhoods (Per Country)
*/
async generateVoronoiNeighborhoods() {
this.logger.log('Generating Voronoi polygons for neighborhoods...');
async generateVoronoiNeighborhoods(country?: string) {
this.logger.log(`Generating Voronoi polygons for ${country || 'all'} neighborhoods...`);
// Pre-flight diagnostics
const prePoints = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_points WHERE district_id IS NOT NULL`);
const preDistricts = await this.dataSource.query(`SELECT district_id, count(*) as cnt FROM neighborhood_points WHERE district_id IS NOT NULL GROUP BY district_id`);
this.logger.log(`Pre-flight: ${prePoints[0].cnt} points across ${preDistricts.length} districts`);
const whereClause = country ? `WHERE district_id IS NOT NULL AND country = '${country}'` : `WHERE district_id IS NOT NULL`;
const prePoints = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_points ${whereClause}`);
const preDistricts = await this.dataSource.query(`SELECT district_id, count(*) as cnt FROM neighborhood_points ${whereClause} GROUP BY district_id`);
// Delete old voronoi polygons for THIS country only
const deleteWhere = country ? `WHERE method = 'voronoi' AND country = '${country}'` : `WHERE method = 'voronoi'`;
await this.dataSource.query(`DELETE FROM neighborhood_polygons ${deleteWhere}`);
this.logger.log(`Deleted old voronoi polygons for ${country || 'all'}.`);
// Log a sample to verify geometry exists
const samplePoints = await this.dataSource.query(`SELECT osm_id, name_ar, district_id, ST_AsText(geometry) as geom_text FROM neighborhood_points WHERE district_id IS NOT NULL LIMIT 3`);
this.logger.log(`Sample points: ${JSON.stringify(samplePoints)}`);
const sampleDistricts = await this.dataSource.query(`SELECT id, name_ar, admin_level, ST_AsText(ST_Centroid(geom)) as centroid FROM admin_boundaries WHERE admin_level IN (6, 8) LIMIT 3`);
this.logger.log(`Sample districts: ${JSON.stringify(sampleDistricts)}`);
// Delete old voronoi polygons
await this.dataSource.query(`DELETE FROM neighborhood_polygons WHERE method = 'voronoi'`);
this.logger.log('Deleted old voronoi polygons.');
// Generate Voronoi per district - do it one district at a time to avoid silent failures
// Generate Voronoi per district
let totalInserted = 0;
for (const row of preDistricts) {
const districtId = row.district_id;
@@ -148,9 +151,8 @@ export class AdministrativeLinkingService {
try {
if (pointCount >= 2) {
// Normal Voronoi for 2+ points
const result = await this.dataSource.query(`
INSERT INTO neighborhood_polygons (osm_id, name_ar, name_en, place_type, parent_id, method, geometry)
INSERT INTO neighborhood_polygons (osm_id, name_ar, name_en, place_type, parent_id, method, country, geometry)
WITH voronoi_cells AS (
SELECT (ST_Dump(ST_VoronoiPolygons(
ST_Collect(np.geometry::geometry),
@@ -164,7 +166,7 @@ export class AdministrativeLinkingService {
),
matched AS (
SELECT
np.osm_id, np.name_ar, np.name_en, np.place_type, np.district_id,
np.osm_id, np.name_ar, np.name_en, np.place_type, np.district_id, np.country,
ST_Multi(ST_Intersection(vc.cell::geometry, d.geom::geometry)) as geometry
FROM voronoi_cells vc
CROSS JOIN LATERAL (
@@ -176,53 +178,41 @@ export class AdministrativeLinkingService {
) np
JOIN admin_boundaries d ON d.id = $1
)
SELECT osm_id, name_ar, name_en, place_type, district_id, 'voronoi', geometry
SELECT osm_id, name_ar, name_en, place_type, district_id, 'voronoi', country, geometry
FROM matched
WHERE ST_IsValid(geometry) AND NOT ST_IsEmpty(geometry)
`, [districtId]);
totalInserted += (result?.length || result?.[1] || 0);
this.logger.log(`District ${districtId}: ${pointCount} points -> ${result?.length || result?.[1] || '?'} polygons`);
} else if (pointCount === 1) {
// Single point: assign entire district boundary
const result = await this.dataSource.query(`
INSERT INTO neighborhood_polygons (osm_id, name_ar, name_en, place_type, parent_id, method, geometry)
SELECT np.osm_id, np.name_ar, np.name_en, np.place_type, np.district_id, 'voronoi', ST_Multi(d.geom::geometry)
await this.dataSource.query(`
INSERT INTO neighborhood_polygons (osm_id, name_ar, name_en, place_type, parent_id, method, country, geometry)
SELECT np.osm_id, np.name_ar, np.name_en, np.place_type, np.district_id, 'voronoi', np.country, ST_Multi(d.geom::geometry)
FROM neighborhood_points np
JOIN admin_boundaries d ON d.id = np.district_id
WHERE np.district_id = $1
`, [districtId]);
totalInserted += (result?.length || result?.[1] || 0);
this.logger.log(`District ${districtId}: 1 point -> assigned district boundary`);
totalInserted++;
}
} catch (err) {
this.logger.error(`Voronoi FAILED for district ${districtId} (${pointCount} points): ${err.message}`);
this.logger.error(`Voronoi FAILED for district ${districtId}: ${err.message}`);
}
}
// Post-flight count
const postCount = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_polygons`);
const finalCount = parseInt(postCount[0].cnt);
this.logger.log(`Step 2 complete: ${finalCount} total polygons in neighborhood_polygons`);
// Sample polygon names
const samplePolygons = await this.dataSource.query(`SELECT id, osm_id, name_ar, parent_id, method FROM neighborhood_polygons LIMIT 5`);
this.logger.log(`Sample polygons: ${JSON.stringify(samplePolygons)}`);
const postCount = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_polygons ${country ? `WHERE country = '${country}'` : ''}`);
return {
status: finalCount > 0 ? 'success' : 'WARNING_EMPTY',
total_polygons: finalCount,
districts_processed: preDistricts.length,
sample: samplePolygons
status: 'success',
country: country || 'all',
total_polygons: parseInt(postCount[0].cnt),
districts_processed: preDistricts.length
};
}
/**
* Step 3: Link all Places (Jordan & Syria) to the full administrative hierarchy
* Step 3: Link all Places (Jordan, Syria, Egypt) to the full administrative hierarchy
*/
async linkPlaces(country: 'jordan' | 'syria') {
const tableName = country === 'jordan' ? 'places_jordan' : 'places_syria';
async linkPlaces(country: 'jordan' | 'syria' | 'egypt') {
const tableName = `places_${country}`;
// Pre-flight diagnostics
const polyCount = await this.dataSource.query(`SELECT count(*) as cnt FROM neighborhood_polygons`);
@@ -247,6 +237,7 @@ export class AdministrativeLinkingService {
SET
neighborhood_id = (
SELECT np.id FROM neighborhood_polygons np
WHERE (np.country = $1 OR np.country IS NULL)
ORDER BY p.location::geometry <-> np.geometry::geometry
LIMIT 1
),
@@ -269,7 +260,7 @@ export class AdministrativeLinkingService {
LIMIT 1
)
WHERE p.location IS NOT NULL;
`);
`, [country]);
// Post-flight diagnostics
const linkedNeighborhood = await this.dataSource.query(`SELECT count(*) as cnt FROM ${tableName} WHERE neighborhood_id IS NOT NULL`);
apps/api/src/geocoding/entities/neighborhood-point.entity.ts
+4
View File
@@ -22,6 +22,10 @@ export class NeighborhoodPoint {
@Column({ type: 'int', nullable: true })
@Index()
district_id: number;
@Column({ nullable: true })
@Index()
country: string;
@Column({ type: 'geometry', spatialFeatureType: 'Point', srid: 4326 })
@Index({ spatial: true })
apps/api/src/geocoding/entities/neighborhood-polygon.entity.ts
+4
View File
@@ -25,6 +25,10 @@ export class NeighborhoodPolygon {
@Column({ default: 'voronoi' })
method: string;
@Column({ nullable: true })
@Index()
country: string;
@Column({ type: 'float', default: 0.7 })
confidence: number;
apps/api/src/geocoding/geocoding.controller.ts
+8 -6
View File
@@ -135,22 +135,24 @@ export class GeocodingController {
@UseGuards(ApiKeyGuard)
@ApiOperation({ summary: 'Sync neighborhood points from OSM for a bbox' })
@ApiQuery({ name: 'bbox', required: false })
async syncNeighborhoods(@Query('bbox') bbox?: string) {
return this.adminLinkingService.syncOsmNeighborhoodPoints(bbox);
@ApiQuery({ name: 'country', required: false, enum: ['jordan', 'syria', 'egypt'] })
async syncNeighborhoods(@Query('bbox') bbox?: string, @Query('country') country?: string) {
return this.adminLinkingService.syncOsmNeighborhoodPoints(bbox, country);
}
@Post('admin/generate-voronoi')
@UseGuards(ApiKeyGuard)
@ApiOperation({ summary: 'Generate Voronoi polygons for neighborhoods' })
async generateVoronoi() {
return this.adminLinkingService.generateVoronoiNeighborhoods();
@ApiQuery({ name: 'country', required: false, enum: ['jordan', 'syria', 'egypt'] })
async generateVoronoi(@Query('country') country?: string) {
return this.adminLinkingService.generateVoronoiNeighborhoods(country);
}
@Post('admin/link-places')
@UseGuards(ApiKeyGuard)
@ApiOperation({ summary: 'Link places to administrative hierarchy' })
@ApiQuery({ name: 'country', required: true, enum: ['jordan', 'syria'] })
async linkPlaces(@Query('country') country: 'jordan' | 'syria') {
@ApiQuery({ name: 'country', required: true, enum: ['jordan', 'syria', 'egypt'] })
async linkPlaces(@Query('country') country: 'jordan' | 'syria' | 'egypt') {
return this.adminLinkingService.linkPlaces(country);
}
}
apps/api/src/geocoding/geocoding.module.ts
+1
View File
@@ -36,5 +36,6 @@ import { NeighborhoodPolygon } from './entities/neighborhood-polygon.entity';
JordanResearchService,
AdministrativeLinkingService
],
exports: [GeocodingService],
})
export class GeocodingModule {}
apps/api/src/geocoding/geocoding.service.ts
+38 -14
View File
@@ -81,11 +81,9 @@ export class GeocodingService {
LEFT JOIN neighborhood_polygons n ON p.neighborhood_id = n.id
LEFT JOIN admin_boundaries d ON p.sub_district_id = d.id
LEFT JOIN admin_boundaries g ON p.governorate_id = g.id
WHERE (p.name_ar % $1 OR p.name % $1 OR p.neighbourhood % $1 OR p.name_ar ILIKE $4 OR p.name ILIKE $4 OR p.neighbourhood ILIKE $4)
${hasLocation ? `AND (p.location && ST_Expand(ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326), $5::float) OR ST_DistanceSphere(p.location, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $6::float)` : ''}
ORDER BY relevance DESC, distance ASC LIMIT 15
ORDER BY relevance DESC, distance ASC LIMIT 25
`;
const results = await repo.query(userQuery, [cleanQuery, lat || null, lon || null, ILikeQuery, radiusInDegrees, radius]);
const results = await repo.query(userQuery, [cleanQuery, lat || null, lon || null]);
allResults.push(...results);
}
@@ -95,17 +93,33 @@ export class GeocodingService {
else if (targetRegion === 'egypt') osmBoundFilter = 'AND ST_Contains(ST_MakeEnvelope(24.5, 22.0, 37.0, 31.8, 4326), geom)';
const osmQuery = `
SELECT osm_id as id, name, name_ar, name_en, COALESCE(amenity, shop, 'place') as category,
latitude, longitude, addr_street as address, 'osm_global' as source,
CASE WHEN $2::float IS NOT NULL THEN ST_DistanceSphere(geom, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) ELSE 0 END as distance,
similarity(COALESCE(name_ar, ''), $1) + similarity(COALESCE(name_en, ''), $1) as relevance
FROM osm_points_with_area
WHERE (name_ar % $1 OR name_en % $1 OR name % $1 OR name_ar ILIKE $4 OR name_en ILIKE $4)
${osmBoundFilter}
${hasLocation ? `AND (geom && ST_Expand(ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326), $5::float) OR ST_DistanceSphere(geom, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) <= $6::float)` : ''}
ORDER BY relevance DESC, distance ASC LIMIT 20
SELECT
o.osm_id as id, o.name, o.name_ar, o.name_en, COALESCE(o.amenity, o.shop, 'place') as category,
o.latitude, o.longitude, o.addr_street as address, 'osm_global' as source,
n.name_ar as neighbourhood,
d.name_ar as district,
g.name_ar as governorate,
CASE WHEN $2::float IS NOT NULL THEN ST_DistanceSphere(o.geom, ST_SetSRID(ST_MakePoint($3::float, $2::float), 4326)) ELSE 0 END as distance,
similarity(COALESCE(o.name_ar, ''), $1) + similarity(COALESCE(o.name_en, ''), $1) as relevance
FROM osm_points_with_area o
LEFT JOIN LATERAL (
SELECT name_ar FROM neighborhood_polygons np
ORDER BY o.geom <-> np.geometry
LIMIT 1
) n ON true
LEFT JOIN LATERAL (
SELECT name_ar FROM admin_boundaries ab
WHERE ab.admin_level = 8 AND ST_Contains(ab.geom, o.geom)
LIMIT 1
) d ON true
LEFT JOIN LATERAL (
SELECT name_ar FROM admin_boundaries ab
WHERE ab.admin_level = 4 AND ST_Contains(ab.geom, o.geom)
LIMIT 1
) g ON true
ORDER BY relevance DESC, distance ASC LIMIT 30
`;
const osmResults = await this.osmPointsRepository.query(osmQuery, [cleanQuery, lat || null, lon || null, ILikeQuery, radiusInDegrees, radius]);
const osmResults = await this.osmPointsRepository.query(osmQuery, [cleanQuery, lat || null, lon || null]);
allResults.push(...osmResults);
// Note: We removed the raw query to 'overture_building' because raw overture tables
@@ -113,8 +127,18 @@ export class GeocodingService {
// scanning them with ILIKE without trigram indices causes a 2-second latency spike.
// Overture data is already properly ingested via the Scraper into places_jordan.
const seenStreets = new Set<string>();
const sortedResults = allResults
.sort((a, b) => hasLocation ? ((a.distance - b.distance) || (b.relevance - a.relevance)) : ((b.relevance - a.relevance) || (a.distance - b.distance)))
.filter(r => {
if (r.category === 'street') {
const key = `${r.name_ar || r.name}_${r.district}_${r.governorate}`;
if (seenStreets.has(key)) return false;
seenStreets.add(key);
}
return true;
})
.slice(0, 25)
.map(r => {
// Build full administrative address from admin_boundaries
apps/api/src/maps/maps.module.ts
+2
View File
@@ -6,11 +6,13 @@ import { MapsController } from './maps.controller';
import { RoadSegmentStat } from './road-stat.entity';
import { CandidateRoad } from './candidate-road.entity';
import { RedisModule } from '../common/redis.module';
import { GeocodingModule } from '../geocoding/geocoding.module';
@Module({
imports: [
TypeOrmModule.forFeature([RoadSegmentStat, CandidateRoad]),
RedisModule,
GeocodingModule,
],
controllers: [MapsController],
providers: [MapsService, TrafficGridService],
apps/api/src/maps/maps.service.ts
+28 -1
View File
@@ -6,6 +6,7 @@ import { RoadSpeedProfile } from './road-speed-profile.entity';
import axios from 'axios';
import { RoadSegmentStat } from './road-stat.entity';
import { TrafficGridService } from './traffic-grid.service';
import { GeocodingService } from '../geocoding/geocoding.service';
@Injectable()
export class MapsService {
@@ -15,7 +16,8 @@ export class MapsService {
private configService: ConfigService,
@InjectRepository(RoadSegmentStat)
private roadStatRepo: Repository<RoadSegmentStat>,
private trafficGrid: TrafficGridService
private trafficGrid: TrafficGridService,
private geocodingService: GeocodingService
) {
this.graphHopperUrl = this.configService.get('GRAPH_HOPPER_URL', 'http://routing:8080');
}
@@ -29,6 +31,29 @@ export class MapsService {
// GraphHopper expects [lng, lat] order
const ghPoints = waypoints.map(wp => [wp[1], wp[0]]);
let startName = 'Unknown Location';
let endName = 'Unknown Location';
try {
const startWp = waypoints[0];
const endWp = waypoints[waypoints.length - 1];
const [startRes, endRes] = await Promise.all([
this.geocodingService.reverseGeocode(startWp[0], startWp[1]),
this.geocodingService.reverseGeocode(endWp[0], endWp[1])
]);
const formatName = (r: any) => {
const parts = [r.name_ar || r.name, r.neighbourhood, r.district, r.governorate].filter(Boolean);
// Deduplicate items continuously (e.g. if name is similar to neighborhood)
const uniqueParts = [...new Set(parts)];
return uniqueParts.length > 0 ? uniqueParts.join('، ') : 'Unknown Location';
};
if (startRes && startRes.length > 0) startName = formatName(startRes[0]);
if (endRes && endRes.length > 0) endName = formatName(endRes[0]);
} catch (e) {
console.warn('Geocoding internal error during routing:', e);
}
const payload: any = {
points: ghPoints,
profile: profile,
@@ -79,6 +104,8 @@ export class MapsService {
duration: Math.round(baseDuration),
trafficAwareDuration: Math.round(trafficAwareDuration),
trafficFactor: Math.round(trafficFactor * 100) / 100,
startName,
endName,
points: route.points,
bbox: route.bbox,
alternatives: alternatives // NEW: array of other routes