feat: sync source-backed OSM facility entrances
This commit is contained in:
Hamza-Ayed
@@ -1,4 +1,5 @@
|
||||
#!/bin/bash
|
||||
# وظيفة الملف: أداة تشغيل أو تجهيز باسم backup db؛ تفاصيل خطواتها ومدخلاتها موضحة في دليل ملفات المشروع.
|
||||
# ==============================================================================
|
||||
# SIRO Maps — Daily Database Backup Script
|
||||
# ==============================================================================
|
||||
@@ -14,10 +15,11 @@ mkdir -p "$BACKUP_DIR"
|
||||
|
||||
echo "💾 [$(date '+%Y-%m-%d %H:%M:%S')] Starting SIRO Maps custom tables backup..."
|
||||
|
||||
# Dump custom tables: places_*, place_gates, billing, users, telemetry
|
||||
# Dump custom tables, including archived legacy gate rows for rollback.
|
||||
docker compose -f "${APP_DIR}/docker-compose.yml" exec -T db pg_dump -U mapuser -d mapdb \
|
||||
-t 'places_*' \
|
||||
-t 'place_gates' \
|
||||
-t 'place_gates_unverified_archive_20260929' \
|
||||
-t 'users' \
|
||||
-t 'api_keys' \
|
||||
-t 'billing_*' \
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
#!/bin/bash
|
||||
# وظيفة الملف: أداة تشغيل أو تجهيز باسم master cron sync؛ تفاصيل خطواتها ومدخلاتها موضحة في دليل ملفات المشروع.
|
||||
# ==============================================================================
|
||||
# SIRO Maps — Master Bi-Monthly Geospatial Synchronization (Every 15 Days)
|
||||
# ==============================================================================
|
||||
# This script runs automatically via crontab on the 1st and 15th of every month.
|
||||
# It checks and syncs:
|
||||
# 1. OpenStreetMap & Overture Maps for Jordan, Iraq, Syria, and Egypt.
|
||||
# 2. Gate & Entrance mapping (Hospitals, Malls, Universities, Hotels, Parks).
|
||||
# 2. Gate & Entrance mapping is handled by the 10-day OSM source sync.
|
||||
# 3. Administrative boundary resolution (Governorates & Districts).
|
||||
# 4. Materialized view concurrent refresh (unified_search_index).
|
||||
# 5. Redis search cache flushing.
|
||||
@@ -33,11 +34,7 @@ python3 -u scripts/enrich_iraq_comprehensive.py >> "$LOG_FILE" 2>&1 || echo "⚠
|
||||
echo "🇪🇬 Syncing Egypt places..." >> "$LOG_FILE"
|
||||
python3 -u scripts/enrich_egypt_places.py >> "$LOG_FILE" 2>&1 || echo "⚠️ Egypt sync had warnings" >> "$LOG_FILE"
|
||||
|
||||
# 3. Populate Gates & Entrances (Hospitals, Malls, Universities, Hotels, Parks)
|
||||
echo "🚪 Updating Place Gates & Entrances..." >> "$LOG_FILE"
|
||||
python3 -u scripts/populate_place_gates.py --country all >> "$LOG_FILE" 2>&1 || echo "⚠️ Gates update had warnings" >> "$LOG_FILE"
|
||||
|
||||
# 4. Flush Redis Search Cache via Python socket
|
||||
# 3. Flush Redis Search Cache via Python socket
|
||||
echo "⚡ Flushing Redis search cache..." >> "$LOG_FILE"
|
||||
python3 -c "
|
||||
import socket
|
||||
|
||||
+13
-295
@@ -1,305 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Populate and enrich `place_gates` table for major complexes:
|
||||
- Hospitals (Main Gate, Emergency & Ambulance Gate, Outpatient/Service Gate)
|
||||
- Malls & Shopping Centers (Main Entrance, Parking Entrance, Delivery/Service Gate)
|
||||
- Universities & Colleges (Main Gate, North Gate, South/Student Gate)
|
||||
- Hotels & Resorts (Main Entrance, Valet/Parking Gate, Service Gate)
|
||||
- Parks & Public Gardens (Main Entrance, Family Entrance, Secondary Gate)
|
||||
"""Compatibility entry point for source-backed OSM gate synchronization.
|
||||
|
||||
Harvests from:
|
||||
1. Real OSM gate/entrance nodes (`planet_osm_point` where barrier in ('gate', 'entrance') or entrance is not null)
|
||||
2. Complex boundary polygons and perimeter road-facing access points
|
||||
The previous implementation generated guessed gate coordinates around POI
|
||||
centroids. That behavior has been removed. Use --pbf to import real OSM nodes.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
import time
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
def haversine(lat1, lon1, lat2, lon2):
|
||||
R = 6371000 # meters
|
||||
phi1 = math.radians(lat1)
|
||||
phi2 = math.radians(lat2)
|
||||
delta_phi = math.radians(lat2 - lat1)
|
||||
delta_lambda = math.radians(lon2 - lon1)
|
||||
a = math.sin(delta_phi / 2)**2 + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2)**2
|
||||
return 2 * R * math.atan2(math.sqrt(a), math.sqrt(1 - a))
|
||||
|
||||
def clean_sql_str(val):
|
||||
if val is None:
|
||||
return 'NULL'
|
||||
s = str(val).strip().replace("'", "''")
|
||||
return f"'{s}'"
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Populate place_gates for major complexes.")
|
||||
parser.add_argument("--db-host", default="127.0.0.1")
|
||||
parser.add_argument("--db-port", type=int, default=5432)
|
||||
parser.add_argument("--db-user", default="mapuser")
|
||||
parser.add_argument("--db-pass", default="mappass")
|
||||
parser.add_argument("--db-name", default="mapdb")
|
||||
parser.add_argument("--country", default="all", choices=["jordan", "iraq", "syria", "egypt", "all"])
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Import real OSM gate/entrance nodes; no coordinates are generated.")
|
||||
parser.add_argument("--pbf", required=True, help="Merged OSM PBF path for the current map-data scope")
|
||||
args = parser.parse_args()
|
||||
target = Path(__file__).with_name("sync_osm_place_gates.py")
|
||||
return subprocess.call([sys.executable, str(target), "--pbf", args.pbf])
|
||||
|
||||
import pg8000.native
|
||||
|
||||
print("=" * 70)
|
||||
print("🚪 SIRO Maps — Intelligent Place Gates & Entrances Ingestion")
|
||||
print("=" * 70)
|
||||
|
||||
con = pg8000.native.Connection(
|
||||
user=args.db_user,
|
||||
password=args.db_pass,
|
||||
host=args.db_host,
|
||||
port=args.db_port,
|
||||
database=args.db_name
|
||||
)
|
||||
|
||||
countries = ["jordan", "iraq", "syria", "egypt"] if args.country == "all" else [args.country]
|
||||
|
||||
# 1. Ensure place_gates schema and indexes
|
||||
con.run("""
|
||||
CREATE TABLE IF NOT EXISTS place_gates (
|
||||
id SERIAL PRIMARY KEY,
|
||||
place_id VARCHAR(64) NOT NULL,
|
||||
gate_name_ar VARCHAR(255) NOT NULL,
|
||||
gate_name_en VARCHAR(255),
|
||||
latitude NUMERIC(10,7) NOT NULL,
|
||||
longitude NUMERIC(10,7) NOT NULL,
|
||||
is_main_gate BOOLEAN DEFAULT FALSE,
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_place_gates_place_id ON place_gates(place_id);
|
||||
""")
|
||||
|
||||
# Get already existing place_ids in place_gates to avoid duplicate inserts
|
||||
existing_place_ids = set(r[0] for r in con.run("SELECT DISTINCT place_id FROM place_gates;"))
|
||||
print(f"📌 Already configured places in place_gates: {len(existing_place_ids):,}")
|
||||
|
||||
total_gates_to_insert = []
|
||||
|
||||
for country in countries:
|
||||
table_name = f"places_{country}"
|
||||
prefix = f"places_{country}_"
|
||||
print(f"\n🔍 Processing {country.upper()} ({table_name})...")
|
||||
|
||||
# Select major complexes that benefit from gates
|
||||
rows = con.run(f"""
|
||||
SELECT id, name, name_ar, category, latitude, longitude
|
||||
FROM {table_name}
|
||||
WHERE latitude IS NOT NULL AND longitude IS NOT NULL
|
||||
AND (
|
||||
category ILIKE '%مستشف%' OR category ILIKE '%hospital%'
|
||||
OR category ILIKE '%مول%' OR category ILIKE '%mall%' OR category ILIKE '%مركز تسوق%'
|
||||
OR category ILIKE '%جامع%' OR category ILIKE '%university%' OR category ILIKE '%college%'
|
||||
OR category ILIKE '%فندق%' OR category ILIKE '%hotel%'
|
||||
OR category ILIKE '%حديق%' OR category ILIKE '%منتزه%' OR category ILIKE '%park%'
|
||||
OR category ILIKE '%مطار%' OR category ILIKE '%airport%'
|
||||
OR category ILIKE '%ملعب%' OR category ILIKE '%استاد%' OR category ILIKE '%stadium%'
|
||||
);
|
||||
""")
|
||||
print(f" -> Found {len(rows):,} major complexes.")
|
||||
|
||||
count_added = 0
|
||||
for r in rows:
|
||||
p_id, p_name, p_name_ar, p_cat, p_lat, p_lon = r
|
||||
full_place_id = f"{prefix}{p_id}"
|
||||
|
||||
if full_place_id in existing_place_ids:
|
||||
continue
|
||||
|
||||
lat = float(p_lat)
|
||||
lon = float(p_lon)
|
||||
cat = str(p_cat or '').lower()
|
||||
|
||||
gates_for_place = []
|
||||
|
||||
# 1. Is it a Hospital? (Emergency Gate + Main Gate + Service Gate)
|
||||
if any(k in cat for k in ('مستشف', 'hospital', 'طبي')):
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'البوابة الرئيسية',
|
||||
'gate_name_en': 'Main Entrance',
|
||||
'latitude': lat + 0.00035,
|
||||
'longitude': lon,
|
||||
'is_main_gate': True
|
||||
})
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'بوابة الطوارئ والإسعاف',
|
||||
'gate_name_en': 'Emergency & Ambulance Gate',
|
||||
'latitude': lat - 0.00025,
|
||||
'longitude': lon + 0.00040,
|
||||
'is_main_gate': False
|
||||
})
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'بوابة العيادات الخارجية والخدمات',
|
||||
'gate_name_en': 'Outpatient Clinics & Service Gate',
|
||||
'latitude': lat,
|
||||
'longitude': lon - 0.00040,
|
||||
'is_main_gate': False
|
||||
})
|
||||
|
||||
# 2. Is it a Mall / Shopping Center?
|
||||
elif any(k in cat for k in ('مول', 'mall', 'مركز تسوق')):
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'البوابة الرئيسية',
|
||||
'gate_name_en': 'Main Entrance',
|
||||
'latitude': lat + 0.00030,
|
||||
'longitude': lon,
|
||||
'is_main_gate': True
|
||||
})
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'بوابة مواقف السيارات',
|
||||
'gate_name_en': 'Parking Entrance',
|
||||
'latitude': lat - 0.00030,
|
||||
'longitude': lon + 0.00030,
|
||||
'is_main_gate': False
|
||||
})
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'بوابة الخدمات والشحن',
|
||||
'gate_name_en': 'Service & Delivery Gate',
|
||||
'latitude': lat,
|
||||
'longitude': lon - 0.00035,
|
||||
'is_main_gate': False
|
||||
})
|
||||
|
||||
# 3. Is it a University / College / Education Campus?
|
||||
elif any(k in cat for k in ('جامع', 'university', 'college', 'معهد')):
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'البوابة الرئيسية',
|
||||
'gate_name_en': 'Main Campus Gate',
|
||||
'latitude': lat + 0.00040,
|
||||
'longitude': lon,
|
||||
'is_main_gate': True
|
||||
})
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'البوابة الشمالية (بوابة الطلاب)',
|
||||
'gate_name_en': 'North Student Gate',
|
||||
'latitude': lat + 0.00060,
|
||||
'longitude': lon + 0.00030,
|
||||
'is_main_gate': False
|
||||
})
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'البوابة الجنوبية / الكليات الطبية',
|
||||
'gate_name_en': 'South / Medical Gate',
|
||||
'latitude': lat - 0.00050,
|
||||
'longitude': lon - 0.00030,
|
||||
'is_main_gate': False
|
||||
})
|
||||
|
||||
# 4. Is it a Hotel / Resort?
|
||||
elif any(k in cat for k in ('فندق', 'hotel', 'منتجع')):
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'مدخل الفندق الرئيسي (الاستقبال)',
|
||||
'gate_name_en': 'Main Hotel Entrance / Lobby',
|
||||
'latitude': lat + 0.00020,
|
||||
'longitude': lon,
|
||||
'is_main_gate': True
|
||||
})
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'مدخل مواقف النزلاء (Valet)',
|
||||
'gate_name_en': 'Valet & Guest Parking Gate',
|
||||
'latitude': lat - 0.00025,
|
||||
'longitude': lon + 0.00025,
|
||||
'is_main_gate': False
|
||||
})
|
||||
|
||||
# 5. Is it a Park / Stadium / Garden?
|
||||
elif any(k in cat for k in ('حديق', 'منتزه', 'park', 'ملعب', 'استاد', 'stadium')):
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'البوابة الرئيسية',
|
||||
'gate_name_en': 'Main Entrance',
|
||||
'latitude': lat + 0.00040,
|
||||
'longitude': lon,
|
||||
'is_main_gate': True
|
||||
})
|
||||
gates_for_place.append({
|
||||
'place_id': full_place_id,
|
||||
'gate_name_ar': 'بوابة العائلات / البوابة الشرقية',
|
||||
'gate_name_en': 'Family / East Entrance',
|
||||
'latitude': lat - 0.00040,
|
||||
'longitude': lon + 0.00030,
|
||||
'is_main_gate': False
|
||||
})
|
||||
|
||||
if gates_for_place:
|
||||
total_gates_to_insert.extend(gates_for_place)
|
||||
existing_place_ids.add(full_place_id)
|
||||
count_added += 1
|
||||
|
||||
print(f" ✓ Generated gates for {count_added:,} complexes in {country.upper()}.")
|
||||
|
||||
print(f"\n🌟 Total Gates to Insert: {len(total_gates_to_insert):,}")
|
||||
|
||||
if args.dry_run:
|
||||
print("💡 Dry run complete. No database changes made.")
|
||||
con.close()
|
||||
return
|
||||
|
||||
# Insert into place_gates
|
||||
print(f"\n🚀 Inserting {len(total_gates_to_insert):,} gates into place_gates...")
|
||||
t_ins = time.time()
|
||||
batch_size = 2000
|
||||
total_ins = 0
|
||||
|
||||
con.run("BEGIN;")
|
||||
for i in range(0, len(total_gates_to_insert), batch_size):
|
||||
batch = total_gates_to_insert[i:i + batch_size]
|
||||
values = []
|
||||
for g in batch:
|
||||
p_id = clean_sql_str(g['place_id'])
|
||||
g_ar = clean_sql_str(g['gate_name_ar'])
|
||||
g_en = clean_sql_str(g['gate_name_en'])
|
||||
g_lat = g['latitude']
|
||||
g_lon = g['longitude']
|
||||
g_main = 'TRUE' if g['is_main_gate'] else 'FALSE'
|
||||
|
||||
line = f"({p_id}, {g_ar}, {g_en}, {g_lat:.7f}, {g_lon:.7f}, {g_main})"
|
||||
values.append(line)
|
||||
|
||||
sql = (
|
||||
"INSERT INTO place_gates ("
|
||||
" place_id, gate_name_ar, gate_name_en, latitude, longitude, is_main_gate"
|
||||
") VALUES " + ",\n".join(values) + ";"
|
||||
)
|
||||
con.run(sql)
|
||||
total_ins += len(batch)
|
||||
print(f" -> Progress: {total_ins:,} / {len(total_gates_to_insert):,} gates inserted...")
|
||||
|
||||
con.run("COMMIT;")
|
||||
print(f"✅ Gates insertion committed in {time.time() - t_ins:.2f}s!")
|
||||
|
||||
con.run("ANALYZE place_gates;")
|
||||
total_count = con.run("SELECT count(*) FROM place_gates;")[0][0]
|
||||
print(f"\n🎯 Total Gates in place_gates: {total_count:,}")
|
||||
|
||||
# Sample verification
|
||||
sample = con.run("""
|
||||
SELECT place_id, gate_name_ar, gate_name_en, is_main_gate, latitude, longitude
|
||||
FROM place_gates
|
||||
ORDER BY id DESC
|
||||
LIMIT 6;
|
||||
""")
|
||||
print("\n🔍 Sample New Gates:")
|
||||
for s in sample:
|
||||
print(f" - [{s[0]}] {s[1]} ({s[2]}) - Main: {s[3]} @ {s[4]}, {s[5]}")
|
||||
|
||||
con.close()
|
||||
print("\n🎉 Place Gates enrichment completed successfully!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sync only explicitly mapped OpenStreetMap entrances into place_gates.
|
||||
|
||||
Entrances are extracted from the same OSM PBF used for the routing/map update.
|
||||
The importer associates a node only when it is covered by exactly one named
|
||||
target facility polygon (hospital/clinic, mall, or government facility).
|
||||
No coordinates or gate types are inferred from facility centroids.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
APP_DIR = Path(__file__).resolve().parents[1]
|
||||
COUNTRY_NAMES = {
|
||||
"AE": "الإمارات", "BH": "البحرين", "EG": "مصر", "IQ": "العراق",
|
||||
"JO": "الأردن", "KW": "الكويت", "OM": "عُمان", "QA": "قطر",
|
||||
"SA": "السعودية", "SY": "سوريا", "YE": "اليمن",
|
||||
}
|
||||
USEFUL_ENTRANCES = {
|
||||
"yes", "main", "secondary", "service", "emergency", "parking", "entrance"
|
||||
}
|
||||
|
||||
|
||||
def run(command: list[str], *, input_text: str | None = None) -> str:
|
||||
proc = subprocess.run(
|
||||
command, cwd=APP_DIR, input=input_text, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
)
|
||||
if proc.returncode:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise RuntimeError(f"Command failed ({proc.returncode}): {' '.join(command)}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
return run([
|
||||
"docker", "compose", "exec", "-T", "db", "psql", "-X", "-v",
|
||||
"ON_ERROR_STOP=1", "-U", os.getenv("DB_USER", "mapuser"), "-d",
|
||||
os.getenv("DB_NAME", "mapdb"), "-At", "-c", sql,
|
||||
])
|
||||
|
||||
|
||||
def label_for(tags: dict[str, str]) -> tuple[str, str, str, bool] | None:
|
||||
entrance = tags.get("entrance", "").lower()
|
||||
routing_entrance = tags.get("routing:entrance", "").lower()
|
||||
barrier = tags.get("barrier", "").lower()
|
||||
if (entrance not in USEFUL_ENTRANCES
|
||||
and routing_entrance not in {"yes", "main"}
|
||||
and barrier not in {"gate", "entrance"}):
|
||||
return None
|
||||
if tags.get("access", "").lower() in {"no", "private"}:
|
||||
return None
|
||||
|
||||
labels = {
|
||||
"main": ("المدخل الرئيسي", "Main entrance", "main", True),
|
||||
"secondary": ("مدخل ثانوي", "Secondary entrance", "secondary", False),
|
||||
"service": ("مدخل الخدمات", "Service entrance", "service", False),
|
||||
"emergency": ("مدخل الطوارئ", "Emergency entrance", "emergency", False),
|
||||
"parking": ("مدخل المواقف", "Parking entrance", "parking", False),
|
||||
}
|
||||
if entrance in labels:
|
||||
fallback = labels[entrance]
|
||||
elif routing_entrance in {"yes", "main"}:
|
||||
fallback = ("مدخل", "Entrance", "routing", False)
|
||||
else:
|
||||
fallback = ("بوابة" if barrier == "gate" else "مدخل", "Gate" if barrier == "gate" else "Entrance", "gate" if barrier == "gate" else "entrance", False)
|
||||
name_ar = tags.get("name:ar") or fallback[0]
|
||||
name_en = tags.get("name:en") or tags.get("name") or fallback[1]
|
||||
return name_ar, name_en, fallback[2], fallback[3]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Import source-mapped OSM entrance nodes.")
|
||||
parser.add_argument("--pbf", required=True, help="The current merged OSM PBF used by the map updater")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Extract and report exact matches without changing the live table")
|
||||
args = parser.parse_args()
|
||||
pbf_path = Path(args.pbf)
|
||||
if not pbf_path.is_absolute():
|
||||
pbf_path = APP_DIR / pbf_path
|
||||
if not pbf_path.is_file():
|
||||
parser.error(f"OSM source file does not exist: {pbf_path}")
|
||||
|
||||
run(["osmium", "--version"])
|
||||
print(f"Extracting tagged OSM entrances from {pbf_path} …", flush=True)
|
||||
with tempfile.TemporaryDirectory(prefix="siro-osm-gates-") as temp:
|
||||
extracted = Path(temp) / "entrances.osm.pbf"
|
||||
geojson_path = Path(temp) / "entrances.geojson"
|
||||
run([
|
||||
"osmium", "tags-filter", str(pbf_path),
|
||||
"n/entrance", "n/barrier=gate", "n/barrier=entrance",
|
||||
"n/routing:entrance", "-o", str(extracted),
|
||||
])
|
||||
run(["osmium", "export", "-u", "type_id", str(extracted), "-o", str(geojson_path)])
|
||||
|
||||
features = json.loads(geojson_path.read_text(encoding="utf-8")).get("features", [])
|
||||
rows: list[tuple] = []
|
||||
tag_counts: Counter[str] = Counter()
|
||||
for feature in features:
|
||||
osm_node_id = feature.get("id", "")
|
||||
match = re.fullmatch(r"n(-?\d+)", str(osm_node_id))
|
||||
geometry = feature.get("geometry") or {}
|
||||
coordinates = geometry.get("coordinates") or []
|
||||
tags = feature.get("properties") or {}
|
||||
if not match or len(coordinates) < 2:
|
||||
continue
|
||||
classified = label_for(tags)
|
||||
if classified is None:
|
||||
continue
|
||||
lon, lat = coordinates[:2]
|
||||
rows.append((
|
||||
int(match.group(1)), float(lon), float(lat),
|
||||
tags.get("entrance"), tags.get("routing:entrance"), tags.get("barrier"),
|
||||
tags.get("access"), tags.get("name"), tags.get("name:ar"),
|
||||
tags.get("name:en"), *classified,
|
||||
json.dumps(tags, ensure_ascii=False, separators=(",", ":")),
|
||||
))
|
||||
tag_counts[tags.get("entrance") or tags.get("barrier") or "other"] += 1
|
||||
|
||||
print(f"Source nodes selected: {len(rows):,}", flush=True)
|
||||
if not rows:
|
||||
raise RuntimeError("No usable entrance nodes found; keeping the live gate table unchanged.")
|
||||
|
||||
# Stage exact WGS84 source points. This table is not read by the API.
|
||||
psql("""
|
||||
CREATE TABLE IF NOT EXISTS place_gates_osm_stage (
|
||||
osm_id bigint PRIMARY KEY, longitude double precision NOT NULL,
|
||||
latitude double precision NOT NULL, entrance text, routing_entrance text,
|
||||
barrier text, access text, name text, name_ar text, name_en text,
|
||||
gate_name_ar text NOT NULL, gate_name_en text NOT NULL,
|
||||
gate_type text NOT NULL, is_main_gate boolean NOT NULL,
|
||||
source_tags jsonb NOT NULL
|
||||
);
|
||||
TRUNCATE place_gates_osm_stage;
|
||||
""")
|
||||
with tempfile.TemporaryFile(mode="w+t", encoding="utf-8", newline="") as tsv:
|
||||
writer = csv.writer(tsv, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL)
|
||||
writer.writerows(rows)
|
||||
tsv.seek(0)
|
||||
run([
|
||||
"docker", "compose", "exec", "-T", "db", "psql", "-X", "-v",
|
||||
"ON_ERROR_STOP=1", "-U", os.getenv("DB_USER", "mapuser"), "-d",
|
||||
os.getenv("DB_NAME", "mapdb"), "-c",
|
||||
"COPY place_gates_osm_stage (osm_id,longitude,latitude,entrance,routing_entrance,barrier,access,name,name_ar,name_en,gate_name_ar,gate_name_en,gate_type,is_main_gate,source_tags) FROM STDIN WITH (FORMAT csv, DELIMITER E'\\t', NULL '')",
|
||||
], input_text=tsv.read())
|
||||
|
||||
psql("""
|
||||
ALTER TABLE place_gates_osm_stage ADD COLUMN IF NOT EXISTS geom geometry(Point,3857);
|
||||
UPDATE place_gates_osm_stage
|
||||
SET geom=ST_Transform(ST_SetSRID(ST_MakePoint(longitude,latitude),4326),3857);
|
||||
CREATE INDEX IF NOT EXISTS place_gates_osm_stage_geom_idx
|
||||
ON place_gates_osm_stage USING gist (geom);
|
||||
ANALYZE place_gates_osm_stage;
|
||||
""")
|
||||
|
||||
print("Matching source nodes to OSM facility polygons by exact polygon coverage …", flush=True)
|
||||
report_sql = """
|
||||
WITH facilities AS (
|
||||
SELECT p.osm_id, p.way, r.country_code,
|
||||
'regional_' || r.country_code || '_osm_polygon_' || r.source_id AS place_id
|
||||
FROM planet_osm_polygon p
|
||||
JOIN regional_place_names r
|
||||
ON r.source = 'osm_polygon' AND r.source_id = p.osm_id::text
|
||||
WHERE p.name IS NOT NULL
|
||||
AND (p.amenity IN ('hospital','clinic','townhall','courthouse')
|
||||
OR p.shop = 'mall' OR p.office = 'government' OR p.building = 'government')
|
||||
), matched AS (
|
||||
SELECT g.osm_id AS gate_osm_id, f.country_code, f.place_id
|
||||
FROM place_gates_osm_stage g
|
||||
JOIN facilities f
|
||||
ON ST_Covers(f.way, g.geom)
|
||||
), unique_matches AS (
|
||||
SELECT gate_osm_id, min(country_code) AS country_code, min(place_id) AS place_id
|
||||
FROM matched GROUP BY gate_osm_id HAVING count(DISTINCT place_id) = 1
|
||||
)
|
||||
SELECT country_code || '|' || count(*) || '|' || count(DISTINCT place_id)
|
||||
FROM unique_matches GROUP BY country_code ORDER BY country_code;
|
||||
"""
|
||||
report = psql(report_sql).strip().splitlines()
|
||||
for line in report:
|
||||
code, gates, places = line.split("|")
|
||||
print(f"{COUNTRY_NAMES.get(code, code)} ({code}): {gates} مصدرًا موثقًا ↔ {places} منشأة", flush=True)
|
||||
if not report:
|
||||
raise RuntimeError("No source nodes matched named target facilities; the live gate table is unchanged.")
|
||||
|
||||
samples = psql("""
|
||||
WITH matches AS (
|
||||
SELECT g.osm_id,g.gate_name_ar,g.gate_type,r.country_code,r.name,
|
||||
'regional_'||r.country_code||'_osm_polygon_'||r.source_id AS place_id
|
||||
FROM place_gates_osm_stage g
|
||||
JOIN planet_osm_polygon p
|
||||
ON ST_Covers(p.way,g.geom)
|
||||
JOIN regional_place_names r ON r.source='osm_polygon' AND r.source_id=p.osm_id::text
|
||||
WHERE p.name IS NOT NULL
|
||||
AND (p.amenity IN ('hospital','clinic','townhall','courthouse') OR p.shop='mall'
|
||||
OR p.office='government' OR p.building='government')
|
||||
), unique_ids AS (
|
||||
SELECT osm_id,min(place_id) AS place_id FROM matches
|
||||
GROUP BY osm_id HAVING count(DISTINCT place_id)=1
|
||||
)
|
||||
SELECT m.country_code||'|'||m.name||'|'||m.gate_name_ar||'|'||m.gate_type||'|'||m.osm_id
|
||||
FROM matches m JOIN unique_ids u ON u.osm_id=m.osm_id AND u.place_id=m.place_id
|
||||
ORDER BY m.country_code,m.name LIMIT 12;
|
||||
""").strip()
|
||||
if samples:
|
||||
print("Source-matched examples (country|facility|gate label|OSM tag type|node id):", flush=True)
|
||||
print(samples, flush=True)
|
||||
if args.dry_run:
|
||||
print("Dry run complete; live place_gates was not changed.", flush=True)
|
||||
return 0
|
||||
|
||||
# Archive unverifiable legacy rows once, then expose only source-backed OSM rows.
|
||||
sync_sql = """
|
||||
BEGIN;
|
||||
ALTER TABLE place_gates ADD COLUMN IF NOT EXISTS source text;
|
||||
ALTER TABLE place_gates ADD COLUMN IF NOT EXISTS source_id text;
|
||||
ALTER TABLE place_gates ADD COLUMN IF NOT EXISTS gate_type text;
|
||||
ALTER TABLE place_gates ADD COLUMN IF NOT EXISTS access text;
|
||||
ALTER TABLE place_gates ADD COLUMN IF NOT EXISTS source_tags jsonb;
|
||||
ALTER TABLE place_gates ADD COLUMN IF NOT EXISTS last_seen_at timestamptz;
|
||||
CREATE TABLE IF NOT EXISTS place_gates_unverified_archive_20260929 AS
|
||||
SELECT * FROM place_gates WHERE source IS NULL AND false;
|
||||
INSERT INTO place_gates_unverified_archive_20260929
|
||||
SELECT * FROM place_gates WHERE source IS NULL;
|
||||
DELETE FROM place_gates WHERE source IS NULL OR source = 'openstreetmap';
|
||||
|
||||
WITH facilities AS (
|
||||
SELECT p.osm_id, p.way, r.country_code,
|
||||
'regional_' || r.country_code || '_osm_polygon_' || r.source_id AS place_id
|
||||
FROM planet_osm_polygon p
|
||||
JOIN regional_place_names r
|
||||
ON r.source = 'osm_polygon' AND r.source_id = p.osm_id::text
|
||||
WHERE p.name IS NOT NULL
|
||||
AND (p.amenity IN ('hospital','clinic','townhall','courthouse')
|
||||
OR p.shop = 'mall' OR p.office = 'government' OR p.building = 'government')
|
||||
), matches AS (
|
||||
SELECT g.*, f.place_id
|
||||
FROM place_gates_osm_stage g
|
||||
JOIN facilities f
|
||||
ON ST_Covers(f.way, g.geom)
|
||||
), unique_matches AS (
|
||||
SELECT osm_id, min(place_id) AS place_id FROM matches
|
||||
GROUP BY osm_id HAVING count(DISTINCT place_id) = 1
|
||||
), exact_matches AS (
|
||||
SELECT DISTINCT ON (m.osm_id) m.*
|
||||
FROM matches m JOIN unique_matches u ON u.osm_id=m.osm_id AND u.place_id=m.place_id
|
||||
)
|
||||
INSERT INTO place_gates
|
||||
(place_id,gate_name_ar,gate_name_en,latitude,longitude,is_main_gate,source,source_id,gate_type,access,source_tags,last_seen_at)
|
||||
SELECT place_id,gate_name_ar,gate_name_en,latitude,longitude,is_main_gate,
|
||||
'openstreetmap',osm_id::text,gate_type,access,source_tags,now()
|
||||
FROM exact_matches;
|
||||
CREATE INDEX IF NOT EXISTS idx_place_gates_source_id ON place_gates(source,source_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_place_gates_osm_identity
|
||||
ON place_gates(source,source_id,place_id) WHERE source='openstreetmap';
|
||||
ANALYZE place_gates;
|
||||
COMMIT;
|
||||
"""
|
||||
psql(sync_sql)
|
||||
totals = psql("SELECT source || '|' || count(*) FROM place_gates GROUP BY source ORDER BY source;")
|
||||
print("Live place_gates rows (source|count):", flush=True)
|
||||
print(totals.strip(), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc: # preserve the existing live table on any failed preflight/import
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user