diff --git a/apps/api/src/geocoding/entities/map-candidate.entity.ts b/apps/api/src/geocoding/entities/map-candidate.entity.ts index af21a9b..98ea9e2 100644 --- a/apps/api/src/geocoding/entities/map-candidate.entity.ts +++ b/apps/api/src/geocoding/entities/map-candidate.entity.ts @@ -40,4 +40,13 @@ export class MapCandidate extends BasePlace { @Column({ type: 'jsonb', nullable: true }) metadata: any; // For photos, extra contact info, etc. + + @Column({ nullable: true }) + verified_name: string; + + @Column({ type: 'int', default: 0 }) + confidence_score: number; + + @Column({ type: 'jsonb', nullable: true }) + verification_metadata: any; } diff --git a/apps/api/src/geocoding/map-refinement.service.ts b/apps/api/src/geocoding/map-refinement.service.ts index 5692ad4..d0d466d 100644 --- a/apps/api/src/geocoding/map-refinement.service.ts +++ b/apps/api/src/geocoding/map-refinement.service.ts @@ -77,8 +77,13 @@ export class MapRefinementService { }); const saved = await this.candidateRepository.save(candidate); - this.logger.log(`✅ Success: Candidate ${saved.id} enriched with Gov:${saved.governorate_id}, Neigh:${saved.neighborhood_id}`); + this.logger.log(`✅ Success: Candidate ${saved.id} enriched with Gov:${saved.governorate_id}, Neigh:${spatialData.neigh_id}`); + // Trigger automated verification in background (don't block the response) + this.runExternalVerification(saved.id).catch(err => + this.logger.error(`❌ Verification failed for ${saved.id}: ${err.message}`) + ); + // Return with enriched names for immediate feedback return { ...saved, @@ -89,6 +94,67 @@ export class MapRefinementService { } as any; } + /** + * Automated verification using Search Engine Metadata Scraping + */ + async runExternalVerification(candidateId: number): Promise { + const candidate = await this.candidateRepository.findOne({ where: { id: candidateId } }); + if (!candidate) return; + + this.logger.log(`🔍 Starting automated verification for candidate ${candidate.id}...`); + + try { + const axios = require('axios'); + // Using DuckDuckGo HTML search for coordinates - very stable for scraping names + const searchUrl = `https://duckduckgo.com/html/?q=${candidate.latitude},${candidate.longitude}`; + + const response = await axios.get(searchUrl, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + }, + timeout: 10000 + }); + + const html = response.data; + const titleMatch = html.match(/([^<]+)<\/title>/); + + if (titleMatch) { + let officialName = titleMatch[1].replace(' at DuckDuckGo', '').replace(' - DuckDuckGo', '').trim(); + + // If the name is just coordinates, it means no POI was specifically identified + const isCoordinates = /^[-+]?([1-8]?\d(\.\d+)?|90(\.0+)?),\s*[-+]?(180(\.0+)?|((1[0-7]\d)|([1-9]?\d))(\.\d+)?)$/.test(officialName); + + if (!isCoordinates && officialName.length > 5) { + candidate.verified_name = officialName; + + // Basic confidence: if the user's name is partially in the official name, high trust + const cleanUser = candidate.name.toLowerCase().trim(); + const cleanOfficial = officialName.toLowerCase(); + + if (cleanOfficial.includes(cleanUser) || cleanUser.includes(cleanOfficial)) { + candidate.confidence_score = 90; + } else { + candidate.confidence_score = 60; // Found a name, but didn't match perfectly + } + + this.logger.log(`🎯 Verified: Found "${officialName}" near suggestion. Trust: ${candidate.confidence_score}%`); + } else { + candidate.confidence_score = 10; // Only coordinates found + this.logger.log(`⚠️ No specific POI name found for coordinates ${candidate.latitude},${candidate.longitude}`); + } + } + + candidate.verification_metadata = { + last_check: new Date().toISOString(), + source: 'DDG_SCRAPE' + }; + + await this.candidateRepository.save(candidate); + } catch (error) { + this.logger.error(`Automated verification error: ${error.message}`); + } + } + async getCandidates(status?: CandidateStatus): Promise<any[]> { const query = ` SELECT diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index b95007d..196fd98 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -10,8 +10,12 @@ async function bootstrap() { logger: ['error', 'warn', 'log', 'debug', 'verbose'], }); - // 1. Modern Security Headers (Prevention of XSS, Clickjacking, etc.) - app.enableCors(); + // 1. Modern Security Headers & Permissive CORS for Production Dashboard + app.enableCors({ + origin: true, // Reflect request origin + methods: 'GET,HEAD,PUT,PATCH,POST,DELETE,OPTIONS', + credentials: true, + }); // 3. Global API prefix app.setGlobalPrefix('api'); diff --git a/apps/dashboard/dashboard.html b/apps/dashboard/dashboard.html index ed08afd..28d9abb 100644 --- a/apps/dashboard/dashboard.html +++ b/apps/dashboard/dashboard.html @@ -435,9 +435,8 @@ <thead> <tr class="text-xs uppercase tracking-widest text-slate-500 bg-slate-900/40"> <th class="px-8 py-5 font-black">Location Name</th> - <th class="px-8 py-5 font-black">Category</th> - <th class="px-8 py-5 font-black">Coordinates</th> - <th class="px-8 py-5 font-black">Submitter</th> + <th class="px-8 py-5 font-black">Automated Match</th> + <th class="px-8 py-5 font-black">Spatial Context</th> <th class="px-8 py-5 font-black text-right">Actions</th> </tr> </thead> diff --git a/apps/dashboard/js/refinement.js b/apps/dashboard/js/refinement.js index d0ca028..ab9559d 100644 --- a/apps/dashboard/js/refinement.js +++ b/apps/dashboard/js/refinement.js @@ -66,34 +66,60 @@ const refinement = { return; } - tbody.innerHTML = refinement.state.candidates.map(c => ` - <tr class="hover:bg-white/[0.02] transition-colors border-b border-white/[0.03]"> - <td class="px-8 py-6"> - <div class="font-bold text-white">${c.name_ar || c.name}</div> - <div class="text-[10px] text-blue-400 font-extrabold uppercase mt-1">ID: ${c.id}</div> - </td> - <td class="px-8 py-6"> - <span class="px-2 py-1 rounded-lg bg-blue-500/10 text-blue-400 text-[10px] font-black uppercase tracking-wider">${c.category || 'General'}</span> - </td> - <td class="px-8 py-6"> - <div class="text-sm text-slate-300 font-medium">${c.governorate_name || 'Unknown Region'}</div> - <div class="text-[11px] text-slate-500 mt-1">${c.neighborhood_name || 'Unknown Neighborhood'}</div> - </td> - <td class="px-8 py-6 text-right space-x-2"> - <button onclick="refinement.showMapModal('${c.latitude}', '${c.longitude}', '${c.name_ar || c.name}')" class="p-2.5 rounded-xl bg-blue-500/10 text-blue-400 hover:bg-blue-500/20 transition-all" title="View on Google Maps"> - <i data-lucide="eye" class="w-4 h-4"></i> - </button> - <button onclick="refinement.reject('${c.id}')" class="p-2.5 rounded-xl bg-red-500/10 text-red-400 hover:bg-red-500/20 transition-all" title="Reject"> - <i data-lucide="x" class="w-4 h-4"></i> - </button> - <button onclick="refinement.approve('${c.id}')" class="p-2.5 rounded-xl bg-emerald-500/10 text-emerald-400 hover:bg-emerald-500/20 transition-all border border-emerald-500/20 shadow-lg shadow-emerald-500/10" title="Approve"> - <i data-lucide="check" class="w-4 h-4"></i> - </button> - </td> - </tr> - `).join(''); + try { + tbody.innerHTML = refinement.state.candidates.map(c => { + const trustColor = c.confidence_score >= 80 ? 'emerald' : (c.confidence_score >= 50 ? 'amber' : 'slate'); + const trustText = c.confidence_score >= 80 ? 'High Trust' : (c.confidence_score >= 50 ? 'Partial Match' : 'Unverified'); + + // Safety checks for coordinates + const lat = typeof c.latitude === 'number' ? c.latitude.toFixed(4) : 'N/A'; + const lng = typeof c.longitude === 'number' ? c.longitude.toFixed(4) : 'N/A'; - lucide.createIcons(); + return ` + <tr class="border-b border-white/5 hover:bg-white/[0.02] transition-colors group"> + <td class="px-8 py-6"> + <div class="flex items-center gap-3"> + <div class="w-10 h-10 rounded-xl bg-${trustColor}-500/10 flex items-center justify-center text-${trustColor}-400"> + <i data-lucide="${c.confidence_score >= 50 ? 'shield-check' : 'shield-alert'}" class="w-5 h-5"></i> + </div> + <div> + <div class="font-bold text-white">${c.name_ar || c.name || 'Untitled'}</div> + <div class="text-xs text-slate-500 mt-1">${c.category || 'Location'}</div> + </div> + </div> + </td> + <td class="px-8 py-6"> + <div class="text-sm text-slate-300 font-medium">${c.verified_name || '<span class="text-slate-600 italic">No external match</span>'}</div> + <div class="flex items-center gap-2 mt-1.5"> + <span class="px-2 py-0.5 rounded-full bg-${trustColor}-500/10 text-${trustColor}-400 text-[10px] font-bold uppercase tracking-wider border border-${trustColor}-500/20"> + ${trustText} (${c.confidence_score || 0}%) + </span> + </div> + </td> + <td class="px-8 py-6"> + <div class="text-sm text-slate-300 font-mono">${lat}, ${lng}</div> + <div class="text-[11px] text-slate-500 mt-1">${c.neighborhood_name || 'Unknown Neighborhood'}</div> + </td> + <td class="px-8 py-6 text-right space-x-2"> + <button onclick="refinement.showMapModal('${c.latitude}', '${c.longitude}', '${(c.name_ar || c.name || "Location").replace(/'/g, "\\'")}')" class="p-2.5 rounded-xl bg-blue-500/10 text-blue-400 hover:bg-blue-500/20 transition-all" title="View on Google Maps"> + <i data-lucide="eye" class="w-4 h-4"></i> + </button> + <button onclick="refinement.reject('${c.id}')" class="p-2.5 rounded-xl bg-red-500/10 text-red-400 hover:bg-red-500/20 transition-all" title="Reject"> + <i data-lucide="x" class="w-4 h-4"></i> + </button> + <button onclick="refinement.approve('${c.id}')" class="p-2.5 rounded-xl bg-emerald-500/10 text-emerald-400 hover:bg-emerald-500/20 transition-all border border-emerald-500/20 shadow-lg shadow-emerald-500/10" title="Approve"> + <i data-lucide="check" class="w-4 h-4"></i> + </button> + </td> + </tr> + `; + }).join(''); + + lucide.createIcons(); + } catch (error) { + console.error('❌ [PlaceAudit] Rendering error:', error); + tbody.innerHTML = `<tr><td colspan="4" class="py-20 text-center text-red-500">Rendering Error: See console for details</td></tr>`; + } }, showMapModal: (lat, lng, name) => { diff --git a/scratch/test_ddg_verify.js b/scratch/test_ddg_verify.js new file mode 100644 index 0000000..97136d9 --- /dev/null +++ b/scratch/test_ddg_verify.js @@ -0,0 +1,47 @@ +const https = require('https'); + +async function testDDGVerify(lat, lng) { + // DuckDuckGo search for coordinates often yields the POI name in the title or a snippet + const url = `https://duckduckgo.com/html/?q=${lat},${lng}`; + console.log(`Checking DuckDuckGo for: ${url}`); + + return new Promise((resolve, reject) => { + const options = { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + } + }; + + https.get(url, options, (res) => { + let body = ''; + res.on('data', chunk => body += chunk); + res.on('end', () => { + // Look for the title which usually contains the location name + const titleMatch = body.match(/<title>([^<]+)<\/title>/); + if (titleMatch) { + const title = titleMatch[1]; + console.log(`\n🎉 EXTRACTED TITLE: ${title}`); + // Usually title is "Names - DuckDuckGo" + const cleanedName = title.split(' at DuckDuckGo')[0].split(' - DuckDuckGo')[0]; + console.log(`Cleaned Name: ${cleanedName}`); + } else { + console.log('No title found in DDG response.'); + } + + // Also search for Arabic POI names in the snippet + const arabicStrings = body.match(/[\u0600-\u06FF\s]{5,}/g); + if (arabicStrings) { + console.log('\nPotential Arabic matches from DDG:'); + [...new Set(arabicStrings)].slice(0, 5).forEach(s => console.log(`- ${s.trim()}`)); + } + + resolve(null); + }); + }).on('error', (err) => { + reject(err); + }); + }); +} + +// Test with Coordinates +testDDGVerify(31.9822, 35.8453); diff --git a/scratch/test_google_deep.js b/scratch/test_google_deep.js new file mode 100644 index 0000000..c0064fe --- /dev/null +++ b/scratch/test_google_deep.js @@ -0,0 +1,54 @@ +const https = require('https'); + +async function testDeepScrape(lat, lng) { + const url = `https://www.google.com/maps/search/${lat},${lng}`; + console.log(`Deep Testing URL: ${url}`); + + return new Promise((resolve, reject) => { + const options = { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept-Language': 'ar,en;q=0.9', + 'Cookie': 'NID=511=...' // Sometimes needed for better results + } + }; + + https.get(url, options, (res) => { + let body = ''; + res.on('data', chunk => body += chunk); + res.on('end', () => { + console.log(`Page size: ${Math.round(body.length / 1024)} KB`); + + // 1. Try to find window.APP_INITIALIZATION_STATE + const initStateMatch = body.match(/window\.APP_INITIALIZATION_STATE=([^;]+);/); + if (initStateMatch) { + const state = initStateMatch[1]; + console.log('✅ Found APP_INITIALIZATION_STATE!'); + + // Look for Arabic strings in this blob + const arabicStrings = state.match(/[\u0600-\u06FF\s]{5,}/g); + if (arabicStrings) { + console.log('\n--- Arabic Strings found in State ---'); + // Take the first few relevant-looking once + const candidates = [...new Set(arabicStrings)].slice(0, 10); + candidates.forEach(s => console.log(`- ${s.trim()}`)); + } + } else { + console.log('❌ APP_INITIALIZATION_STATE not found.'); + } + + // 2. Try searching for "مجمع" or other POI keywords in the whole body + const poiKeywords = ['مجمع', 'بنك', 'مطعم', 'شركة', 'مكاتب']; + const foundKeywords = poiKeywords.filter(kw => body.includes(kw)); + console.log('\nKeywords found in HTML:', foundKeywords.join(', ') || 'None'); + + resolve(null); + }); + }).on('error', (err) => { + reject(err); + }); + }); +} + +// Test with coordinates +testDeepScrape(31.9822, 35.8453); diff --git a/scratch/test_google_preview.js b/scratch/test_google_preview.js new file mode 100644 index 0000000..d029016 --- /dev/null +++ b/scratch/test_google_preview.js @@ -0,0 +1,48 @@ +const https = require('https'); + +async function testPlacePreview(lat, lng) { + // This is an internal-ish URL that sometimes returns cleaner data or direct names + const url = `https://www.google.com/maps/preview/place/${lat},${lng}`; + console.log(`Testing Preview URL: ${url}`); + + return new Promise((resolve, reject) => { + const options = { + headers: { + 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Mobile/15E148 Safari/604.1', + 'Accept-Language': 'ar,en;q=0.9' + } + }; + + https.get(url, options, (res) => { + let body = ''; + res.on('data', chunk => body += chunk); + res.on('end', () => { + // Search for the name in common patterns + // Google often returns a JSON-like structure here + + // Try to find any quoted strings that look like "Name" + const nameRegex = /\[null,null,"([^"]+)",\[/i; + const match = body.match(nameRegex); + + if (match) { + console.log(`\n🎉 POSSIBLE NAME FOUND: ${match[1]}`); + } else { + console.log('\n❌ No name found in preview response.'); + // Check if there's any Arabic text at all + const arabicMatch = body.match(/[\u0600-\u06FF\s]+/g); + if (arabicMatch && arabicMatch.some(s => s.length > 5)) { + console.log('Found some Arabic text, but not isolated as a name.'); + } + } + + // console.log(body.substring(0, 5000)); + resolve(null); + }); + }).on('error', (err) => { + reject(err); + }); + }); +} + +// Test with King Hussein Business Park +testPlacePreview(31.9822, 35.8453);