From 1ae8acad7a4eb5026541014eb012f34e933f1b3d Mon Sep 17 00:00:00 2001 From: Hamza-Ayed Date: Tue, 30 Jun 2026 21:12:26 +0300 Subject: [PATCH] Update: 2026-06-30 21:12:26 --- .../Admin/marketing/get_market_anomalies.php | 2 +- .../Admin/marketing/get_price_comparison.php | 8 ++--- .../Admin/marketing/get_price_gap_heatmap.php | 8 ++--- .../marketing/surge_opportunity_index.php | 6 ++-- backend/Admin/marketing/trigger_campaign.php | 6 ++-- backend/Admin/marketing/what_if_simulator.php | 6 ++-- backend/bot/cron_empty_results_to_db.php | 18 ++++++++-- backend/bot/cron_surge_opportunity.php | 6 ++-- backend/bot/cron_weekly_health_report.php | 6 ++-- backend/bot/generate_price_tasks.php | 34 ++++++++++++------- backend/bot/worker.php | 10 +++--- backend/ride/pricing/auto_adapt.php | 2 +- backend/ride/pricing/get.php | 22 ++++++------ backend/schema_primary.sql | 11 +++++- 14 files changed, 90 insertions(+), 55 deletions(-) diff --git a/backend/Admin/marketing/get_market_anomalies.php b/backend/Admin/marketing/get_market_anomalies.php index a15913d6..bfbd61ba 100644 --- a/backend/Admin/marketing/get_market_anomalies.php +++ b/backend/Admin/marketing/get_market_anomalies.php @@ -37,7 +37,7 @@ try { $anomalies = $stmt->fetchAll(PDO::FETCH_ASSOC); // Fetch some recent competitor prices for context - $sqlPrices = "SELECT * FROM competitor_prices"; + $sqlPrices = "SELECT * FROM scraped_competitor_prices"; $paramsPrices = []; if ($countryCode) { $sqlPrices .= " WHERE country_code = :country"; diff --git a/backend/Admin/marketing/get_price_comparison.php b/backend/Admin/marketing/get_price_comparison.php index 3a7275be..35414e3e 100644 --- a/backend/Admin/marketing/get_price_comparison.php +++ b/backend/Admin/marketing/get_price_comparison.php @@ -15,7 +15,7 @@ try { DATE_FORMAT(created_at, '%Y-%m-%d %H:00:00') AS hour_bucket, AVG(price_per_km) AS avg_price_per_km, COUNT(*) AS sample_count - FROM competitor_prices + FROM scraped_competitor_prices WHERE created_at >= DATE_SUB(NOW(), INTERVAL 24 HOUR)"; $compParams = []; if ($countryCode) { @@ -33,12 +33,12 @@ try { // 2. PCI by region — group competitor prices by ~0.02° grid cells $pciSql = "SELECT - ROUND(from_latitude * 50, 0) / 50 AS lat_group, - ROUND(from_longitude * 50, 0) / 50 AS lng_group, + ROUND(start_lat * 50, 0) / 50 AS lat_group, + ROUND(start_lng * 50, 0) / 50 AS lng_group, competitor_name, AVG(price_per_km) AS avg_price_per_km, COUNT(*) AS samples - FROM competitor_prices + FROM scraped_competitor_prices WHERE created_at >= DATE_SUB(NOW(), INTERVAL 7 DAY)"; $pciParams = []; if ($countryCode) { diff --git a/backend/Admin/marketing/get_price_gap_heatmap.php b/backend/Admin/marketing/get_price_gap_heatmap.php index 822ca787..13f87051 100644 --- a/backend/Admin/marketing/get_price_gap_heatmap.php +++ b/backend/Admin/marketing/get_price_gap_heatmap.php @@ -35,13 +35,13 @@ try { // Aggregate competitor data by geographical grid (approx 1.5km x 1.5km) $sql = "SELECT - ROUND(from_latitude * 74, 0) / 74 AS lat_group, - ROUND(from_longitude * 74, 0) / 74 AS lng_group, + ROUND(start_lat * 74, 0) / 74 AS lat_group, + ROUND(start_lng * 74, 0) / 74 AS lng_group, AVG(price_per_km) as avg_competitor_price_per_km, COUNT(*) as trip_count - FROM competitor_prices + FROM scraped_competitor_prices WHERE country_code = :country - AND distance_km > 0 + AND price_per_km > 0 AND created_at >= DATE_SUB(NOW(), INTERVAL 7 DAY) GROUP BY lat_group, lng_group HAVING trip_count >= 3"; // Require at least 3 trips for a reliable heatmap point diff --git a/backend/Admin/marketing/surge_opportunity_index.php b/backend/Admin/marketing/surge_opportunity_index.php index 733e4e74..ffe6bc95 100644 --- a/backend/Admin/marketing/surge_opportunity_index.php +++ b/backend/Admin/marketing/surge_opportunity_index.php @@ -32,8 +32,8 @@ try { // 1. حساب الـ baseline (آخر 7 أيام، بدون آخر 6 ساعات) // و current (آخر ساعتين) لكل منافس في كل خلية grid $sql = "SELECT - ROUND(cp.from_latitude * 74, 0) / 74 AS lat_group, - ROUND(cp.from_longitude * 74, 0) / 74 AS lng_group, + ROUND(cp.start_lat * 74, 0) / 74 AS lat_group, + ROUND(cp.start_lng * 74, 0) / 74 AS lng_group, cp.competitor_name, cp.country_code, AVG(CASE WHEN cp.created_at < DATE_SUB(NOW(), INTERVAL 6 HOUR) @@ -42,7 +42,7 @@ try { THEN cp.price_per_km END) AS current_avg, COUNT(*) AS total_samples, SUM(CASE WHEN cp.created_at >= DATE_SUB(NOW(), INTERVAL 2 HOUR) THEN 1 ELSE 0 END) AS recent_samples - FROM competitor_prices cp + FROM scraped_competitor_prices cp WHERE cp.created_at >= DATE_SUB(NOW(), INTERVAL 7 DAY) AND cp.price_per_km > 0 $where diff --git a/backend/Admin/marketing/trigger_campaign.php b/backend/Admin/marketing/trigger_campaign.php index 47988912..eb579c8f 100644 --- a/backend/Admin/marketing/trigger_campaign.php +++ b/backend/Admin/marketing/trigger_campaign.php @@ -21,9 +21,9 @@ $siroBasePrice = filterRequest('siro_base_price', 'float') ?? 10000.0; try { // 3. Fetch recent competitor prices for this region to supply context to Gemini - $sqlPrices = "SELECT competitor_name, total_price, distance_km - FROM competitor_prices - WHERE country_code = :country + $sqlPrices = "SELECT competitor_name, price_amount AS total_price, (price_amount / price_per_km) AS distance_km + FROM scraped_competitor_prices + WHERE country_code = :country AND price_per_km > 0 ORDER BY created_at DESC LIMIT 10"; $stmtPrices = $con->prepare($sqlPrices); $stmtPrices->execute([':country' => strtoupper($countryCode)]); diff --git a/backend/Admin/marketing/what_if_simulator.php b/backend/Admin/marketing/what_if_simulator.php index ba680dcf..7c2c8c02 100644 --- a/backend/Admin/marketing/what_if_simulator.php +++ b/backend/Admin/marketing/what_if_simulator.php @@ -22,10 +22,10 @@ try { } // 1. Fetch recent competitor trips (last 7 days, limit 500 for fast simulation) - $sql = "SELECT distance_km, total_price, competitor_name - FROM competitor_prices + $sql = "SELECT (price_amount / price_per_km) AS distance_km, price_amount AS total_price, competitor_name + FROM scraped_competitor_prices WHERE country_code = :country - AND distance_km > 0 + AND price_per_km > 0 AND created_at >= DATE_SUB(NOW(), INTERVAL 7 DAY) ORDER BY created_at DESC LIMIT 500"; diff --git a/backend/bot/cron_empty_results_to_db.php b/backend/bot/cron_empty_results_to_db.php index de9d4120..f3d9a604 100644 --- a/backend/bot/cron_empty_results_to_db.php +++ b/backend/bot/cron_empty_results_to_db.php @@ -35,7 +35,7 @@ if (empty($data) || !is_array($data)) { } $insertedCount = 0; -$stmt = $con->prepare("INSERT INTO scraped_competitor_prices (task_id, app_name, start_location, end_location, price_amount, currency) VALUES (?, ?, ?, ?, ?, ?)"); +$stmt = $con->prepare("INSERT INTO scraped_competitor_prices (task_id, app_name, competitor_name, start_location, end_location, start_lat, start_lng, end_lat, end_lng, price_amount, price_per_km, currency, country_code) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); foreach ($data as $row) { if (isset($row['status']) && $row['status'] !== 'success') { @@ -83,7 +83,21 @@ foreach ($data as $row) { continue; } - if ($stmt->execute([$taskId, $appName, $startLoc, $endLoc, $amount, $currency])) { + $distanceKm = (float)($resultData['distance_km'] ?? 1); + if ($distanceKm <= 0) $distanceKm = 1; + $pricePerKm = $amount / $distanceKm; + + $startLat = $resultData['start_lat'] ?? null; + $startLng = $resultData['start_lng'] ?? null; + $endLat = $resultData['end_lat'] ?? null; + $endLng = $resultData['end_lng'] ?? null; + $countryCode = 'JO'; // Default for now, as scraping is in Jordan + + if ($stmt->execute([ + $taskId, $appName, $appName, $startLoc, $endLoc, + $startLat, $startLng, $endLat, $endLng, + $amount, $pricePerKm, $currency, $countryCode + ])) { $insertedCount++; } else { echo "Failed to insert task_id: $taskId. Error: " . implode(" ", $stmt->errorInfo()) . "\n"; diff --git a/backend/bot/cron_surge_opportunity.php b/backend/bot/cron_surge_opportunity.php index 335eb3bd..7be7f1ef 100644 --- a/backend/bot/cron_surge_opportunity.php +++ b/backend/bot/cron_surge_opportunity.php @@ -30,8 +30,8 @@ echo "[".date('Y-m-d H:i:s')."] Starting cron_surge_opportunity...\n"; try { // 1. حساب الـ baseline و current $sql = "SELECT - ROUND(cp.from_latitude * 74, 0) / 74 AS lat_group, - ROUND(cp.from_longitude * 74, 0) / 74 AS lng_group, + ROUND(cp.start_lat * 74, 0) / 74 AS lat_group, + ROUND(cp.start_lng * 74, 0) / 74 AS lng_group, cp.competitor_name, cp.country_code, AVG(CASE WHEN cp.created_at < DATE_SUB(NOW(), INTERVAL 6 HOUR) @@ -40,7 +40,7 @@ try { THEN cp.price_per_km END) AS current_avg, COUNT(*) AS total_samples, SUM(CASE WHEN cp.created_at >= DATE_SUB(NOW(), INTERVAL 2 HOUR) THEN 1 ELSE 0 END) AS recent_samples - FROM competitor_prices cp + FROM scraped_competitor_prices cp WHERE cp.created_at >= DATE_SUB(NOW(), INTERVAL 7 DAY) AND cp.price_per_km > 0 GROUP BY lat_group, lng_group, cp.competitor_name, cp.country_code diff --git a/backend/bot/cron_weekly_health_report.php b/backend/bot/cron_weekly_health_report.php index 0befb9f9..ca5d06c9 100644 --- a/backend/bot/cron_weekly_health_report.php +++ b/backend/bot/cron_weekly_health_report.php @@ -26,10 +26,10 @@ try { foreach ($countries as $countryCode) { // 1. Calculate Average PCI and Market Share - $sql = "SELECT distance_km, total_price - FROM competitor_prices + $sql = "SELECT (price_amount / price_per_km) AS distance_km, price_amount AS total_price + FROM scraped_competitor_prices WHERE country_code = :country - AND distance_km > 0 + AND price_per_km > 0 AND created_at >= DATE_SUB(NOW(), INTERVAL 7 DAY)"; $stmt = $con->prepare($sql); $stmt->execute([':country' => $countryCode]); diff --git a/backend/bot/generate_price_tasks.php b/backend/bot/generate_price_tasks.php index 34423a28..dfcfaa5f 100644 --- a/backend/bot/generate_price_tasks.php +++ b/backend/bot/generate_price_tasks.php @@ -17,19 +17,27 @@ try { // 1. Ensure Table Exists $sql = " -CREATE TABLE IF NOT EXISTS competitor_prices ( - id INT AUTO_INCREMENT PRIMARY KEY, - competitor_name VARCHAR(50) NOT NULL, - from_latitude VARCHAR(30) NOT NULL, - from_longitude VARCHAR(30) NOT NULL, - to_latitude VARCHAR(30) NOT NULL, - to_longitude VARCHAR(30) NOT NULL, - distance_km DECIMAL(8,2) NOT NULL, - total_price DECIMAL(10,2) NOT NULL, - price_per_km DECIMAL(8,2) NOT NULL, - country_code VARCHAR(5) NOT NULL DEFAULT 'SY', - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - INDEX idx_competitor_country (competitor_name, country_code) +CREATE TABLE IF NOT EXISTS `scraped_competitor_prices` ( + `id` BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + `task_id` varchar(100) DEFAULT NULL, + `app_name` varchar(100) NOT NULL, + `competitor_name` varchar(100) NOT NULL, + `start_location` varchar(255) NOT NULL, + `end_location` varchar(255) NOT NULL, + `start_lat` decimal(10,7) DEFAULT NULL, + `start_lng` decimal(10,7) DEFAULT NULL, + `end_lat` decimal(10,7) DEFAULT NULL, + `end_lng` decimal(10,7) DEFAULT NULL, + `price_amount` decimal(8,2) NOT NULL, + `price_per_km` decimal(8,2) NOT NULL, + `currency` varchar(10) NOT NULL DEFAULT 'JOD', + `country_code` varchar(10) NOT NULL DEFAULT 'JO', + `scraped_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + KEY `idx_app_name` (`app_name`), + KEY `idx_competitor_name` (`competitor_name`), + KEY `idx_start_location` (`start_location`), + KEY `idx_country_code` (`country_code`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; "; $con->exec($sql); diff --git a/backend/bot/worker.php b/backend/bot/worker.php index 9673746b..913a5bac 100644 --- a/backend/bot/worker.php +++ b/backend/bot/worker.php @@ -133,11 +133,13 @@ if ($method === 'GET') { // 1. Save to MySQL $stmt = $con->prepare(" - INSERT INTO competitor_prices - (competitor_name, from_latitude, from_longitude, to_latitude, to_longitude, distance_km, total_price, price_per_km, country_code) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO scraped_competitor_prices + (task_id, app_name, competitor_name, start_location, end_location, start_lat, start_lng, end_lat, end_lng, price_amount, price_per_km, currency, country_code) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "); - $stmt->execute([$app_name, (string)$start_lat, (string)$start_lng, (string)$end_lat, (string)$end_lng, $distance_km, $price, $pricePerKm, $country_code]); + $start_loc = "Lat: $start_lat, Lng: $start_lng"; + $end_loc = "Lat: $end_lat, Lng: $end_lng"; + $stmt->execute([$task_id, $app_name, $app_name, $start_loc, $end_loc, (string)$start_lat, (string)$start_lng, (string)$end_lat, (string)$end_lng, $price, $pricePerKm, 'JOD', $country_code]); // 2. Save to Redis (Calculate Price Per KM) if ($distance_km > 0 && $price > 0) { diff --git a/backend/ride/pricing/auto_adapt.php b/backend/ride/pricing/auto_adapt.php index 2c626221..83eadae1 100644 --- a/backend/ride/pricing/auto_adapt.php +++ b/backend/ride/pricing/auto_adapt.php @@ -41,7 +41,7 @@ $logEntries = []; foreach ($countries as $country => $cc) { // 1. متوسط سعر الكيلو لكل منافس (آخر 24 ساعة) $sql = "SELECT competitor_name, AVG(price_per_km) AS avg_ppm - FROM competitor_prices + FROM scraped_competitor_prices WHERE country_code = :cc AND created_at >= DATE_SUB(NOW(), INTERVAL 24 HOUR) AND price_per_km > 0 diff --git a/backend/ride/pricing/get.php b/backend/ride/pricing/get.php index 7637c565..f466dda2 100644 --- a/backend/ride/pricing/get.php +++ b/backend/ride/pricing/get.php @@ -288,13 +288,14 @@ function calculateDynamicPrice($country, $minFare, $distance, $duration, $kazanR $maxTlng = $destLng + $lngDelta; // Layer 1: Start and End match within bounding box - $sqlComp = "SELECT total_price, distance_km - FROM competitor_prices + $sqlComp = "SELECT price_amount AS total_price, (price_amount / price_per_km) AS distance_km + FROM scraped_competitor_prices WHERE country_code = :country_code - AND (from_latitude + 0.0) BETWEEN :min_flat AND :max_flat - AND (from_longitude + 0.0) BETWEEN :min_flng AND :max_flng - AND (to_latitude + 0.0) BETWEEN :min_tlat AND :max_tlat - AND (to_longitude + 0.0) BETWEEN :min_tlng AND :max_tlng + AND (start_lat + 0.0) BETWEEN :min_flat AND :max_flat + AND (start_lng + 0.0) BETWEEN :min_flng AND :max_flng + AND (end_lat + 0.0) BETWEEN :min_tlat AND :max_tlat + AND (end_lng + 0.0) BETWEEN :min_tlng AND :max_tlng + AND price_per_km > 0 AND created_at >= DATE_SUB(NOW(), INTERVAL 24 HOUR) ORDER BY created_at DESC LIMIT 5"; $stmtComp = $con->prepare($sqlComp); @@ -313,11 +314,12 @@ function calculateDynamicPrice($country, $minFare, $distance, $duration, $kazanR if (empty($matches)) { // Layer 2 Fallback: Start match only within bounding box - $sqlFallback = "SELECT total_price, distance_km - FROM competitor_prices + $sqlFallback = "SELECT price_amount AS total_price, (price_amount / price_per_km) AS distance_km + FROM scraped_competitor_prices WHERE country_code = :country_code - AND (from_latitude + 0.0) BETWEEN :min_flat AND :max_flat - AND (from_longitude + 0.0) BETWEEN :min_flng AND :max_flng + AND (start_lat + 0.0) BETWEEN :min_flat AND :max_flat + AND (start_lng + 0.0) BETWEEN :min_flng AND :max_flng + AND price_per_km > 0 AND created_at >= DATE_SUB(NOW(), INTERVAL 24 HOUR) ORDER BY created_at DESC LIMIT 10"; $stmtFallback = $con->prepare($sqlFallback); diff --git a/backend/schema_primary.sql b/backend/schema_primary.sql index 543ee8fe..570e1e98 100644 --- a/backend/schema_primary.sql +++ b/backend/schema_primary.sql @@ -1954,12 +1954,21 @@ CREATE TABLE IF NOT EXISTS `scraped_competitor_prices` ( `id` BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, `task_id` varchar(100) DEFAULT NULL, `app_name` varchar(100) NOT NULL, + `competitor_name` varchar(100) NOT NULL, `start_location` varchar(255) NOT NULL, `end_location` varchar(255) NOT NULL, + `start_lat` decimal(10,7) DEFAULT NULL, + `start_lng` decimal(10,7) DEFAULT NULL, + `end_lat` decimal(10,7) DEFAULT NULL, + `end_lng` decimal(10,7) DEFAULT NULL, `price_amount` decimal(8,2) NOT NULL, + `price_per_km` decimal(8,2) NOT NULL, `currency` varchar(10) NOT NULL DEFAULT 'JOD', + `country_code` varchar(10) NOT NULL DEFAULT 'JO', `scraped_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, KEY `idx_app_name` (`app_name`), - KEY `idx_start_location` (`start_location`) + KEY `idx_competitor_name` (`competitor_name`), + KEY `idx_start_location` (`start_location`), + KEY `idx_country_code` (`country_code`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;