Update: 2026-07-06 17:00:43

backend/pricing-engine/src/analysis/clustering.ts

@@ -0,0 +1,84 @@
+import { RideSample, PricingTier } from './types';
+import { kMeans } from '../utils/math';
+
+const TIER_LABELS: Array<'economy' | 'standard' | 'premium'> = [
+  'economy',
+  'standard',
+  'premium',
+];
+
+/**
+ * Cluster rides into pricing tiers based on price_per_km using K-Means.
+ * Returns sorted tiers (economy < standard < premium).
+ */
+export function clusterTiers(
+  samples: RideSample[],
+  k: number = 3
+): PricingTier[] {
+  if (samples.length < k) {
+    return [{
+      label: 'standard',
+      samples,
+      ppkRange: [0, Infinity],
+      regression: null,
+    }];
+  }
+
+  const ppkValues = samples.map(s => s.ppk);
+  const assignments = kMeans(ppkValues, k);
+
+  // Calculate centroids for sorting
+  const centroids = new Array(k).fill(0).map((_, c) => {
+    const cluster = samples.filter((_, i) => assignments[i] === c);
+    return cluster.length > 0
+      ? cluster.reduce((sum, s) => sum + s.ppk, 0) / cluster.length
+      : 0;
+  });
+
+  // Sort clusters by centroid (ascending)
+  const sortedClusterIndices = centroids
+    .map((c, i) => ({ centroid: c, index: i }))
+    .filter(c => !isNaN(c.centroid) && c.centroid > 0)
+    .sort((a, b) => a.centroid - b.centroid);
+
+  const tiers: PricingTier[] = sortedClusterIndices.map((cluster, idx) => {
+    const clusterSamples = samples.filter((_, i) => assignments[i] === cluster.index);
+    const clusterPPKs = clusterSamples.map(s => s.ppk);
+
+    return {
+      label: TIER_LABELS[idx] || 'unknown',
+      samples: clusterSamples,
+      ppkRange: [
+        Math.min(...clusterPPKs),
+        Math.max(...clusterPPKs),
+      ],
+      regression: null,
+    };
+  });
+
+  return tiers;
+}
+
+/**
+ * Assign zones to routes based on coordinate grid.
+ * Grid size ~2.5km (0.025 degrees).
+ */
+export function assignZone(lat: number, lng: number): string {
+  const gridLat = Math.round(lat / 0.025) * 0.025;
+  const gridLng = Math.round(lng / 0.025) * 0.025;
+  return `${gridLat.toFixed(3)},${gridLng.toFixed(3)}`;
+}
+
+/**
+ * Classify zone type based on distance from city center (Amman: 31.95, 35.90).
+ */
+export function classifyZoneType(lat: number, lng: number): string {
+  const dlat = lat - 31.95;
+  const dlng = lng - 35.90;
+  const dist = Math.sqrt(dlat * dlat + dlng * dlng);
+
+  if (dist < 0.025) return 'centre';
+  if (dist < 0.050) return 'mid';
+  if (dist < 0.100) return 'suburb';
+  return 'outskirts';
+}

backend/pricing-engine/src/analysis/engine.ts

@@ -0,0 +1,136 @@
+import { RideSample, AnalysisReport } from './types';
+import { removeOutliers, groupByRoute, extractBasePrices } from './outliers';
+import { clusterTiers } from './clustering';
+import { analyzeAllTiers } from './regression';
+import { detectSurge, aggregateSurgeHours } from './surge';
+import { analyzeByZone, analyzeByZoneType } from './zone';
+
+export interface EngineOptions {
+  competitorName?: string;
+  countryCode?: string;
+  cleanOutliers?: boolean;
+  surgeThreshold?: number;
+  tierCount?: number;
+}
+
+/**
+ * Main pricing analysis engine.
+ * Orchestrates the full pipeline: fetch → clean → cluster → regress → surge → zone.
+ */
+export async function runAnalysis(
+  samples: RideSample[],
+  options: EngineOptions = {}
+): Promise<AnalysisReport> {
+  const {
+    cleanOutliers = true,
+    surgeThreshold = 0.12,
+    tierCount = 3,
+  } = options;
+
+  if (samples.length < 5) {
+    throw new Error(`Insufficient samples (${samples.length}). Need at least 5.`);
+  }
+
+  const firstSample = samples[0];
+
+  // Step 1: Remove statistical outliers (MAD on PPK)
+  const cleanSamples = cleanOutliers ? removeOutliers(samples) : samples;
+
+  // Step 2: Group by route and extract base (non-surge) prices
+  const routeGroups = groupByRoute(cleanSamples);
+  const baseSamples = extractBasePrices(routeGroups, surgeThreshold);
+
+  // Step 3: Cluster into pricing tiers by PPK
+  const rawTiers = clusterTiers(cleanSamples, tierCount);
+
+  // Step 4: Run regression on each tier
+  const analyzedTiers = analyzeAllTiers(rawTiers);
+
+  // Step 5: Detect surge patterns
+  const surgePatterns = detectSurge(cleanSamples, surgeThreshold);
+  const surgeHours = aggregateSurgeHours(surgePatterns);
+
+  // Step 6: Zone analysis
+  const zones = analyzeByZone(cleanSamples);
+  const zoneTypes = analyzeByZoneType(cleanSamples);
+
+  // Build report
+  const report: AnalysisReport = {
+    competitorName: firstSample.competitorName,
+    countryCode: firstSample.countryCode,
+    tiers: analyzedTiers,
+    surgePatterns,
+    zones,
+    totalSamples: samples.length,
+    analyzedAt: new Date().toISOString(),
+  };
+
+  // Print summary
+  printSummary(report, baseSamples, surgeHours, zoneTypes);
+
+  return report;
+}
+
+function printSummary(
+  report: AnalysisReport,
+  baseSamples: RideSample[],
+  surgeHours: ReturnType<typeof aggregateSurgeHours>,
+  zoneTypes: ReturnType<typeof analyzeByZoneType>
+): void {
+  const sep = '═══════════════════════════════════════════════════════';
+  console.log(`\n${sep}`);
+  console.log(`  📊 Pricing Analysis Report — ${report.competitorName} (${report.countryCode})`);
+  console.log(`  ${report.totalSamples} total samples, ${baseSamples.length} base-price samples`);
+  console.log(`  Analyzed at: ${report.analyzedAt}`);
+  console.log(sep);
+
+  // Tiers
+  console.log(`\n📦 PRICING TIERS:`);
+  for (const tier of report.tiers) {
+    const reg = tier.regression;
+    if (reg) {
+      const tierIcon = tier.label === 'economy' ? '💰' : tier.label === 'standard' ? '🚗' : '💎';
+      console.log(`  ${tierIcon} ${tier.label.toUpperCase()}:`);
+      console.log(`    Base Fare: ${reg.baseFare.toFixed(3)} ${report.countryCode === 'JO' ? 'JOD' : report.countryCode === 'SY' ? 'SYP' : 'CUR'}`);
+      console.log(`    Per KM:    ${reg.kmRate.toFixed(3)}`);
+      console.log(`    Per Min:   ${reg.minRate.toFixed(3)}`);
+      console.log(`    Min Fare:  ${reg.minFare.toFixed(3)} ${reg.hasMinFare ? '✅ active' : ''}`);
+      console.log(`    RMSE:      ${reg.rmse.toFixed(4)}`);
+      console.log(`    R²:        ${reg.rSquared.toFixed(4)}`);
+      console.log(`    Samples:   ${reg.sampleCount}`);
+      console.log(`    PPK range: ${tier.ppkRange[0].toFixed(3)} – ${tier.ppkRange[1].toFixed(3)}`);
+    } else {
+      console.log(`  📄 ${tier.label.toUpperCase()}: ${tier.samples.length} samples (insufficient for regression)`);
+    }
+    console.log('');
+  }
+
+  // Surge
+  if (surgeHours.length > 0) {
+    console.log(`⚡ SURGE PATTERNS (by hour-of-day):`);
+    for (const sh of surgeHours) {
+      console.log(`  Hour ${sh.hour.toString().padStart(2, '0')}:00 → avg ${sh.avgMultiplier.toFixed(3)}x (${sh.routeCount} routes)`);
+    }
+  } else {
+    console.log(`\nℹ️  No significant surge patterns detected.`);
+  }
+
+  // Zones
+  if (zoneTypes.length > 0) {
+    console.log(`\n📍 ZONE TYPE ANALYSIS:`);
+    for (const zt of zoneTypes) {
+      console.log(`  ${zt.zoneType.padEnd(12)} → avg ${zt.avgPpk.toFixed(3)}/km (${zt.sampleCount} rides)`);
+    }
+  }
+
+  // Surge route details
+  if (report.surgePatterns.length > 0) {
+    console.log(`\n🔍 TOP SURGE ROUTES:`);
+    for (const sr of report.surgePatterns.slice(0, 5)) {
+      console.log(`  ${sr.distanceKm.toFixed(1)}km → base ${sr.basePrice.toFixed(2)}, peak ${(sr.basePrice * sr.maxMultiplier).toFixed(2)} JOD (${sr.maxMultiplier.toFixed(3)}x)`);
+    }
+  }
+
+  console.log(sep);
+  console.log('');
+}

backend/pricing-engine/src/analysis/outliers.ts

@@ -0,0 +1,72 @@
+import { RideSample } from './types';
+import { findInliersMAD } from '../utils/math';
+
+/**
+ * Remove outlier rides using MAD on price_per_km.
+ * Also removes rides where price is clearly a surge outlier
+ * by comparing same-route prices.
+ */
+export function removeOutliers(
+  samples: RideSample[],
+  ppkThreshold: number = 3.5
+): RideSample[] {
+  if (samples.length < 10) return samples;
+
+  const ppkValues = samples.map(s => s.ppk);
+  const inlierIndices = new Set(findInliersMAD(ppkValues, ppkThreshold));
+
+  // Also remove rides with price_per_km > 3x the median
+  const sortedPPK = [...ppkValues].sort((a, b) => a - b);
+  const medianPPK = sortedPPK[Math.floor(sortedPPK.length / 2)];
+  const upperBound = medianPPK * 3;
+
+  return samples.filter((s, i) =>
+    inlierIndices.has(i) && s.ppk <= upperBound && s.ppk > 0
+  );
+}
+
+/**
+ * Group samples by unique route (start/end coordinates rounded to 4 decimals).
+ */
+export function groupByRoute(samples: RideSample[]): Map<string, RideSample[]> {
+  const groups = new Map<string, RideSample[]>();
+  for (const s of samples) {
+    const key = `${s.startLat.toFixed(4)},${s.startLng.toFixed(4)}->${s.endLat.toFixed(4)},${s.endLng.toFixed(4)}`;
+    if (!groups.has(key)) groups.set(key, []);
+    groups.get(key)!.push(s);
+  }
+  return groups;
+}
+
+/**
+ * For each route, keep only the lowest price (non-surge baseline)
+ * if the price variation exceeds threshold.
+ */
+export function extractBasePrices(
+  groups: Map<string, RideSample[]>,
+  surgeThreshold: number = 0.15
+): RideSample[] {
+  const base: RideSample[] = [];
+
+  for (const [, rides] of groups) {
+    if (rides.length === 1) {
+      base.push(rides[0]);
+      continue;
+    }
+
+    const prices = rides.map(r => r.price);
+    const minPrice = Math.min(...prices);
+    const maxPrice = Math.max(...prices);
+
+    // If variation is small, use all rides
+    if (maxPrice - minPrice <= surgeThreshold) {
+      base.push(...rides);
+    } else {
+      // Only keep rides within 5% of minimum price
+      const baseRides = rides.filter(r => r.price <= minPrice * 1.05);
+      base.push(...baseRides);
+    }
+  }
+
+  return base;
+}

backend/pricing-engine/src/analysis/regression.ts

@@ -0,0 +1,120 @@
+import { PricingTier, RegressionResult, RideSample } from './types';
+import {
+  robustMultipleLinearRegression,
+  calcRMSE,
+  calcRSquared,
+  detectMinimumFare,
+} from '../utils/math';
+import { mean } from 'simple-statistics';
+
+/**
+ * Run multiple linear regression on each pricing tier.
+ * Detects minimum fare and computes RMSE/R².
+ */
+export function analyzeTier(tier: PricingTier): PricingTier {
+  const samples = tier.samples;
+  if (samples.length < 5) {
+    tier.regression = null;
+    return tier;
+  }
+
+  // Primary model: price = baseFare + kmRate * dist + minRate * dur
+  // We use robust regression to strip out surge outliers and find the floor price
+  const mlrResult = robustMultipleLinearRegression(
+    samples.map(s => ({
+      distance_km: s.distance_km,
+      duration_min: s.duration_min,
+      price: s.price,
+    }))
+  );
+
+  if (!mlrResult) {
+    tier.regression = null;
+    return tier;
+  }
+
+  // Predict and compute RMSE/R²
+  const actualPrices = samples.map(s => s.price);
+  const predictedPrices = samples.map(s =>
+    mlrResult.baseFare +
+    mlrResult.kmRate * s.distance_km +
+    mlrResult.minRate * s.duration_min
+  );
+
+  const rmse = calcRMSE(actualPrices, predictedPrices);
+  const rSquared = calcRSquared(actualPrices, predictedPrices);
+
+  // Detect minimum fare
+  const minFare = detectMinimumFare(
+    samples.map(s => s.distance_km),
+    samples.map(s => s.price),
+    mlrResult.kmRate
+  );
+
+  // If minFare is detected and the short-ride residuals improve,
+  // apply minFare-adjusted model
+  let hasMinFare = false;
+  let adjustedRMSE = rmse;
+  let adjustedRSquared = rSquared;
+
+  if (minFare && minFare > 0) {
+    const adjustedPredicted = samples.map(s => {
+      const raw = mlrResult.baseFare + mlrResult.kmRate * s.distance_km + mlrResult.minRate * s.duration_min;
+      return Math.max(raw, minFare);
+    });
+    const adjRmse = calcRMSE(actualPrices, adjustedPredicted);
+    const adjRsq = calcRSquared(actualPrices, adjustedPredicted);
+
+    // If minimum fare improves the fit, use it
+    if (adjRmse < rmse) {
+      hasMinFare = true;
+      adjustedRMSE = adjRmse;
+      adjustedRSquared = adjRsq;
+    }
+  }
+
+  tier.regression = {
+    baseFare: mlrResult.baseFare,
+    kmRate: mlrResult.kmRate,
+    minRate: mlrResult.minRate,
+    minFare: minFare || 0,
+    rmse: adjustedRMSE,
+    rSquared: adjustedRSquared,
+    sampleCount: samples.length,
+    hasMinFare,
+  };
+
+  return tier;
+}
+
+/**
+ * Run regression on all tiers.
+ */
+export function analyzeAllTiers(tiers: PricingTier[]): PricingTier[] {
+  return tiers.map(tier => analyzeTier(tier));
+}
+
+/**
+ * Simple distance-only regression for comparison.
+ * price = kmRate * dist
+ */
+export function distanceOnlyRegression(
+  samples: RideSample[]
+): { kmRate: number; rmse: number } | null {
+  if (samples.length < 3) return null;
+
+  const distances = samples.map(s => s.distance_km);
+  const prices = samples.map(s => s.price);
+
+  // Simple average of price/km
+  const ratios = distances.map((d, i) => d > 0 ? prices[i] / d : 0)
+    .filter(r => r > 0 && isFinite(r));
+
+  if (ratios.length < 3) return null;
+
+  const kmRate = mean(ratios);
+  const predicted = distances.map(d => kmRate * d);
+  const rmse = calcRMSE(prices, predicted);
+
+  return { kmRate, rmse };
+}

backend/pricing-engine/src/analysis/surge.ts

@@ -0,0 +1,96 @@
+import { RideSample, SurgeResult } from './types';
+import { groupByRoute } from './outliers';
+
+/**
+ * Detect surge pricing by analyzing price variation per route across time.
+ * For routes with multiple samples, identifies base price (minimum)
+ * and surge multipliers per hour-of-day (aggregated across all days).
+ */
+export function detectSurge(
+  samples: RideSample[],
+  surgeThreshold: number = 0.12
+): SurgeResult[] {
+  const routes = groupByRoute(samples);
+  const results: SurgeResult[] = [];
+
+  for (const [routeKey, rides] of routes) {
+    if (rides.length < 3) continue;
+
+    const prices = rides.map(r => r.price);
+    const minPrice = Math.min(...prices);
+    const maxPrice = Math.max(...prices);
+
+    // Only analyze routes with meaningful variation
+    if (maxPrice - minPrice <= surgeThreshold) continue;
+
+    // Find the time of the base price
+    const baseRide = rides.find(r => r.price === minPrice);
+
+    // Aggregate surge by hour-of-day across ALL days
+    const surgeByHour = new Map<number, number[]>();
+    for (const r of rides) {
+      const hour = r.scrapedAt.getHours();
+      if (!surgeByHour.has(hour)) surgeByHour.set(hour, []);
+      surgeByHour.get(hour)!.push(r.price);
+    }
+
+    const surgePrices: SurgeResult['surgePrices'] = [];
+    let maxMultiplier = 1;
+
+    // Sort hours and compute average multiplier per hour
+    for (const [hour, hourPrices] of [...surgeByHour.entries()].sort((a, b) => a[0] - b[0])) {
+      const avgTimePrice = hourPrices.reduce((a, b) => a + b, 0) / hourPrices.length;
+      const multiplier = minPrice > 0 ? avgTimePrice / minPrice : 1;
+      if (multiplier > maxMultiplier) maxMultiplier = multiplier;
+
+      surgePrices.push({
+        time: `${hour.toString().padStart(2, '0')}:00`,
+        price: Math.round(avgTimePrice * 100) / 100,
+        multiplier: Math.round(multiplier * 1000) / 1000,
+      });
+    }
+
+    if (maxMultiplier > 1.05) {
+      results.push({
+        routeKey,
+        distanceKm: rides[0].distance_km,
+        basePrice: minPrice,
+        baseTime: baseRide ? baseRide.scrapedAt.toISOString() : '',
+        surgePrices,
+        maxMultiplier: Math.round(maxMultiplier * 1000) / 1000,
+      });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Aggregate surge patterns across all routes to find global peak hours.
+ * Groups by hour-of-day (0-23) across all detected routes.
+ */
+export function aggregateSurgeHours(
+  surgeResults: SurgeResult[]
+): Array<{ hour: number; avgMultiplier: number; routeCount: number }> {
+  const hourlyData = new Map<number, number[]>();
+
+  for (const sr of surgeResults) {
+    for (const sp of sr.surgePrices) {
+      const hour = parseInt(sp.time.split(':')[0]);
+      if (!isNaN(hour)) {
+        if (!hourlyData.has(hour)) hourlyData.set(hour, []);
+        hourlyData.get(hour)!.push(sp.multiplier);
+      }
+    }
+  }
+
+  return Array.from(hourlyData.entries())
+    .map(([hour, multipliers]) => ({
+      hour,
+      avgMultiplier: Math.round(
+        (multipliers.reduce((a, b) => a + b, 0) / multipliers.length) * 1000
+      ) / 1000,
+      routeCount: multipliers.length,
+    }))
+    .sort((a, b) => a.hour - b.hour);
+}

backend/pricing-engine/src/analysis/types.ts

@@ -0,0 +1,89 @@
+export interface ScrapedRide {
+  id: number;
+  task_id: string;
+  app_name: string;
+  competitor_name: string;
+  start_lat: number;
+  start_lng: number;
+  end_lat: number;
+  end_lng: number;
+  price_amount: number;
+  price_per_km: number;
+  distance_km: number;
+  duration_min: number;
+  currency: string;
+  country_code: string;
+  scraped_at: string;
+  created_at: string;
+}
+
+export interface RideSample {
+  distance_km: number;
+  duration_min: number;
+  price: number;
+  ppk: number;
+  startLat: number;
+  startLng: number;
+  endLat: number;
+  endLng: number;
+  scrapedAt: Date;
+  competitorName: string;
+  countryCode: string;
+}
+
+export interface RouteGroup {
+  key: string;
+  rides: RideSample[];
+  minPrice: number;
+  maxPrice: number;
+  avgPrice: number;
+  distanceKm: number;
+  durationMin: number;
+  surgeMultiplier: number | null;
+}
+
+export interface PricingTier {
+  label: 'economy' | 'standard' | 'premium' | 'unknown';
+  samples: RideSample[];
+  ppkRange: [number, number];
+  regression: RegressionResult | null;
+}
+
+export interface RegressionResult {
+  baseFare: number;
+  kmRate: number;
+  minRate: number;
+  minFare: number;
+  rmse: number;
+  rSquared: number;
+  sampleCount: number;
+  hasMinFare: boolean;
+}
+
+export interface SurgeResult {
+  routeKey: string;
+  distanceKm: number;
+  basePrice: number;
+  baseTime: string;
+  surgePrices: Array<{ time: string; price: number; multiplier: number }>;
+  maxMultiplier: number;
+}
+
+export interface ZoneAnalysis {
+  zoneKey: string;
+  centerLat: number;
+  centerLng: number;
+  samples: RideSample[];
+  avgPpk: number;
+  tierDistribution: Record<string, number>;
+}
+
+export interface AnalysisReport {
+  competitorName: string;
+  countryCode: string;
+  tiers: PricingTier[];
+  surgePatterns: SurgeResult[];
+  zones: ZoneAnalysis[];
+  totalSamples: number;
+  analyzedAt: string;
+}

backend/pricing-engine/src/analysis/zone.ts

@@ -0,0 +1,80 @@
+import { RideSample, ZoneAnalysis } from './types';
+import { assignZone, classifyZoneType } from './clustering';
+
+/**
+ * Analyze pricing by geographical zone (2.5km grid).
+ * Groups samples into zones and computes per-zone statistics.
+ */
+export function analyzeByZone(samples: RideSample[]): ZoneAnalysis[] {
+  const zoneMap = new Map<string, RideSample[]>();
+
+  for (const s of samples) {
+    // Use start location for zone assignment
+    const zone = assignZone(s.startLat, s.startLng);
+    if (!zoneMap.has(zone)) zoneMap.set(zone, []);
+    zoneMap.get(zone)!.push(s);
+  }
+
+  const results: ZoneAnalysis[] = [];
+
+  for (const [zoneKey, zoneSamples] of zoneMap) {
+    if (zoneSamples.length < 3) continue;
+
+    const ppkValues = zoneSamples.map(s => s.ppk);
+    const avgPpk = Math.round(
+      (ppkValues.reduce((a, b) => a + b, 0) / ppkValues.length) * 1000
+    ) / 1000;
+
+    // Count by tier — thresholds depend on currency scale
+    const tierCounts: Record<string, number> = {};
+    const sample = zoneSamples[0];
+    const isHighDenom = sample.countryCode === 'SY' || sample.countryCode === 'IQ';
+    const econThreshold = isHighDenom ? 15 : 0.35;
+    const stdThreshold = isHighDenom ? 40 : 0.55;
+
+    for (const s of zoneSamples) {
+      const tier =
+        s.ppk < econThreshold ? 'economy' :
+        s.ppk < stdThreshold  ? 'standard' : 'premium';
+      tierCounts[tier] = (tierCounts[tier] || 0) + 1;
+    }
+
+    const [latStr, lngStr] = zoneKey.split(',');
+    results.push({
+      zoneKey,
+      centerLat: parseFloat(latStr),
+      centerLng: parseFloat(lngStr),
+      samples: zoneSamples,
+      avgPpk,
+      tierDistribution: tierCounts,
+    });
+  }
+
+  return results.sort((a, b) => a.avgPpk - b.avgPpk);
+}
+
+/**
+ * Analyze pricing by zone type (centre, mid, suburb, outskirts).
+ */
+export function analyzeByZoneType(
+  samples: RideSample[]
+): Array<{ zoneType: string; avgPpk: number; sampleCount: number; avgPrice: number }> {
+  const typeMap = new Map<string, number[]>();
+
+  for (const s of samples) {
+    const zoneType = classifyZoneType(s.startLat, s.startLng);
+    if (!typeMap.has(zoneType)) typeMap.set(zoneType, []);
+    typeMap.get(zoneType)!.push(s.ppk);
+  }
+
+  return Array.from(typeMap.entries())
+    .map(([zoneType, ppks]) => ({
+      zoneType,
+      avgPpk: Math.round(
+        (ppks.reduce((a, b) => a + b, 0) / ppks.length) * 1000
+      ) / 1000,
+      sampleCount: ppks.length,
+      avgPrice: 0, // calculated below if needed
+    }))
+    .sort((a, b) => a.avgPpk - b.avgPpk);
+}