Deploy on 2026-06-05 17:13:07

app/Services/Crawler/AiAnalyzer.php

@@ -8,6 +8,7 @@ class AiAnalyzer
 {
     private ?string $apiKey;
     private string $model;
+    private static ?float $lastCallTime = null;
 
     public function __construct()
     {
@@ -108,6 +109,16 @@ PROMPT;
      */
     private function callGemini(string $prompt): string
     {
+        if (self::$lastCallTime !== null) {
+            $elapsed = microtime(true) - self::$lastCallTime;
+            $minInterval = 4.5; // Space out requests to under 15 RPM
+            if ($elapsed < $minInterval) {
+                $sleepTime = $minInterval - $elapsed;
+                usleep((int)($sleepTime * 1000000));
+            }
+        }
+        self::$lastCallTime = microtime(true);
+
         $url = "https://generativelanguage.googleapis.com/v1beta/models/{$this->model}:generateContent?key={$this->apiKey}";
 
         $payload = json_encode([

app/Services/Crawler/Collector.php

@@ -16,6 +16,10 @@ class Collector
     private ActivityLogger $logger;
     private TelegramNotifier $notifier;
 
+    private $lockHandle = null;
+    private int $maxNewPerFeed = 5;
+    private int $maxNewTotal = 100;
+
     public function __construct(
         Connection $connection,
         RssParser $rssParser,
@@ -35,6 +39,10 @@ class Collector
      */
     public function collectAll(): array
     {
+        if (!$this->acquireLock()) {
+            throw new \Exception("Another collector run is currently in progress.");
+        }
+
         $results = [
             'total_sources' => 0,
             'processed' => 0,
@@ -44,12 +52,28 @@ class Collector
             'details' => [],
         ];
 
+        try {
+            $this->maxNewPerFeed = (int)($this->getSetting('crawler_max_new_per_feed') ?: 5);
+            $this->maxNewTotal = (int)($this->getSetting('crawler_max_new_total') ?: 100);
+
             $sources = $this->getActiveSources();
+            $totalNewProcessed = 0;
 
             foreach ($sources as $source) {
                 $results['total_sources']++;
+
+                if ($totalNewProcessed >= $this->maxNewTotal) {
+                    $results['details'][] = [
+                        'source' => $source['name'],
+                        'type' => $source['type'],
+                        'status' => 'skipped',
+                        'reason' => 'Global limit reached',
+                    ];
+                    continue;
+                }
+
                 try {
-                $result = $this->collectSource($source);
+                    $result = $this->collectSource($source, $totalNewProcessed);
                     $results['processed']++;
                     $results['new_opportunities'] += $result['opportunities'];
                     $results['new_organizations'] += $result['organizations'];
@@ -86,13 +110,17 @@ class Collector
                 $this->notifier->notifyCollectorResults($results);
             }
 
+        } finally {
+            $this->releaseLock();
+        }
+
         return $results;
     }
 
     /**
      * Collect from a single source.
      */
-    public function collectSource(array $source): array
+    public function collectSource(array $source, int &$totalNewProcessed): array
     {
         $result = [
             'entries_found' => 0,
@@ -104,14 +132,60 @@ class Collector
             $entries = $this->rssParser->fetchEntries($source['url']);
             $result['entries_found'] = count($entries);
 
+            $newEntriesInSource = 0;
             foreach ($entries as $entry) {
+                if ($totalNewProcessed >= $this->maxNewTotal) {
+                    break;
+                }
+
+                if ($this->rssParser->entryExists($entry['url'])) {
+                    continue;
+                }
+
+                if ($newEntriesInSource >= $this->maxNewPerFeed) {
+                    break; // stop processing entries since they are sorted newest-first
+                }
+
                 $this->processEntry($entry, $source, $result);
+                $newEntriesInSource++;
+                $totalNewProcessed++;
             }
         }
 
         return $result;
     }
 
+    /**
+     * Acquire run lock.
+     */
+    private function acquireLock(): bool
+    {
+        $lockFile = __DIR__ . '/../../../collector.lock';
+        $this->lockHandle = @fopen($lockFile, 'c');
+        if (!$this->lockHandle) {
+            return false;
+        }
+        if (!flock($this->lockHandle, LOCK_EX | LOCK_NB)) {
+            fclose($this->lockHandle);
+            $this->lockHandle = null;
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Release run lock.
+     */
+    private function releaseLock(): void
+    {
+        if ($this->lockHandle) {
+            flock($this->lockHandle, LOCK_UN);
+            fclose($this->lockHandle);
+            @unlink(__DIR__ . '/../../../collector.lock');
+            $this->lockHandle = null;
+        }
+    }
+
     /**
      * Process a single entry: analyze, save opportunity, save organization.
      */

database/seeds/DatabaseSeeder.php

@@ -89,6 +89,8 @@ class DatabaseSeeder
             ['key' => 'system_email', 'value' => 'info@scoutiq.intaleqapp.com', 'description' => 'Primary sender email address.'],
             ['key' => 'crawler_enabled', 'value' => 'true', 'description' => 'Global toggle for data collection crawlers.'],
             ['key' => 'crawler_interval_hours', 'value' => '24', 'description' => 'Delay between crawler runs.'],
+            ['key' => 'crawler_max_new_per_feed', 'value' => '5', 'description' => 'Maximum new opportunities to process per feed in a single run.'],
+            ['key' => 'crawler_max_new_total', 'value' => '100', 'description' => 'Maximum total new opportunities to process in a single collector run.'],
         ];
 
         foreach ($settings as $setting) {