Deploy: 2026-05-22 01:13:50

backend/app/Controllers/WhatsAppController.php

@@ -218,13 +218,14 @@ class WhatsAppController extends BaseController
             }
             
             // 2. Log the incoming message in history log
+            $isAudioMsg = !empty($msgData['audio']) && !empty($msgData['mimeType']);
             \App\Models\MessageLog::logMessage([
                 'company_id' => $session['company_id'],
                 'session_id' => $session['id'],
                 'contact_phone' => $msgData['phone'],
                 'direction' => 'inbound',
-                'message_type' => 'text',
-                'message_body' => $msgData['body'],
+                'message_type' => $isAudioMsg ? 'audio' : 'text',
+                'message_body' => $isAudioMsg ? ($msgData['body'] ?: '[Voice Note]') : $msgData['body'],
                 'whatsapp_message_id' => $msgData['id'],
                 'status' => 'read'
             ]);
@@ -287,14 +288,19 @@ class WhatsAppController extends BaseController
                 return;
             }
 
-            $incomingText = trim($msgData['body']);
-            if (empty($incomingText)) {
+            $incomingText = isset($msgData['body']) ? trim($msgData['body']) : '';
+            $hasAudio = !empty($msgData['audio']) && !empty($msgData['mimeType']);
+
+            if (empty($incomingText) && !$hasAudio) {
                 return;
             }
 
             $replyText = null;
 
             if ($rule['trigger_type'] === 'keyword') {
+                if (empty($incomingText)) {
+                    return;
+                }
                 $keywords = array_filter(array_map('trim', explode(',', $rule['keyword'])));
                 $matched = false;
                 foreach ($keywords as $kw) {
@@ -317,7 +323,15 @@ class WhatsAppController extends BaseController
                 // Enforce language matching rule dynamically
                 $systemPrompt .= "\n\nIMPORTANT LANGUAGE RULE: Detect the language of the incoming message. If the incoming message is in English, you MUST reply in English. If the incoming message is in Arabic, you MUST reply in Arabic. Override any default language instruction to match the user's language.";
                 
-                $replyText = \App\Services\GeminiService::generateResponse($apiKey, $systemPrompt, $incomingText);
+                if ($hasAudio) {
+                    $mimeType = $msgData['mimeType'];
+                    if (strpos($mimeType, ';') !== false) {
+                        $mimeType = trim(explode(';', $mimeType)[0]);
+                    }
+                    $replyText = \App\Services\GeminiService::generateResponseFromAudio($apiKey, $systemPrompt, $msgData['audio'], $mimeType);
+                } else {
+                    $replyText = \App\Services\GeminiService::generateResponse($apiKey, $systemPrompt, $incomingText);
+                }
             }
 
             if (!empty($replyText)) {

backend/app/Services/GeminiService.php

@@ -96,4 +96,62 @@ class GeminiService
         $data = json_decode($response, true);
         return $data['candidates'][0]['content']['parts'][0]['text'] ?? null;
     }
+
+    /**
+     * Call Gemini API with audio inline data and system instruction to generate a response text
+     */
+    public static function generateResponseFromAudio(string $apiKey, string $systemPrompt, string $audioBase64, string $mimeType): ?string
+    {
+        $url = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-flash-lite-latest:generateContent?key=' . $apiKey;
+
+        // Clean mimeType if it contains codec info
+        if (strpos($mimeType, ';') !== false) {
+            $mimeType = trim(explode(';', $mimeType)[0]);
+        }
+
+        $payload = json_encode([
+            'contents' => [
+                [
+                    'role' => 'user',
+                    'parts' => [
+                        [
+                            'inlineData' => [
+                                'mimeType' => $mimeType,
+                                'data' => $audioBase64
+                            ]
+                        ],
+                        [
+                            'text' => "استمع إلى التسجيل الصوتي المرفق وأجب عليه مباشرة باللغة المناسبة بناءً على الإرشادات المحددة."
+                        ]
+                    ]
+                ]
+            ],
+            'systemInstruction' => [
+                'parts' => [
+                    ['text' => $systemPrompt]
+                ]
+            ]
+        ]);
+
+        $ch = curl_init($url);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+        curl_setopt($ch, CURLOPT_POST, true);
+        curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
+        curl_setopt($ch, CURLOPT_HTTPHEADER, [
+            'Content-Type: application/json'
+        ]);
+        curl_setopt($ch, CURLOPT_TIMEOUT, 35); // 35 seconds timeout for audio analysis
+
+        $response = curl_exec($ch);
+        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+        curl_close($ch);
+
+        if ($httpCode !== 200) {
+            error_log("[Gemini Audio Response Error] HTTP " . $httpCode . " | Response: " . $response);
+            return null;
+        }
+
+        $data = json_decode($response, true);
+        return $data['candidates'][0]['content']['parts'][0]['text'] ?? null;
+    }
 }