Add MultimodalChatWithToolsProvider

appinfo/info.xml

@@ -101,7 +101,7 @@ Negative:
 
 Learn more about the Nextcloud Ethical AI Rating [in our blog](https://nextcloud.com/blog/nextcloud-ethical-ai-rating/).
 ]]>	</description>
-	<version>4.5.1</version>
+	<version>4.6.0-dev</version>
 	<licence>agpl</licence>
 	<author>Julien Veyssier</author>
 	<namespace>OpenAi</namespace>

lib/AppInfo/Application.php

@@ -143,11 +143,12 @@ public function register(IRegistrationContext $context): void {
 			$context->registerTaskProcessingProvider(EmojiProvider::class);
 			$context->registerTaskProcessingProvider(ChangeToneProvider::class);
 			$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\TextToTextChatWithToolsProvider::class);
+			$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\MultimodalChatWithToolsProvider::class);
 			$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ProofreadProvider::class);
 			if (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToTextReformatParagraphs')) {
 				$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider::class);
 			}
-			if ($isUsingOpenAI || $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled') === '1') {
+			if ($this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1') === '1') {
 				$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\AnalyzeImagesProvider::class);
 			}
 		}

lib/Migration/Version040600Date20260708151000.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\OpenAi\Migration;
+
+use Closure;
+use OCA\OpenAi\AppInfo\Application;
+use OCP\IAppConfig;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version040600Date20260708151000 extends SimpleMigrationStep {
+
+	public function __construct(
+		private IAppConfig $appConfig,
+	) {
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure
+	 * @param array $options
+	 */
+	public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
+		$unset = '__unset__';
+		$newValue = $this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', $unset);
+		if ($newValue !== $unset) {
+			return;
+		}
+
+		$oldValue = $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled', $unset);
+		if (!in_array($oldValue, ['0', '1'], true)) {
+			return;
+		}
+
+		$this->appConfig->setValueString(Application::APP_ID, 'multimodal_image_enabled', $oldValue);
+	}
+}

lib/Service/OpenAiAPIService.php

@@ -29,6 +29,7 @@
 use OCP\IL10N;
 use OCP\Lock\LockedException;
 use OCP\Notification\IManager as INotificationManager;
+use OCP\TaskProcessing\Exception\ProcessingException;
 use OCP\TaskProcessing\Exception\UserFacingProcessingException;
 use OCP\TaskProcessing\ShapeEnumValue;
 use Psr\Log\LoggerInterface;
@@ -51,6 +52,7 @@ public function __construct(
 		private QuotaUsageMapper $quotaUsageMapper,
 		private OpenAiSettingsService $openAiSettingsService,
 		private StreamingService $streamingService,
+		private OpenAiFileService $openAiFileService,
 		private INotificationManager $notificationManager,
 		private QuotaRuleService $quotaRuleService,
 		IClientService $clientService,
@@ -552,8 +554,7 @@ public function createStreamedChatCompletion(
 		?array $extraParams = null,
 		?string $toolMessage = null,
 		?array $tools = null,
-		?string $userAudioPromptBase64 = null,
-		?string $userAudioPromptFormat = null,
+		?array $files = null,
 	): \Generator {
 		if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT)) {
 			throw new Exception($this->l10n->t('Text generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS);
@@ -570,8 +571,7 @@ public function createStreamedChatCompletion(
 			$extraParams,
 			$toolMessage,
 			$tools,
-			$userAudioPromptBase64,
-			$userAudioPromptFormat,
+			$files,
 			true,
 		);
 
@@ -612,13 +612,11 @@ public function createChatCompletion(
 		?array $extraParams = null,
 		?string $toolMessage = null,
 		?array $tools = null,
-		?string $userAudioPromptBase64 = null,
-		?string $userAudioPromptFormat = null,
+		?array $files = null,
 	): array {
 		$response = $this->requestChatCompletion(
 			$userId, $model, $userPrompt, $systemPrompt, $history,
-			$n, $maxTokens, $extraParams, $toolMessage, $tools,
-			$userAudioPromptBase64, $userAudioPromptFormat,
+			$n, $maxTokens, $extraParams, $toolMessage, $tools, $files,
 			false,
 		);
 
@@ -646,8 +644,7 @@ public function createChatCompletion(
 	 * @param array|null $extraParams
 	 * @param string|null $toolMessage JSON string with role, content, tool_call_id
 	 * @param array|null $tools
-	 * @param string|null $userAudioPromptBase64
-	 * @param string|null $userAudioPromptFormat
+	 * @param array|null $files Array of File objects
 	 * @return array{messages?: array<string>, tool_calls?: array<string>, audio_messages?: list<array<string, mixed>>, usage?: array<string, mixed>}
 	 * @throws Exception
 	 */
@@ -662,8 +659,7 @@ public function requestChatCompletion(
 		?array $extraParams = null,
 		?string $toolMessage = null,
 		?array $tools = null,
-		?string $userAudioPromptBase64 = null,
-		?string $userAudioPromptFormat = null,
+		?array $files = null,
 		bool $stream = false,
 	): array {
 		if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT)) {
@@ -681,8 +677,7 @@ public function requestChatCompletion(
 			$extraParams,
 			$toolMessage,
 			$tools,
-			$userAudioPromptBase64,
-			$userAudioPromptFormat,
+			$files,
 			$stream,
 		);
 
@@ -700,8 +695,7 @@ public function requestChatCompletion(
 	 * @param array|null $extraParams
 	 * @param string|null $toolMessage
 	 * @param array|null $tools
-	 * @param string|null $userAudioPromptBase64
-	 * @param string|null $userAudioPromptFormat
+	 * @param array|null $files Array of File objects
 	 * @param bool $stream
 	 * @return array<string, mixed>
 	 */
@@ -716,8 +710,7 @@ private function buildChatCompletionRequestParams(
 		?array $extraParams = null,
 		?string $toolMessage = null,
 		?array $tools = null,
-		?string $userAudioPromptBase64 = null,
-		?string $userAudioPromptFormat = null,
+		?array $files = null,
 		bool $stream = false,
 	): array {
 		$modelRequestParam = $model === Application::DEFAULT_MODEL_ID
@@ -758,32 +751,62 @@ private function buildChatCompletionRequestParams(
 						return $formattedToolCall;
 					}, $message['tool_calls']);
 				}
+				// Handle file attachments in the history
+				if (isset($message['content']) && is_array($message['content'])) {
+					$content = [];
+					foreach ($message['content'] as $item) {
+						if (!isset($item['type'])) {
+							throw new UserFacingProcessingException(
+								'Invalid message history content',
+								0,
+								null,
+								$this->l10n->t('Invalid message history content'),
+							);
+						}
+						if ($item['type'] === 'file') {
+							if (!isset($item['file_id'])) {
+								throw new UserFacingProcessingException(
+									'Invalid message history content',
+									0,
+									null,
+									$this->l10n->t('Invalid message history content'),
+								);
+							}
+							// If the history contains a file that isn't supported anymore we should skip it so the chat isn't broken
+							try {
+								$content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null));
+							} catch (ProcessingException|UserFacingProcessingException $e) {
+								$this->logger->warning('Could not build file content from id: ' . $item['file_id'] . '. Error: ' . $e->getMessage(), ['app' => Application::APP_ID]);
+							}
+						} else {
+							$content[] = $item;
+						}
+					}
+					$message['content'] = $content;
+				}
 				$messages[] = $message;
 			}
 		}
-		if ($userAudioPromptBase64 !== null) {
-			// if there is audio, use the new message format (content is a list of objects)
-			$message = [
-				'role' => 'user',
-				'content' => [
-					[
-						'type' => 'input_audio',
-						'input_audio' => [
-							'data' => $userAudioPromptBase64,
-							'format' => $userAudioPromptFormat ?? 'wav',
-						],
-					],
-				],
-			];
+		// Attach all files when necessary
+		if ($files !== null && count($files) > 0) {
+			if (count($files) > 500) {
+				throw new UserFacingProcessingException($this->l10n->t('Too many files. Max is 500'), Http::STATUS_BAD_REQUEST);
+			}
+			$content = [];
+			foreach ($files as $file) {
+				$content = array_merge($content, $this->openAiFileService->buildFileContentFromFile($file));
+			}
 			if ($userPrompt !== null) {
-				$message['content'][] = [
+				$content[] = [
 					'type' => 'text',
 					'text' => $userPrompt,
 				];
 			}
-			$messages[] = $message;
+			$messages[] = [
+				'role' => 'user',
+				'content' => $content,
+			];
 		} elseif ($userPrompt !== null) {
-			// if there is only text, use the old message format (content is a string)
 			$messages[] = [
 				'role' => 'user',
 				'content' => $userPrompt,
@@ -1463,6 +1486,7 @@ private function normalizeChatCompletionResponse(array $response): array {
 			'reasoning_messages' => [],
 			'tool_calls' => [],
 			'audio_messages' => [],
+			'images' => [],
 		];
 
 		foreach ($response['choices'] as $choice) {
@@ -1515,6 +1539,11 @@ private function normalizeChatCompletionResponse(array $response): array {
 			if (isset($choice['message']['audio'], $choice['message']['audio']['data']) && is_string($choice['message']['audio']['data'])) {
 				$completions['audio_messages'][] = $choice['message'];
 			}
+			if (isset($choice['message']['images']) && is_array($choice['message']['images'])) {
+				foreach ($choice['message']['images'] as $image) {
+					$completions['images'][] = $image;
+				}
+			}
 		}
 
 		return $completions;
@@ -1598,7 +1627,6 @@ public function autoDetectFeatures(): array {
 		$config['stt_provider_enabled'] = $this->isSTTAvailable();
 		$config['tts_provider_enabled'] = $this->isTTSAvailable();
 		$this->openAiSettingsService->setAdminConfig($config);
-		$config['analyze_image_provider_enabled'] = $this->openAiSettingsService->getAnalyzeImageProviderEnabled();
 		return $config;
 	}
 }