From 0568acbeb546287487090baf3670ad64408079a1 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Wed, 8 Jul 2026 13:46:27 -0400 Subject: [PATCH 01/10] Add MultimodalChatWithToolsProvider Signed-off-by: Lukas Schaefer --- lib/AppInfo/Application.php | 1 + lib/Service/OpenAiAPIService.php | 18 +- lib/Service/OpenAiFileService.php | 205 +++++++++++++++++ .../MultimodalChatWithToolsProvider.php | 210 ++++++++++++++++++ psalm.xml | 1 + tests/unit/Providers/OpenAiProviderTest.php | 5 + tests/unit/Quota/QuotaTest.php | 5 + tests/unit/Service/ServiceOverrideTest.php | 5 + 8 files changed, 449 insertions(+), 1 deletion(-) create mode 100644 lib/Service/OpenAiFileService.php create mode 100644 lib/TaskProcessing/MultimodalChatWithToolsProvider.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 2804e5fd..3ae8acee 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -143,6 +143,7 @@ public function register(IRegistrationContext $context): void { $context->registerTaskProcessingProvider(EmojiProvider::class); $context->registerTaskProcessingProvider(ChangeToneProvider::class); $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\TextToTextChatWithToolsProvider::class); + $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\MultimodalChatWithToolsProvider::class); $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ProofreadProvider::class); if (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToTextReformatParagraphs')) { $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider::class); diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 0c1679d5..64623a04 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -51,6 +51,7 @@ public function __construct( private QuotaUsageMapper $quotaUsageMapper, private OpenAiSettingsService $openAiSettingsService, private StreamingService $streamingService, + private OpenAiFileService $openAiFileService, private INotificationManager $notificationManager, private QuotaRuleService $quotaRuleService, IClientService $clientService, @@ -533,6 +534,7 @@ public function createStreamedChatCompletion( ?array $tools = null, ?string $userAudioPromptBase64 = null, ?string $userAudioPromptFormat = null, + ?array $files = null, ): \Generator { if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT)) { throw new Exception($this->l10n->t('Text generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS); @@ -551,6 +553,7 @@ public function createStreamedChatCompletion( $tools, $userAudioPromptBase64, $userAudioPromptFormat, + $files, true, ); @@ -593,11 +596,12 @@ public function createChatCompletion( ?array $tools = null, ?string $userAudioPromptBase64 = null, ?string $userAudioPromptFormat = null, + ?array $files = null, ): array { $response = $this->requestChatCompletion( $userId, $model, $userPrompt, $systemPrompt, $history, $n, $maxTokens, $extraParams, $toolMessage, $tools, - $userAudioPromptBase64, $userAudioPromptFormat, + $userAudioPromptBase64, $userAudioPromptFormat, $files, false, ); @@ -627,6 +631,7 @@ public function createChatCompletion( * @param array|null $tools * @param string|null $userAudioPromptBase64 * @param string|null $userAudioPromptFormat + * @param array|null $files Array of File objects * @return array{messages?: array, tool_calls?: array, audio_messages?: list>, usage?: array} * @throws Exception */ @@ -643,6 +648,7 @@ public function requestChatCompletion( ?array $tools = null, ?string $userAudioPromptBase64 = null, ?string $userAudioPromptFormat = null, + ?array $files = null, bool $stream = false, ): array { if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT)) { @@ -662,6 +668,7 @@ public function requestChatCompletion( $tools, $userAudioPromptBase64, $userAudioPromptFormat, + $files, $stream, ); @@ -681,6 +688,7 @@ public function requestChatCompletion( * @param array|null $tools * @param string|null $userAudioPromptBase64 * @param string|null $userAudioPromptFormat + * @param array|null $files Array of File objects * @param bool $stream * @return array */ @@ -697,6 +705,7 @@ private function buildChatCompletionRequestParams( ?array $tools = null, ?string $userAudioPromptBase64 = null, ?string $userAudioPromptFormat = null, + ?array $files = null, bool $stream = false, ): array { $modelRequestParam = $model === Application::DEFAULT_MODEL_ID @@ -740,6 +749,13 @@ private function buildChatCompletionRequestParams( $messages[] = $message; } } + // Attach all files when necessary + if ($files !== null) { + $messages[] = [ + 'role' => 'user', + 'content' => $this->openAiFileService->buildFileContents($files), + ]; + } if ($userAudioPromptBase64 !== null) { // if there is audio, use the new message format (content is a list of objects) $message = [ diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php new file mode 100644 index 00000000..3e284f32 --- /dev/null +++ b/lib/Service/OpenAiFileService.php @@ -0,0 +1,205 @@ + 'mp3', + 'audio/mpeg' => 'mp3', + 'audio/wav' => 'wav', + 'audio/x-wav' => 'wav', + ]; + + private const VALID_TEXT_MIME_TYPES = [ + 'application/javascript', + 'application/typescript', + 'message/rfc822', + 'application/x-sql', + 'application/x-scala', + 'application/x-rust', + 'application/x-powershell', + 'application/x-patch', + 'application/x-php', + 'application/x-httpd-php', + 'application/x-httpd-php-source', + 'application/json', + 'application/x-bash', + 'application/x-protobuf', + 'application/x-terraform', + 'application/x-toml', + 'application/graphql', + 'application/x-graphql', + 'application/x-ndjson', + 'application/json5', + 'application/x-json5', + 'application/toml', + 'application/x-yaml', + 'application/yaml', + 'application/x-awk', + 'application/x-subrip', + 'application/csv', + ]; + + public function __construct( + private IL10N $l10n, + private OpenAiSettingsService $openAiSettingsService, + ) { + } + + /** + * @param array $files Array of File objects + * @return list> + */ + public function buildFileContents(array $files): array { + $fileContents = []; + foreach ($files as $file) { + if (!$file instanceof File || !$file->isReadable()) { + throw new ProcessingException('File is not readable'); + } + // Maximum file size for openai is 50MB. + if ($this->isUsingOpenAi() && $file->getSize() > self::MAX_FILE_SIZE_BYTES) { + throw new UserFacingProcessingException( + 'Filesize of input files too large. Max is 50MB', + 0, + null, + $this->l10n->t('The size of the input file is too large. A maximum of 50MB is allowed.'), + ); + } + + $fileType = $file->getMimeType(); + if (str_starts_with($fileType, 'image/')) { + $fileContents[] = $this->buildImageContent($file); + } elseif (str_starts_with($fileType, 'audio/')) { + $fileContents[] = $this->buildAudioContent($file); + } elseif (str_starts_with($fileType, 'video/')) { + $fileContents[] = $this->buildVideoContent($file); + } elseif ($fileType === 'application/pdf') { + $fileContents[] = $this->buildDocumentContent($file); + } else { + $fileContents[] = $this->buildTextContent($file); + } + } + return $fileContents; + } + + /** + * @return array{type: string, image_url: array{url: string}} + */ + private function buildImageContent(File $file): array { + $fileType = $file->getMimeType(); + if ($this->isUsingOpenAi() && !in_array($fileType, self::VALID_IMAGE_MIME_TYPES, true)) { + throw new UserFacingProcessingException( + 'Invalid input file type for OpenAI ' . $fileType, + 0, + null, + $this->l10n->t('Invalid input file type "%1$s".', [$fileType]), + ); + } + return [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))), + ], + ]; + } + + /** + * @return array{type: string, input_audio: array{data: string, format: string}} + */ + private function buildAudioContent(File $file): array { + $fileType = $file->getMimeType(); + + if (!array_key_exists($fileType, self::SUPPORTED_INPUT_AUDIO_FORMATS)) { + throw new UserFacingProcessingException( + 'Invalid input file type for OpenAI ' . $fileType, + 0, + null, + $this->l10n->t('Invalid input file type "%1$s".', [$fileType]), + ); + } + $format = self::SUPPORTED_INPUT_AUDIO_FORMATS[$fileType]; + return [ + 'type' => 'input_audio', + 'input_audio' => [ + 'data' => base64_encode(stream_get_contents($file->fopen('rb'))), + 'format' => $format, + ], + ]; + } + + /** + * @return array{type: string, video_url: array{url: string}} + */ + private function buildVideoContent(File $file): array { + $fileType = $file->getMimeType(); + return [ + 'type' => 'video_url', + 'video_url' => [ + 'url' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))), + ], + ]; + } + + /** + * @return array{type: string, file: array{filename: string, file_data: string}} + */ + private function buildDocumentContent(File $file): array { + $fileType = $file->getMimeType(); + return [ + 'type' => 'file', + 'file' => [ + 'filename' => $file->getName(), + 'file_data' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))), + ], + ]; + } + + /** + * @return array{type: string, text: string} + */ + private function buildTextContent(File $file): array { + $fileType = $file->getMimeType(); + // Sanity check that this isn't a binary + if (!str_starts_with($fileType, 'text/') && !in_array($fileType, self::VALID_TEXT_MIME_TYPES, true)) { + throw new UserFacingProcessingException( + 'Invalid input file type: ' . $fileType, + 0, + null, + $this->l10n->t('Invalid input file type: "%1$s".', [$fileType]), + ); + } + return [ + 'type' => 'text', + 'text' => 'Filename:' . $file->getName() . "\nContent:\n" . stream_get_contents($file->fopen('rb')), + ]; + } + + private function isUsingOpenAi(): bool { + $serviceUrl = $this->openAiSettingsService->getServiceUrl(); + return $serviceUrl === '' || $serviceUrl === Application::OPENAI_API_BASE_URL; + } +} diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php new file mode 100644 index 00000000..f6e5048f --- /dev/null +++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php @@ -0,0 +1,210 @@ +openAiAPIService->getServiceName(); + } + + public function getTaskTypeId(): string { + return MultimodalChatWithTools::ID; + } + + public function getExpectedRuntime(): int { + return $this->openAiAPIService->getExpTextProcessingTime(); + } + + public function getInputShapeEnumValues(): array { + return []; + } + + public function getInputShapeDefaults(): array { + return []; + } + + public function getOptionalInputShape(): array { + return [ + 'max_tokens' => new ShapeDescriptor( + $this->l->t('Maximum output words'), + $this->l->t('The maximum number of words/tokens that can be generated in the completion.'), + EShapeType::Number + ), + ]; + } + + public function getOptionalInputShapeEnumValues(): array { + return []; + } + + public function getOptionalInputShapeDefaults(): array { + return []; + } + + public function getOutputShapeEnumValues(): array { + return []; + } + + public function getOptionalOutputShape(): array { + return [ + 'reasoning' => new ShapeDescriptor( + $this->l->t('Reasoning content'), + $this->l->t('The model reasoning behind the output'), + EShapeType::Text, + ), + ]; + } + + public function getOptionalOutputShapeEnumValues(): array { + return []; + } + + public function process( + ?string $userId, array $input, callable $reportProgress, SynchronousProviderOptions $options = new SynchronousProviderOptions(), + ): array { + $reportOutput = $options->getReportIntermediateOutput(); + $preferStreaming = false; //$options->getPreferStreaming(); + $startTime = time(); + $adminModel = $this->openAiSettingsService->getAdminDefaultCompletionModelId(); + + if (!isset($input['input']) || !is_string($input['input'])) { + throw new ProcessingException('Invalid input'); + } + $userPrompt = $input['input']; + if ($userPrompt === '') { + $userPrompt = null; + } + + if (!isset($input['system_prompt']) || !is_string($input['system_prompt'])) { + throw new ProcessingException('Invalid system_prompt'); + } + $systemPrompt = $input['system_prompt']; + + if (!isset($input['tool_message']) || !is_string($input['tool_message'])) { + throw new ProcessingException('Invalid tool_message'); + } + $toolMessage = $input['tool_message']; + if ($toolMessage === '') { + $toolMessage = null; + } + + if (!isset($input['tools']) || !is_string($input['tools'])) { + throw new ProcessingException('Invalid tools'); + } + $tools = json_decode($input['tools']); + if (!is_array($tools) || !\array_is_list($tools)) { + throw new ProcessingException('Invalid JSON tools'); + } + + if (!isset($input['history']) || !is_array($input['history']) || !\array_is_list($input['history'])) { + throw new ProcessingException('Invalid history'); + } + $history = $input['history']; + + if (!isset($input['input_attachments']) || !is_array($input['input_attachments'])) { + throw new ProcessingException('Invalid input_attachments'); + } + $inputAttachments = $input['input_attachments']; + + $maxTokens = null; + if (isset($input['max_tokens']) && is_int($input['max_tokens'])) { + $maxTokens = $input['max_tokens']; + } + + try { + if ($preferStreaming) { + $chunks = $this->openAiAPIService->createStreamedChatCompletion( + $userId, $adminModel, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, null, null, $inputAttachments + ); + $time = microtime(true); + $streamedOutput = ''; + $streamedReasoning = ''; + foreach ($chunks as $chunk) { + if (!in_array($chunk['kind'] ?? null, ['content', 'reasoning_content'], true)) { + continue; + } + if ($chunk['kind'] === 'reasoning_content') { + $streamedReasoning .= $chunk['text']; + } elseif ($chunk['kind'] === 'content') { + $streamedOutput .= $chunk['text']; + } + // we don't report more often than every 250ms + if (microtime(true) - $time >= 0.25) { + $running = $reportOutput([ + 'output' => $streamedOutput, + 'reasoning' => $streamedReasoning, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + $time = microtime(true); + } + } + if ($streamedOutput !== '' || $streamedReasoning !== '') { + $running = $reportOutput([ + 'output' => $streamedOutput, + 'reasoning' => $streamedReasoning, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + } + $returnValue = $chunks->getReturn(); + } else { + $returnValue = $this->openAiAPIService->createChatCompletion( + $userId, $adminModel, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, null, null, $inputAttachments + ); + } + } catch (UserFacingProcessingException $e) { + throw $e; + } catch (\Throwable $e) { + xdebug_break(); + throw new ProcessingException('OpenAI/LocalAI request failed: ' . $e->getMessage()); + } + if (count($returnValue['messages']) > 0 || count($returnValue['tool_calls']) > 0) { + $endTime = time(); + $this->openAiAPIService->updateExpTextProcessingTime($endTime - $startTime); + var_dump($returnValue); + return [ + 'output' => array_pop($returnValue['messages']) ?? '', + 'output_attachments' => [], + 'reasoning' => count($returnValue['reasoning_messages']) > 0 ? array_pop($returnValue['reasoning_messages']) : '', + 'tool_calls' => array_pop($returnValue['tool_calls']) ?? '', + ]; + } + + throw new ProcessingException('No result in OpenAI/LocalAI response.'); + } +} diff --git a/psalm.xml b/psalm.xml index 36b2c87e..4e84d9f1 100644 --- a/psalm.xml +++ b/psalm.xml @@ -47,6 +47,7 @@ + diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php index 6e31e2a6..7be7200f 100644 --- a/tests/unit/Providers/OpenAiProviderTest.php +++ b/tests/unit/Providers/OpenAiProviderTest.php @@ -16,6 +16,7 @@ use OCA\OpenAi\Db\QuotaUsageMapper; use OCA\OpenAi\Service\ChunkService; use OCA\OpenAi\Service\OpenAiAPIService; +use OCA\OpenAi\Service\OpenAiFileService; use OCA\OpenAi\Service\OpenAiSettingsService; use OCA\OpenAi\Service\QuotaRuleService; use OCA\OpenAi\Service\StreamingService; @@ -96,6 +97,10 @@ protected function setUp(): void { \OCP\Server::get(QuotaUsageMapper::class), $this->openAiSettingsService, $this->streamingService, + new OpenAiFileService( + $this->createMock(\OCP\IL10N::class), + $this->openAiSettingsService, + ), $this->createMock(\OCP\Notification\IManager::class), \OCP\Server::get(QuotaRuleService::class), $clientService, diff --git a/tests/unit/Quota/QuotaTest.php b/tests/unit/Quota/QuotaTest.php index f4c9cb48..86082b48 100644 --- a/tests/unit/Quota/QuotaTest.php +++ b/tests/unit/Quota/QuotaTest.php @@ -16,6 +16,7 @@ use OCA\OpenAi\Db\EntityType; use OCA\OpenAi\Db\QuotaUsageMapper; use OCA\OpenAi\Service\OpenAiAPIService; +use OCA\OpenAi\Service\OpenAiFileService; use OCA\OpenAi\Service\OpenAiSettingsService; use OCA\OpenAi\Service\QuotaRuleService; use OCA\OpenAi\Service\StreamingService; @@ -87,6 +88,10 @@ protected function setUp(): void { new StreamingService( $this->createMock(IL10N::class), ), + new OpenAiFileService( + $this->createMock(IL10N::class), + $this->openAiSettingsService, + ), $this->notificationManager, \OCP\Server::get(QuotaRuleService::class), \OCP\Server::get(IClientService::class), diff --git a/tests/unit/Service/ServiceOverrideTest.php b/tests/unit/Service/ServiceOverrideTest.php index 9fc7fa72..d73205ce 100644 --- a/tests/unit/Service/ServiceOverrideTest.php +++ b/tests/unit/Service/ServiceOverrideTest.php @@ -16,6 +16,7 @@ use OCA\OpenAi\Db\QuotaUsageMapper; use OCA\OpenAi\Service\ChunkService; use OCA\OpenAi\Service\OpenAiAPIService; +use OCA\OpenAi\Service\OpenAiFileService; use OCA\OpenAi\Service\OpenAiSettingsService; use OCA\OpenAi\Service\QuotaRuleService; use OCA\OpenAi\Service\StreamingService; @@ -90,6 +91,10 @@ protected function setUp(): void { new StreamingService( $this->createMock(\OCP\IL10N::class), ), + new OpenAiFileService( + $this->createMock(\OCP\IL10N::class), + $this->openAiSettingsService, + ), $this->createMock(\OCP\Notification\IManager::class), \OCP\Server::get(QuotaRuleService::class), $clientService, From 60ee11d93b0bbf884a99e90befc16e0d6ec0af4a Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Wed, 8 Jul 2026 15:02:31 -0400 Subject: [PATCH 02/10] Add settings for multimodal providers and limit use the new file option for older providers Signed-off-by: Lukas Schaefer --- appinfo/info.xml | 2 +- lib/AppInfo/Application.php | 2 +- .../Version040600Date20260708151000.php | 44 ++++++++++ lib/Service/OpenAiAPIService.php | 48 ++--------- lib/Service/OpenAiFileService.php | 46 ++++++++++- lib/Service/OpenAiSettingsService.php | 82 +++++++++++++++---- lib/TaskProcessing/AnalyzeImagesProvider.php | 68 +-------------- .../AudioToAudioChatProvider.php | 7 +- .../MultimodalChatWithToolsProvider.php | 6 +- src/components/AdminSettings.vue | 33 ++++++-- 10 files changed, 198 insertions(+), 140 deletions(-) create mode 100644 lib/Migration/Version040600Date20260708151000.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 0fb1275b..8190d944 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -101,7 +101,7 @@ Negative: Learn more about the Nextcloud Ethical AI Rating [in our blog](https://nextcloud.com/blog/nextcloud-ethical-ai-rating/). ]]> - 4.5.1 + 4.6.0-dev agpl Julien Veyssier OpenAi diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 3ae8acee..af357c51 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -148,7 +148,7 @@ public function register(IRegistrationContext $context): void { if (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToTextReformatParagraphs')) { $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider::class); } - if ($isUsingOpenAI || $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled') === '1') { + if ($this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1', lazy: true) === '1') { $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\AnalyzeImagesProvider::class); } } diff --git a/lib/Migration/Version040600Date20260708151000.php b/lib/Migration/Version040600Date20260708151000.php new file mode 100644 index 00000000..87c9c6df --- /dev/null +++ b/lib/Migration/Version040600Date20260708151000.php @@ -0,0 +1,44 @@ +appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', $unset); + if ($newValue !== $unset) { + return; + } + + $oldValue = $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled', $unset); + if (!in_array($oldValue, ['0', '1'], true)) { + return; + } + + $this->appConfig->setValueString(Application::APP_ID, 'multimodal_image_enabled', $oldValue); + } +} diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 64623a04..c2817a89 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -532,8 +532,6 @@ public function createStreamedChatCompletion( ?array $extraParams = null, ?string $toolMessage = null, ?array $tools = null, - ?string $userAudioPromptBase64 = null, - ?string $userAudioPromptFormat = null, ?array $files = null, ): \Generator { if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT)) { @@ -551,8 +549,6 @@ public function createStreamedChatCompletion( $extraParams, $toolMessage, $tools, - $userAudioPromptBase64, - $userAudioPromptFormat, $files, true, ); @@ -594,14 +590,11 @@ public function createChatCompletion( ?array $extraParams = null, ?string $toolMessage = null, ?array $tools = null, - ?string $userAudioPromptBase64 = null, - ?string $userAudioPromptFormat = null, ?array $files = null, ): array { $response = $this->requestChatCompletion( $userId, $model, $userPrompt, $systemPrompt, $history, - $n, $maxTokens, $extraParams, $toolMessage, $tools, - $userAudioPromptBase64, $userAudioPromptFormat, $files, + $n, $maxTokens, $extraParams, $toolMessage, $tools, $files, false, ); @@ -629,8 +622,6 @@ public function createChatCompletion( * @param array|null $extraParams * @param string|null $toolMessage JSON string with role, content, tool_call_id * @param array|null $tools - * @param string|null $userAudioPromptBase64 - * @param string|null $userAudioPromptFormat * @param array|null $files Array of File objects * @return array{messages?: array, tool_calls?: array, audio_messages?: list>, usage?: array} * @throws Exception @@ -646,8 +637,6 @@ public function requestChatCompletion( ?array $extraParams = null, ?string $toolMessage = null, ?array $tools = null, - ?string $userAudioPromptBase64 = null, - ?string $userAudioPromptFormat = null, ?array $files = null, bool $stream = false, ): array { @@ -666,8 +655,6 @@ public function requestChatCompletion( $extraParams, $toolMessage, $tools, - $userAudioPromptBase64, - $userAudioPromptFormat, $files, $stream, ); @@ -686,8 +673,6 @@ public function requestChatCompletion( * @param array|null $extraParams * @param string|null $toolMessage * @param array|null $tools - * @param string|null $userAudioPromptBase64 - * @param string|null $userAudioPromptFormat * @param array|null $files Array of File objects * @param bool $stream * @return array @@ -703,8 +688,6 @@ private function buildChatCompletionRequestParams( ?array $extraParams = null, ?string $toolMessage = null, ?array $tools = null, - ?string $userAudioPromptBase64 = null, - ?string $userAudioPromptFormat = null, ?array $files = null, bool $stream = false, ): array { @@ -751,34 +734,18 @@ private function buildChatCompletionRequestParams( } // Attach all files when necessary if ($files !== null) { - $messages[] = [ - 'role' => 'user', - 'content' => $this->openAiFileService->buildFileContents($files), - ]; - } - if ($userAudioPromptBase64 !== null) { - // if there is audio, use the new message format (content is a list of objects) - $message = [ - 'role' => 'user', - 'content' => [ - [ - 'type' => 'input_audio', - 'input_audio' => [ - 'data' => $userAudioPromptBase64, - 'format' => $userAudioPromptFormat ?? 'wav', - ], - ], - ], - ]; + $content = $this->openAiFileService->buildFileContents($files); if ($userPrompt !== null) { - $message['content'][] = [ + $content[] = [ 'type' => 'text', 'text' => $userPrompt, ]; } - $messages[] = $message; + $messages[] = [ + 'role' => 'user', + 'content' => $content, + ]; } elseif ($userPrompt !== null) { - // if there is only text, use the old message format (content is a string) $messages[] = [ 'role' => 'user', 'content' => $userPrompt, @@ -1582,7 +1549,6 @@ public function autoDetectFeatures(): array { $config['stt_provider_enabled'] = $this->isSTTAvailable(); $config['tts_provider_enabled'] = $this->isTTSAvailable(); $this->openAiSettingsService->setAdminConfig($config); - $config['analyze_image_provider_enabled'] = $this->openAiSettingsService->getAnalyzeImageProviderEnabled(); return $config; } } diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index 3e284f32..218edc22 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -61,7 +61,7 @@ class OpenAiFileService { 'application/yaml', 'application/x-awk', 'application/x-subrip', - 'application/csv', + 'application/csv' ]; public function __construct( @@ -76,6 +76,14 @@ public function __construct( */ public function buildFileContents(array $files): array { $fileContents = []; + if (count($files) > 500) { + throw new UserFacingProcessingException( + 'Too many files. Max is 500', + 0, + null, + $this->l10n->t('The number of input files is too large. A maximum of 500 files is allowed.'), + ); + } foreach ($files as $file) { if (!$file instanceof File || !$file->isReadable()) { throw new ProcessingException('File is not readable'); @@ -93,8 +101,10 @@ public function buildFileContents(array $files): array { $fileType = $file->getMimeType(); if (str_starts_with($fileType, 'image/')) { $fileContents[] = $this->buildImageContent($file); + // OpenAI only supports this for very specific models and support is not that common } elseif (str_starts_with($fileType, 'audio/')) { $fileContents[] = $this->buildAudioContent($file); + // OpenAI does not currently support video attachments } elseif (str_starts_with($fileType, 'video/')) { $fileContents[] = $this->buildVideoContent($file); } elseif ($fileType === 'application/pdf') { @@ -110,6 +120,14 @@ public function buildFileContents(array $files): array { * @return array{type: string, image_url: array{url: string}} */ private function buildImageContent(File $file): array { + if (!$this->openAiSettingsService->getMultimodalImageEnabled()) { + throw new UserFacingProcessingException( + 'Image attachments are disabled', + 0, + null, + $this->l10n->t('Image attachments are unsupported.'), + ); + } $fileType = $file->getMimeType(); if ($this->isUsingOpenAi() && !in_array($fileType, self::VALID_IMAGE_MIME_TYPES, true)) { throw new UserFacingProcessingException( @@ -131,8 +149,16 @@ private function buildImageContent(File $file): array { * @return array{type: string, input_audio: array{data: string, format: string}} */ private function buildAudioContent(File $file): array { + if (!$this->openAiSettingsService->getMultimodalAudioEnabled()) { + throw new UserFacingProcessingException( + 'Audio attachments are disabled', + 0, + null, + $this->l10n->t('Audio attachments are unsupported.'), + ); + } $fileType = $file->getMimeType(); - + if (!array_key_exists($fileType, self::SUPPORTED_INPUT_AUDIO_FORMATS)) { throw new UserFacingProcessingException( 'Invalid input file type for OpenAI ' . $fileType, @@ -155,6 +181,14 @@ private function buildAudioContent(File $file): array { * @return array{type: string, video_url: array{url: string}} */ private function buildVideoContent(File $file): array { + if (!$this->openAiSettingsService->getMultimodalVideoEnabled()) { + throw new UserFacingProcessingException( + 'Video attachments are disabled', + 0, + null, + $this->l10n->t('Video attachments are unsupported.'), + ); + } $fileType = $file->getMimeType(); return [ 'type' => 'video_url', @@ -168,6 +202,14 @@ private function buildVideoContent(File $file): array { * @return array{type: string, file: array{filename: string, file_data: string}} */ private function buildDocumentContent(File $file): array { + if (!$this->openAiSettingsService->getMultimodalDocumentEnabled()) { + throw new UserFacingProcessingException( + 'Document attachments are disabled', + 0, + null, + $this->l10n->t('Document attachments are unsupported.'), + ); + } $fileType = $file->getMimeType(); return [ 'type' => 'file', diff --git a/lib/Service/OpenAiSettingsService.php b/lib/Service/OpenAiSettingsService.php index 30661239..f63705c1 100644 --- a/lib/Service/OpenAiSettingsService.php +++ b/lib/Service/OpenAiSettingsService.php @@ -43,7 +43,10 @@ class OpenAiSettingsService { 't2i_provider_enabled' => 'boolean', 'stt_provider_enabled' => 'boolean', 'tts_provider_enabled' => 'boolean', - 'analyze_image_provider_enabled' => 'boolean', + 'multimodal_image_enabled' => 'boolean', + 'multimodal_audio_enabled' => 'boolean', + 'multimodal_video_enabled' => 'boolean', + 'multimodal_document_enabled' => 'boolean', 'chat_endpoint_enabled' => 'boolean', 'basic_user' => 'string', 'basic_password' => 'string', @@ -587,7 +590,10 @@ public function getAdminConfig(): array { 't2i_provider_enabled' => $this->getT2iProviderEnabled(), 'stt_provider_enabled' => $this->getSttProviderEnabled(), 'tts_provider_enabled' => $this->getTtsProviderEnabled(), - 'analyze_image_provider_enabled' => $this->getAnalyzeImageProviderEnabled(), + 'multimodal_image_enabled' => $this->getMultimodalImageEnabled(), + 'multimodal_audio_enabled' => $this->getMultimodalAudioEnabled(), + 'multimodal_video_enabled' => $this->getMultimodalVideoEnabled(), + 'multimodal_document_enabled' => $this->getMultimodalDocumentEnabled(), 'chat_endpoint_enabled' => $this->getChatEndpointEnabled(), 'basic_user' => $this->getAdminBasicUser(), 'basic_password' => $this->getAdminBasicPassword(), @@ -699,14 +705,29 @@ public function getTtsProviderEnabled(): bool { /** * @return bool */ - public function getAnalyzeImageProviderEnabled(): bool { - $config = $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled'); - if ($config === '') { - $serviceUrl = $this->getServiceUrl(); - $isUsingOpenAI = $serviceUrl === '' || $serviceUrl === Application::OPENAI_API_BASE_URL; - return $isUsingOpenAI; - } - return $config === '1'; + public function getMultimodalImageEnabled(): bool { + return $this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1') === '1'; + } + + /** + * @return bool + */ + public function getMultimodalAudioEnabled(): bool { + return $this->appConfig->getValueString(Application::APP_ID, 'multimodal_audio_enabled', '1') === '1'; + } + + /** + * @return bool + */ + public function getMultimodalVideoEnabled(): bool { + return $this->appConfig->getValueString(Application::APP_ID, 'multimodal_video_enabled', '0') === '1'; + } + + /** + * @return bool + */ + public function getMultimodalDocumentEnabled(): bool { + return $this->appConfig->getValueString(Application::APP_ID, 'multimodal_document_enabled', '1') === '1'; } //////////////////////////////////////////// @@ -1268,12 +1289,21 @@ public function setAdminConfig(array $adminConfig): void { if (isset($adminConfig['tts_provider_enabled'])) { $this->setTtsProviderEnabled($adminConfig['tts_provider_enabled']); } - if (isset($adminConfig['analyze_image_provider_enabled'])) { - $this->setAnalyzeImageProviderEnabled($adminConfig['analyze_image_provider_enabled']); - } if (isset($adminConfig['default_tts_voice'])) { $this->setAdminDefaultTtsVoice($adminConfig['default_tts_voice']); } + if (isset($adminConfig['multimodal_image_enabled'])) { + $this->setMultimodalImageEnabled($adminConfig['multimodal_image_enabled']); + } + if (isset($adminConfig['multimodal_audio_enabled'])) { + $this->setMultimodalAudioEnabled($adminConfig['multimodal_audio_enabled']); + } + if (isset($adminConfig['multimodal_video_enabled'])) { + $this->setMultimodalVideoEnabled($adminConfig['multimodal_video_enabled']); + } + if (isset($adminConfig['multimodal_document_enabled'])) { + $this->setMultimodalDocumentEnabled($adminConfig['multimodal_document_enabled']); + } if (isset($adminConfig['chat_endpoint_enabled'])) { $this->setChatEndpointEnabled($adminConfig['chat_endpoint_enabled']); } @@ -1444,10 +1474,30 @@ public function setTtsProviderEnabled(bool $enabled): void { /** * @param bool $enabled - * @return void */ - public function setAnalyzeImageProviderEnabled(bool $enabled): void { - $this->appConfig->setValueString(Application::APP_ID, 'analyze_image_provider_enabled', $enabled ? '1' : '0'); + public function setMultimodalImageEnabled(bool $enabled): void { + $this->appConfig->setValueString(Application::APP_ID, 'multimodal_image_enabled', $enabled ? '1' : '0'); + } + + /** + * @param bool $enabled + */ + public function setMultimodalAudioEnabled(bool $enabled): void { + $this->appConfig->setValueString(Application::APP_ID, 'multimodal_audio_enabled', $enabled ? '1' : '0'); + } + + /** + * @param bool $enabled + */ + public function setMultimodalVideoEnabled(bool $enabled): void { + $this->appConfig->setValueString(Application::APP_ID, 'multimodal_video_enabled', $enabled ? '1' : '0'); + } + + /** + * @param bool $enabled + */ + public function setMultimodalDocumentEnabled(bool $enabled): void { + $this->appConfig->setValueString(Application::APP_ID, 'multimodal_document_enabled', $enabled ? '1' : '0'); } /** diff --git a/lib/TaskProcessing/AnalyzeImagesProvider.php b/lib/TaskProcessing/AnalyzeImagesProvider.php index 65934143..df822c12 100644 --- a/lib/TaskProcessing/AnalyzeImagesProvider.php +++ b/lib/TaskProcessing/AnalyzeImagesProvider.php @@ -12,7 +12,6 @@ use OCA\OpenAi\AppInfo\Application; use OCA\OpenAi\Service\OpenAiAPIService; use OCA\OpenAi\Service\OpenAiSettingsService; -use OCP\Files\File; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Exception\ProcessingException; @@ -120,68 +119,7 @@ public function process( if (!isset($input['images']) || !is_array($input['images'])) { throw new ProcessingException('Invalid file list'); } - // Maximum file count for openai is 500. Seems reasonable enough to enforce for all apis though (https://platform.openai.com/docs/guides/images-vision?api-mode=responses&format=url#image-input-requirements) - if (count($input['images']) > 500) { - throw new UserFacingProcessingException( - 'Too many files given. Max is 500', - 0, - null, - $this->l->t('Too many files given. A maximum of 500 files is allowed.'), - ); - } - $fileSize = 0; - foreach ($input['images'] as $image) { - if (!$image instanceof File || !$image->isReadable()) { - throw new ProcessingException('Invalid input file'); - } - $fileSize += intval($image->getSize()); - // Maximum file size for openai is 50MB. Seems reasonable enough to enforce for all apis though. (https://platform.openai.com/docs/guides/images-vision?api-mode=responses&format=url#image-input-requirements) - if ($fileSize > 50 * 1000 * 1000) { - throw new UserFacingProcessingException( - 'Filesize of input files too large. Max is 50MB', - 0, - null, - $this->l->t('The total size of the input files is too large. A maximum of 50MB is allowed.'), - ); - } - $inputFile = base64_encode(stream_get_contents($image->fopen('rb'))); - $fileType = $image->getMimeType(); - if (!str_starts_with($fileType, 'image/')) { - throw new UserFacingProcessingException( - 'Invalid input file type ' . $fileType, - 0, - null, - $this->l->t('Invalid input file type "%1$s". Only image files are supported.', [$fileType]), - ); - } - if ($this->openAiAPIService->isUsingOpenAi()) { - $validFileTypes = [ - 'image/jpeg', - 'image/png', - 'image/gif', - 'image/webp', - ]; - if (!in_array($fileType, $validFileTypes)) { - throw new UserFacingProcessingException( - 'Invalid input file type for OpenAI ' . $fileType, - 0, - null, - $this->l->t('Invalid input file type "%1$s". Only JPEG, PNG, GIF and WebP images are supported.', [$fileType]), - ); - } - } - $history[] = json_encode([ - 'role' => 'user', - 'content' => [ - [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => 'data:' . $fileType . ';base64,' . $inputFile, - ], - ], - ], - ]); - } + $images = $input['images']; if (!isset($input['input']) || !is_string($input['input'])) { throw new ProcessingException('Invalid prompt'); @@ -202,7 +140,7 @@ public function process( try { $systemPrompt = 'Take the user\'s question and answer it based on the provided images. Ensure that the answer matches the language of the user\'s question.'; if ($preferStreaming) { - $chunks = $this->openAiAPIService->createStreamedChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens); + $chunks = $this->openAiAPIService->createStreamedChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens, null, null, null, $images); $time = microtime(true); $streamedOutput = ''; $streamedReasoning = ''; @@ -240,7 +178,7 @@ public function process( $completion = $returnValue['messages']; $reasoning = $returnValue['reasoning_messages']; } else { - $returnValue = $this->openAiAPIService->createChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens); + $returnValue = $this->openAiAPIService->createChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens, null, null, null, $images); $completion = $returnValue['messages']; $reasoning = $returnValue['reasoning_messages']; } diff --git a/lib/TaskProcessing/AudioToAudioChatProvider.php b/lib/TaskProcessing/AudioToAudioChatProvider.php index dd34027b..c7f36194 100644 --- a/lib/TaskProcessing/AudioToAudioChatProvider.php +++ b/lib/TaskProcessing/AudioToAudioChatProvider.php @@ -125,7 +125,7 @@ public function getOptionalInputShapeDefaults(): array { $isUsingOpenAi = $this->openAiAPIService->isUsingOpenAi(); $adminVoice = $this->appConfig->getValueString(Application::APP_ID, 'default_speech_voice', lazy: true) ?: Application::DEFAULT_SPEECH_VOICE; $adminLlmModel = $isUsingOpenAi - ? 'gpt-4o-audio-preview' + ? 'gpt-audio' : $this->openAiSettingsService->getAdminDefaultCompletionModelId(); $defaults = [ 'voice' => $adminVoice, @@ -234,9 +234,6 @@ private function oneStep( string $sttModel, string $llmModel, string $ttsModel, float $speed, string $serviceName, ): array { $result = []; - $audioInputMimetype = mime_content_type($inputFile->fopen('rb')); - $audioInputFormat = self::SUPPORTED_INPUT_AUDIO_FORMATS[$audioInputMimetype] ?? 'wav'; - $b64Audio = base64_encode($inputFile->getContent()); $extraParams = [ 'modalities' => ['text', 'audio'], 'audio' => ['voice' => $outputVoice, 'format' => 'mp3'], @@ -244,7 +241,7 @@ private function oneStep( $systemPrompt .= ' Producing text responses will break the user interface. Important: You have multimodal voice capability, and you use voice exclusively to respond.'; $completion = $this->openAiAPIService->createChatCompletion( $userId, $llmModel, null, $systemPrompt, $history, 1, 1000, - $extraParams, null, null, $b64Audio, $audioInputFormat + $extraParams, null, null, [$inputFile] ); $message = array_pop($completion['audio_messages']); // TODO find a way to force the model to answer with audio when there is only text in the history diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php index f6e5048f..84a8a662 100644 --- a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php +++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php @@ -146,7 +146,7 @@ public function process( try { if ($preferStreaming) { $chunks = $this->openAiAPIService->createStreamedChatCompletion( - $userId, $adminModel, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, null, null, $inputAttachments + $userId, $adminModel, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments ); $time = microtime(true); $streamedOutput = ''; @@ -184,19 +184,17 @@ public function process( $returnValue = $chunks->getReturn(); } else { $returnValue = $this->openAiAPIService->createChatCompletion( - $userId, $adminModel, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, null, null, $inputAttachments + $userId, $adminModel, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments ); } } catch (UserFacingProcessingException $e) { throw $e; } catch (\Throwable $e) { - xdebug_break(); throw new ProcessingException('OpenAI/LocalAI request failed: ' . $e->getMessage()); } if (count($returnValue['messages']) > 0 || count($returnValue['tool_calls']) > 0) { $endTime = time(); $this->openAiAPIService->updateExpTextProcessingTime($endTime - $startTime); - var_dump($returnValue); return [ 'output' => array_pop($returnValue['messages']) ?? '', 'output_attachments' => [], diff --git a/src/components/AdminSettings.vue b/src/components/AdminSettings.vue index 01551409..73f99bba 100644 --- a/src/components/AdminSettings.vue +++ b/src/components/AdminSettings.vue @@ -326,6 +326,34 @@ {{ t('integration_openai', 'Use "{newParam}" parameter instead of the deprecated "{deprecatedParam}"', { newParam: 'max_completion_tokens', deprecatedParam: 'max_tokens' }) }} +

+ {{ t('integration_openai', 'Multimodal LLM Support') }} +

+ + {{ t('integration_openai', 'Multimodal LLM Support allows you to enable or disable the use of images, audio, video and document attachments in the LLM.') }} + + + + {{ t('integration_openai', 'Image attachments') }} + + + {{ t('integration_openai', 'Audio attachments') }} + + + {{ t('integration_openai', 'Video attachments') }} + + + {{ t('integration_openai', 'Document attachments') }} + +

@@ -633,11 +661,6 @@ @update:model-value="onCheckboxChanged($event, 'tts_provider_enabled', false)"> {{ t('integration_openai', 'Text-to-speech provider') }} - - {{ t('integration_openai', 'Analyze image provider') }} -

From 9c0c9243d6cc2afaccd2b523461ae414ebd9d6c4 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Fri, 10 Jul 2026 14:59:42 -0400 Subject: [PATCH 03/10] support multimodal chat outputs Signed-off-by: Lukas Schaefer --- lib/Service/OpenAiAPIService.php | 7 +++++ lib/Service/StreamingService.php | 3 ++ .../MultimodalChatWithToolsProvider.php | 29 ++++++++++++++++--- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index c2817a89..3e8244b1 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -1425,8 +1425,10 @@ private function normalizeChatCompletionResponse(array $response): array { 'reasoning_messages' => [], 'tool_calls' => [], 'audio_messages' => [], + 'images' => [], ]; + foreach ($response['choices'] as $choice) { if (!is_array($choice)) { continue; @@ -1466,6 +1468,11 @@ private function normalizeChatCompletionResponse(array $response): array { if (isset($choice['message']['audio'], $choice['message']['audio']['data']) && is_string($choice['message']['audio']['data'])) { $completions['audio_messages'][] = $choice['message']; } + if (isset($choice['message']['images']) && is_array($choice['message']['images'])) { + foreach ($choice['message']['images'] as $image) { + $completions['images'][] = $image; + } + } } return $completions; diff --git a/lib/Service/StreamingService.php b/lib/Service/StreamingService.php index 4bda5392..06080a64 100644 --- a/lib/Service/StreamingService.php +++ b/lib/Service/StreamingService.php @@ -176,6 +176,9 @@ private function parseSseEvent(string $event, bool &$done, ?array &$usage, array $choices[$index]['message']['audio'] = $choice['message']['audio']; } + if (isset($choice['message']['images']) && is_array($choice['message']['images'])) { + $choices[$index]['message']['images'] = $choice['message']['images']; + } // TODO decide if we stream the tool_calls if (isset($choice['delta']['tool_calls']) && is_array($choice['delta']['tool_calls'])) { diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php index 84a8a662..904e7d8c 100644 --- a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php +++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php @@ -12,6 +12,8 @@ use OCA\OpenAi\AppInfo\Application; use OCA\OpenAi\Service\OpenAiAPIService; use OCA\OpenAi\Service\OpenAiSettingsService; +use OCA\OpenAi\Service\WatermarkingService; +use OCP\Http\Client\IClientService; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Exception\ProcessingException; @@ -21,6 +23,7 @@ use OCP\TaskProcessing\ShapeDescriptor; use OCP\TaskProcessing\SynchronousProviderOptions; use OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools; +use Psr\Log\LoggerInterface; class MultimodalChatWithToolsProvider implements IProvider, ISynchronousOptionsAwareProvider { @@ -28,6 +31,9 @@ public function __construct( private OpenAiAPIService $openAiAPIService, private OpenAiSettingsService $openAiSettingsService, private IL10N $l, + private LoggerInterface $logger, + private IClientService $clientService, + private WatermarkingService $watermarkingService, ) { } @@ -95,7 +101,7 @@ public function process( ?string $userId, array $input, callable $reportProgress, SynchronousProviderOptions $options = new SynchronousProviderOptions(), ): array { $reportOutput = $options->getReportIntermediateOutput(); - $preferStreaming = false; //$options->getPreferStreaming(); + $preferStreaming = $options->getPreferStreaming(); $startTime = time(); $adminModel = $this->openAiSettingsService->getAdminDefaultCompletionModelId(); @@ -142,7 +148,7 @@ public function process( if (isset($input['max_tokens']) && is_int($input['max_tokens'])) { $maxTokens = $input['max_tokens']; } - + xdebug_break(); try { if ($preferStreaming) { $chunks = $this->openAiAPIService->createStreamedChatCompletion( @@ -192,12 +198,27 @@ public function process( } catch (\Throwable $e) { throw new ProcessingException('OpenAI/LocalAI request failed: ' . $e->getMessage()); } - if (count($returnValue['messages']) > 0 || count($returnValue['tool_calls']) > 0) { + if (count($returnValue['messages']) > 0 || count($returnValue['tool_calls']) > 0 || count($returnValue['images']) > 0 || count($returnValue['audio_messages']) > 0) { $endTime = time(); $this->openAiAPIService->updateExpTextProcessingTime($endTime - $startTime); + $attachments = []; + + // Handle image output + foreach ($returnValue['images'] as $image) { + if ($image['type'] === 'image_url') { + $url = $image['image_url']['url']; + $base64Str = explode(',', $url)[1] ?? ''; + $image = base64_decode($base64Str); + $image = $this->watermarkingService->markImage($image); + $attachments[] = $image; + } else { + $this->logger->warning('Encountered an unknown image type in multimodal chat: ' . $image['type']); + } + } + return [ 'output' => array_pop($returnValue['messages']) ?? '', - 'output_attachments' => [], + 'output_attachments' => $attachments, 'reasoning' => count($returnValue['reasoning_messages']) > 0 ? array_pop($returnValue['reasoning_messages']) : '', 'tool_calls' => array_pop($returnValue['tool_calls']) ?? '', ]; From 0b591c1845c18a9531e1850cd3e6ba701d240335 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 13 Jul 2026 16:34:55 -0400 Subject: [PATCH 04/10] Handle history for attachments correctly Signed-off-by: Lukas Schaefer --- lib/Service/OpenAiAPIService.php | 19 ++++- lib/Service/OpenAiFileService.php | 82 +++++++++++-------- lib/Service/StreamingService.php | 8 +- .../MultimodalChatWithToolsProvider.php | 1 - 4 files changed, 70 insertions(+), 40 deletions(-) diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 3e8244b1..58b4e211 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -729,12 +729,27 @@ private function buildChatCompletionRequestParams( return $formattedToolCall; }, $message['tool_calls']); } + // Handle file attachments in the history + if (isset($message['content']) && is_array($message['content'])) { + $content = []; + foreach ($message['content'] as $item) { + if ($item['type'] === 'file') { + $content[] = $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId); + } else { + $content[] = $item; + } + } + $message['content'] = $content; + } $messages[] = $message; } } // Attach all files when necessary - if ($files !== null) { - $content = $this->openAiFileService->buildFileContents($files); + if ($files !== null && count($files) > 0) { + if (count($files) > 500) { + throw new UserFacingProcessingException($this->l10n->t('Too many files. Max is 500'), Http::STATUS_BAD_REQUEST); + } + $content = array_map([$this->openAiFileService, 'buildFileContentFromFile'], $files); if ($userPrompt !== null) { $content[] = [ 'type' => 'text', diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index 218edc22..beb72039 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -11,6 +11,7 @@ use OCA\OpenAi\AppInfo\Application; use OCP\Files\File; +use OCP\Files\IRootFolder; use OCP\IL10N; use OCP\TaskProcessing\Exception\ProcessingException; use OCP\TaskProcessing\Exception\UserFacingProcessingException; @@ -67,55 +68,64 @@ class OpenAiFileService { public function __construct( private IL10N $l10n, private OpenAiSettingsService $openAiSettingsService, + private IRootFolder $rootFolder, ) { } /** - * @param array $files Array of File objects - * @return list> + * Builds file content from a file ID within a given user folder. + * + * @param int $fileId The ID of the file to build content from. + * @param string $userId The user ID. + * @return array Content array suitable for OpenAI API or other handlers. + * @throws ProcessingException + * @throws UserFacingProcessingException */ - public function buildFileContents(array $files): array { - $fileContents = []; - if (count($files) > 500) { + public function buildFileContentFromId(int $fileId, string $userId): array { + $userFolder = $this->rootFolder->getUserFolder($userId); + $file = $userFolder->getFirstNodeById($fileId); + return $this->buildFileContentFromFile($file); + } + + /** + * Builds file content from a File object. + * + * @param ?File $file The file to build content from. + * @return array Content array suitable for OpenAI API or other handlers. + * @throws ProcessingException + * @throws UserFacingProcessingException + */ + public function buildFileContentFromFile(?File $file): array { + if (!$file instanceof File || !$file->isReadable()) { + throw new ProcessingException('File is not readable'); + } + // Maximum file size for openai is 50MB. + if ($this->isUsingOpenAi() && $file->getSize() > self::MAX_FILE_SIZE_BYTES) { throw new UserFacingProcessingException( - 'Too many files. Max is 500', + 'Filesize of input files too large. Max is 50MB', 0, null, - $this->l10n->t('The number of input files is too large. A maximum of 500 files is allowed.'), + $this->l10n->t('The size of the input file is too large. A maximum of 50MB is allowed.'), ); } - foreach ($files as $file) { - if (!$file instanceof File || !$file->isReadable()) { - throw new ProcessingException('File is not readable'); - } - // Maximum file size for openai is 50MB. - if ($this->isUsingOpenAi() && $file->getSize() > self::MAX_FILE_SIZE_BYTES) { - throw new UserFacingProcessingException( - 'Filesize of input files too large. Max is 50MB', - 0, - null, - $this->l10n->t('The size of the input file is too large. A maximum of 50MB is allowed.'), - ); - } - - $fileType = $file->getMimeType(); - if (str_starts_with($fileType, 'image/')) { - $fileContents[] = $this->buildImageContent($file); - // OpenAI only supports this for very specific models and support is not that common - } elseif (str_starts_with($fileType, 'audio/')) { - $fileContents[] = $this->buildAudioContent($file); - // OpenAI does not currently support video attachments - } elseif (str_starts_with($fileType, 'video/')) { - $fileContents[] = $this->buildVideoContent($file); - } elseif ($fileType === 'application/pdf') { - $fileContents[] = $this->buildDocumentContent($file); - } else { - $fileContents[] = $this->buildTextContent($file); - } + + $fileType = $file->getMimeType(); + if (str_starts_with($fileType, 'image/')) { + return $this->buildImageContent($file); + // OpenAI only supports this for very specific models and support is not that common + } elseif (str_starts_with($fileType, 'audio/')) { + return $this->buildAudioContent($file); + // OpenAI does not currently support video attachments + } elseif (str_starts_with($fileType, 'video/')) { + return $this->buildVideoContent($file); + } elseif ($fileType === 'application/pdf') { + return $this->buildDocumentContent($file); + } else { + return $this->buildTextContent($file); } - return $fileContents; } + /** * @return array{type: string, image_url: array{url: string}} */ diff --git a/lib/Service/StreamingService.php b/lib/Service/StreamingService.php index 06080a64..1d528131 100644 --- a/lib/Service/StreamingService.php +++ b/lib/Service/StreamingService.php @@ -176,7 +176,13 @@ private function parseSseEvent(string $event, bool &$done, ?array &$usage, array $choices[$index]['message']['audio'] = $choice['message']['audio']; } - if (isset($choice['message']['images']) && is_array($choice['message']['images'])) { + if (isset($choice['delta']['images']) && is_array($choice['delta']['images'])) { + $existingImages = $choices[$index]['message']['images'] ?? []; + if (!is_array($existingImages)) { + $existingImages = []; + } + $choices[$index]['message']['images'] = array_merge($existingImages, $choice['delta']['images']); + } elseif (isset($choice['message']['images']) && is_array($choice['message']['images'])) { $choices[$index]['message']['images'] = $choice['message']['images']; } // TODO decide if we stream the tool_calls diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php index 904e7d8c..3c6385df 100644 --- a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php +++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php @@ -148,7 +148,6 @@ public function process( if (isset($input['max_tokens']) && is_int($input['max_tokens'])) { $maxTokens = $input['max_tokens']; } - xdebug_break(); try { if ($preferStreaming) { $chunks = $this->openAiAPIService->createStreamedChatCompletion( From a173a2cf67d845772d828d9b53ad249557ba3c7b Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Tue, 14 Jul 2026 12:41:59 -0400 Subject: [PATCH 05/10] Support pdfs by converting them to images Signed-off-by: Lukas Schaefer --- lib/AppInfo/Application.php | 2 +- lib/Service/OpenAiAPIService.php | 7 +- lib/Service/OpenAiFileService.php | 108 +++++++++++++--- .../MultimodalChatWithToolsProvider.php | 20 ++- psalm.xml | 1 + tests/stubs/oc_hooks_emitter.php | 29 +++++ tests/unit/Providers/OpenAiProviderTest.php | 115 ++++++++++++++++++ tests/unit/Quota/QuotaTest.php | 2 + tests/unit/Service/ServiceOverrideTest.php | 2 + 9 files changed, 264 insertions(+), 22 deletions(-) create mode 100644 tests/stubs/oc_hooks_emitter.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index af357c51..7854fa4b 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -148,7 +148,7 @@ public function register(IRegistrationContext $context): void { if (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToTextReformatParagraphs')) { $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider::class); } - if ($this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1', lazy: true) === '1') { + if ($this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1') === '1') { $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\AnalyzeImagesProvider::class); } } diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 58b4e211..bc4f6591 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -734,7 +734,7 @@ private function buildChatCompletionRequestParams( $content = []; foreach ($message['content'] as $item) { if ($item['type'] === 'file') { - $content[] = $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId); + $content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId)); } else { $content[] = $item; } @@ -749,7 +749,10 @@ private function buildChatCompletionRequestParams( if (count($files) > 500) { throw new UserFacingProcessingException($this->l10n->t('Too many files. Max is 500'), Http::STATUS_BAD_REQUEST); } - $content = array_map([$this->openAiFileService, 'buildFileContentFromFile'], $files); + $content = []; + foreach ($files as $file) { + $content = array_merge($content, $this->openAiFileService->buildFileContentFromFile($file)); + } if ($userPrompt !== null) { $content[] = [ 'type' => 'text', diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index beb72039..6b46fd1b 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -9,12 +9,15 @@ namespace OCA\OpenAi\Service; +use Imagick; use OCA\OpenAi\AppInfo\Application; use OCP\Files\File; use OCP\Files\IRootFolder; use OCP\IL10N; use OCP\TaskProcessing\Exception\ProcessingException; use OCP\TaskProcessing\Exception\UserFacingProcessingException; +use Psr\Log\LoggerInterface; +use RuntimeException; class OpenAiFileService { private const MAX_FILE_SIZE_BYTES = 50_000_000; @@ -69,6 +72,7 @@ public function __construct( private IL10N $l10n, private OpenAiSettingsService $openAiSettingsService, private IRootFolder $rootFolder, + private LoggerInterface $logger, ) { } @@ -77,7 +81,7 @@ public function __construct( * * @param int $fileId The ID of the file to build content from. * @param string $userId The user ID. - * @return array Content array suitable for OpenAI API or other handlers. + * @return list> Content parts suitable for OpenAI chat message content. * @throws ProcessingException * @throws UserFacingProcessingException */ @@ -91,7 +95,7 @@ public function buildFileContentFromId(int $fileId, string $userId): array { * Builds file content from a File object. * * @param ?File $file The file to build content from. - * @return array Content array suitable for OpenAI API or other handlers. + * @return list> Content parts suitable for OpenAI chat message content. * @throws ProcessingException * @throws UserFacingProcessingException */ @@ -127,7 +131,7 @@ public function buildFileContentFromFile(?File $file): array { /** - * @return array{type: string, image_url: array{url: string}} + * @return list */ private function buildImageContent(File $file): array { if (!$this->openAiSettingsService->getMultimodalImageEnabled()) { @@ -147,16 +151,16 @@ private function buildImageContent(File $file): array { $this->l10n->t('Invalid input file type "%1$s".', [$fileType]), ); } - return [ + return [[ 'type' => 'image_url', 'image_url' => [ 'url' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))), ], - ]; + ]]; } /** - * @return array{type: string, input_audio: array{data: string, format: string}} + * @return list */ private function buildAudioContent(File $file): array { if (!$this->openAiSettingsService->getMultimodalAudioEnabled()) { @@ -178,17 +182,17 @@ private function buildAudioContent(File $file): array { ); } $format = self::SUPPORTED_INPUT_AUDIO_FORMATS[$fileType]; - return [ + return [[ 'type' => 'input_audio', 'input_audio' => [ 'data' => base64_encode(stream_get_contents($file->fopen('rb'))), 'format' => $format, ], - ]; + ]]; } /** - * @return array{type: string, video_url: array{url: string}} + * @return list */ private function buildVideoContent(File $file): array { if (!$this->openAiSettingsService->getMultimodalVideoEnabled()) { @@ -200,19 +204,23 @@ private function buildVideoContent(File $file): array { ); } $fileType = $file->getMimeType(); - return [ + return [[ 'type' => 'video_url', 'video_url' => [ 'url' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))), ], - ]; + ]]; } /** - * @return array{type: string, file: array{filename: string, file_data: string}} + * @return list */ private function buildDocumentContent(File $file): array { if (!$this->openAiSettingsService->getMultimodalDocumentEnabled()) { + // Fallback to image if documents are not supported + if ($this->openAiSettingsService->getMultimodalImageEnabled()) { + return $this->buildImageFromFile($file); + } throw new UserFacingProcessingException( 'Document attachments are disabled', 0, @@ -221,17 +229,17 @@ private function buildDocumentContent(File $file): array { ); } $fileType = $file->getMimeType(); - return [ + return [[ 'type' => 'file', 'file' => [ 'filename' => $file->getName(), 'file_data' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))), ], - ]; + ]]; } /** - * @return array{type: string, text: string} + * @return list */ private function buildTextContent(File $file): array { $fileType = $file->getMimeType(); @@ -244,14 +252,80 @@ private function buildTextContent(File $file): array { $this->l10n->t('Invalid input file type: "%1$s".', [$fileType]), ); } - return [ + return [[ 'type' => 'text', 'text' => 'Filename:' . $file->getName() . "\nContent:\n" . stream_get_contents($file->fopen('rb')), - ]; + ]]; } private function isUsingOpenAi(): bool { $serviceUrl = $this->openAiSettingsService->getServiceUrl(); return $serviceUrl === '' || $serviceUrl === Application::OPENAI_API_BASE_URL; } + + /** + * @return list + */ + private function buildImageFromFile(File $file): array { + if (!extension_loaded('imagick')) { + throw new RuntimeException('Imagick extension not available can not process PDF'); + } + if (empty(Imagick::queryFormats('PDF'))) { + throw new RuntimeException('Imagick has no PDF support (Ghostscript missing or blocked by policy.xml)'); + } + $this->logger->debug('Building image from PDF file: {file}', ['file' => $file->getPath()]); + + $pdfContent = $file->getContent(); + + // pingImageBlob avoids rasterizing every page into the pixel cache + $probe = new Imagick(); + try { + $probe->pingImageBlob($pdfContent); + $pageCount = $probe->getNumberImages(); + } finally { + $probe->clear(); + $probe->destroy(); + } + + // Limit pages to avoid overwhelming Imagick memory and the API + $pages = min(10, $pageCount); + $images = []; + + $document = new Imagick(); + try { + // Keep resolution low 72 is still readable just very pixelated + $document->setResolution(72, 72); + $document->readImageBlob($pdfContent); + + for ($i = 0; $i < $pages; $i++) { + $document->setIteratorIndex($i); + $page = $document->getImage(); + $flat = null; + try { + $page->setBackgroundColor('white'); + $flat = $page->mergeImageLayers(Imagick::LAYERMETHOD_FLATTEN); + $flat->setImageFormat('jpeg'); + $flat->setImageCompressionQuality(85); + $images[] = [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:image/jpeg;base64,' . base64_encode($flat->getImageBlob()), + ], + ]; + } finally { + $page->clear(); + $page->destroy(); + if ($flat instanceof Imagick) { + $flat->clear(); + $flat->destroy(); + } + } + } + } finally { + $document->clear(); + $document->destroy(); + } + + return $images; + } } diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php index 3c6385df..8624c972 100644 --- a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php +++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php @@ -13,7 +13,6 @@ use OCA\OpenAi\Service\OpenAiAPIService; use OCA\OpenAi\Service\OpenAiSettingsService; use OCA\OpenAi\Service\WatermarkingService; -use OCP\Http\Client\IClientService; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Exception\ProcessingException; @@ -27,12 +26,13 @@ class MultimodalChatWithToolsProvider implements IProvider, ISynchronousOptionsAwareProvider { + private const MAX_INPUT_ATTACHMENTS = 10; + public function __construct( private OpenAiAPIService $openAiAPIService, private OpenAiSettingsService $openAiSettingsService, private IL10N $l, private LoggerInterface $logger, - private IClientService $clientService, private WatermarkingService $watermarkingService, ) { } @@ -143,6 +143,14 @@ public function process( throw new ProcessingException('Invalid input_attachments'); } $inputAttachments = $input['input_attachments']; + if (count($inputAttachments) > self::MAX_INPUT_ATTACHMENTS) { + throw new UserFacingProcessingException( + 'Too many files. Max is ' . self::MAX_INPUT_ATTACHMENTS, + 0, + null, + $this->l->t('Too many files given. A maximum of %d files is allowed.', [self::MAX_INPUT_ATTACHMENTS]), + ); + } $maxTokens = null; if (isset($input['max_tokens']) && is_int($input['max_tokens'])) { @@ -215,6 +223,14 @@ public function process( } } + // Handle audio output + foreach ($returnValue['audio_messages'] as $audioMessage) { + if (!isset($audioMessage['audio']['data']) || !is_string($audioMessage['audio']['data'])) { + continue; + } + $attachments[] = base64_decode($audioMessage['audio']['data']); + } + return [ 'output' => array_pop($returnValue['messages']) ?? '', 'output_attachments' => $attachments, diff --git a/psalm.xml b/psalm.xml index 4e84d9f1..0e06a050 100644 --- a/psalm.xml +++ b/psalm.xml @@ -25,6 +25,7 @@ + diff --git a/tests/stubs/oc_hooks_emitter.php b/tests/stubs/oc_hooks_emitter.php new file mode 100644 index 00000000..c631dc00 --- /dev/null +++ b/tests/stubs/oc_hooks_emitter.php @@ -0,0 +1,29 @@ +createMock(\OCP\IL10N::class), $this->openAiSettingsService, + $this->createMock(\OCP\Files\IRootFolder::class), + $this->createMock(\Psr\Log\LoggerInterface::class), ), $this->createMock(\OCP\Notification\IManager::class), \OCP\Server::get(QuotaRuleService::class), @@ -241,6 +244,7 @@ public function testCreateStreamedChatCompletionReturnsStructuredChatResult(): v 'reasoning_messages' => [], 'tool_calls' => [], 'audio_messages' => [], + 'images' => [], ], $generator->getReturn()); $usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_TEXT); @@ -299,6 +303,7 @@ public function testCreateStreamedChatCompletionCanYieldStructuredReasoningChunk 'reasoning_messages' => ['Thinking... done.'], 'tool_calls' => [], 'audio_messages' => [], + 'images' => [], ], $generator->getReturn()); $usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_TEXT); @@ -1015,4 +1020,114 @@ public function testReformatParagraphsProvider(): void { $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); } + public function testMultimodalChatWithToolsProvider(): void { + $provider = new MultimodalChatWithToolsProvider( + $this->openAiApiService, + $this->openAiSettingsService, + $this->createMock(\OCP\IL10N::class), + $this->createMock(\Psr\Log\LoggerInterface::class), + \OCP\Server::get(WatermarkingService::class), + ); + + $prompt = 'What is in this image?'; + $systemPrompt = 'You are a helpful assistant.'; + $toolsJson = '[{"type":"function","function":{"name":"get_weather","description":"Get the weather","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}}]'; + $history = [ + json_encode(['role' => 'user', 'content' => 'Hello']), + json_encode(['role' => 'assistant', 'content' => 'Hi there!']), + ]; + $imageContent = 'fake-jpeg-bytes'; + $stream = fopen('php://temp', 'r+'); + if ($stream === false) { + throw new \RuntimeException('Could not open temp stream'); + } + fwrite($stream, $imageContent); + rewind($stream); + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getSize')->willReturn(strlen($imageContent)); + $file->method('getMimeType')->willReturn('image/jpeg'); + $file->method('fopen')->with('rb')->willReturn($stream); + + $response = '{ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-4.1-mini", + "system_fingerprint": "fp_44709d6fcb", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "This is a test response." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + }'; + + $url = self::OPENAI_API_BASE . 'chat/completions'; + $options = ['timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, 'headers' => ['User-Agent' => Application::USER_AGENT, 'Authorization' => self::AUTHORIZATION_HEADER, 'Content-Type' => 'application/json']]; + $options['body'] = json_encode([ + 'model' => Application::DEFAULT_COMPLETION_MODEL_ID, + 'messages' => [ + ['role' => 'system', 'content' => $systemPrompt], + ['role' => 'user', 'content' => 'Hello'], + ['role' => 'assistant', 'content' => 'Hi there!'], + [ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:image/jpeg;base64,' . base64_encode($imageContent), + ], + ], + [ + 'type' => 'text', + 'text' => $prompt, + ], + ], + ], + ], + 'n' => 1, + 'stream' => false, + 'max_completion_tokens' => Application::DEFAULT_MAX_NUM_OF_TOKENS, + 'tools' => json_decode($toolsJson), + 'user' => self::TEST_USER1, + ]); + + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getBody')->willReturn($response); + $iResponse->method('getStatusCode')->willReturn(200); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + + $this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse); + + $result = $provider->process(self::TEST_USER1, [ + 'input' => $prompt, + 'system_prompt' => $systemPrompt, + 'tool_message' => '', + 'tools' => $toolsJson, + 'history' => $history, + 'input_attachments' => [$file], + ], fn () => true, new SynchronousProviderOptions(preferStreaming: false)); + + $this->assertEquals('This is a test response.', $result['output']); + $this->assertEquals([], $result['output_attachments']); + $this->assertEquals('', $result['reasoning']); + $this->assertEquals('', $result['tool_calls']); + + $usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_TEXT); + $this->assertEquals(21, $usage); + $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); + } + } diff --git a/tests/unit/Quota/QuotaTest.php b/tests/unit/Quota/QuotaTest.php index 86082b48..3698f769 100644 --- a/tests/unit/Quota/QuotaTest.php +++ b/tests/unit/Quota/QuotaTest.php @@ -91,6 +91,8 @@ protected function setUp(): void { new OpenAiFileService( $this->createMock(IL10N::class), $this->openAiSettingsService, + $this->createMock(\OCP\Files\IRootFolder::class), + $this->createMock(LoggerInterface::class), ), $this->notificationManager, \OCP\Server::get(QuotaRuleService::class), diff --git a/tests/unit/Service/ServiceOverrideTest.php b/tests/unit/Service/ServiceOverrideTest.php index d73205ce..206dcd42 100644 --- a/tests/unit/Service/ServiceOverrideTest.php +++ b/tests/unit/Service/ServiceOverrideTest.php @@ -94,6 +94,8 @@ protected function setUp(): void { new OpenAiFileService( $this->createMock(\OCP\IL10N::class), $this->openAiSettingsService, + $this->createMock(\OCP\Files\IRootFolder::class), + $this->createMock(\Psr\Log\LoggerInterface::class), ), $this->createMock(\OCP\Notification\IManager::class), \OCP\Server::get(QuotaRuleService::class), From 18084484959800edbf713879d4e02e7f75468c78 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Wed, 15 Jul 2026 15:41:26 -0400 Subject: [PATCH 06/10] Support files specified in history by task id directly as needed for context agent Signed-off-by: Lukas Schaefer --- lib/Service/OpenAiAPIService.php | 3 +- lib/Service/OpenAiFileService.php | 54 +++++++++++++-------- tests/unit/Providers/OpenAiProviderTest.php | 1 + tests/unit/Quota/QuotaTest.php | 1 + tests/unit/Service/ServiceOverrideTest.php | 1 + 5 files changed, 39 insertions(+), 21 deletions(-) diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index bc4f6591..5341bcb6 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -734,7 +734,7 @@ private function buildChatCompletionRequestParams( $content = []; foreach ($message['content'] as $item) { if ($item['type'] === 'file') { - $content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId)); + $content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null)); } else { $content[] = $item; } @@ -1446,7 +1446,6 @@ private function normalizeChatCompletionResponse(array $response): array { 'images' => [], ]; - foreach ($response['choices'] as $choice) { if (!is_array($choice)) { continue; diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index 6b46fd1b..5dce19fb 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -16,6 +16,7 @@ use OCP\IL10N; use OCP\TaskProcessing\Exception\ProcessingException; use OCP\TaskProcessing\Exception\UserFacingProcessingException; +use OCP\TaskProcessing\IManager as ITaskProcessingManager; use Psr\Log\LoggerInterface; use RuntimeException; @@ -73,6 +74,7 @@ public function __construct( private OpenAiSettingsService $openAiSettingsService, private IRootFolder $rootFolder, private LoggerInterface $logger, + private ITaskProcessingManager $taskProcessingManager, ) { } @@ -81,13 +83,29 @@ public function __construct( * * @param int $fileId The ID of the file to build content from. * @param string $userId The user ID. + * @param ?int $taskId The ID of the task * @return list> Content parts suitable for OpenAI chat message content. * @throws ProcessingException * @throws UserFacingProcessingException */ - public function buildFileContentFromId(int $fileId, string $userId): array { - $userFolder = $this->rootFolder->getUserFolder($userId); - $file = $userFolder->getFirstNodeById($fileId); + public function buildFileContentFromId(int $fileId, string $userId, ?int $taskId): array { + $file = null; + if ($taskId !== null) { + $task = $this->taskProcessingManager->getUserTask($taskId, $userId); + $files = $this->taskProcessingManager->extractFileIdsFromTask($task); + + if (array_search($fileId, $files, true) === false) { + throw new ProcessingException('File does not exist'); + } + $file = $this->rootFolder->getFirstNodeById($fileId); + + if ($file === null) { + $file = $this->rootFolder->getFirstNodeByIdInPath($fileId, '/' . $this->rootFolder->getAppDataDirectoryName() . '/'); + } + } else { + $userFolder = $this->rootFolder->getUserFolder($userId); + $file = $userFolder->getFirstNodeById($fileId); + } return $this->buildFileContentFromFile($file); } @@ -114,26 +132,29 @@ public function buildFileContentFromFile(?File $file): array { } $fileType = $file->getMimeType(); + // Backup incase the file does not have an extension + if ($fileType === 'application/octet-stream') { + $fileType = mime_content_type($file->fopen('rb')); + } if (str_starts_with($fileType, 'image/')) { - return $this->buildImageContent($file); + return $this->buildImageContent($file, $fileType); // OpenAI only supports this for very specific models and support is not that common } elseif (str_starts_with($fileType, 'audio/')) { - return $this->buildAudioContent($file); + return $this->buildAudioContent($file, $fileType); // OpenAI does not currently support video attachments } elseif (str_starts_with($fileType, 'video/')) { - return $this->buildVideoContent($file); + return $this->buildVideoContent($file, $fileType); } elseif ($fileType === 'application/pdf') { - return $this->buildDocumentContent($file); + return $this->buildDocumentContent($file, $fileType); } else { - return $this->buildTextContent($file); + return $this->buildTextContent($file, $fileType); } } - /** * @return list */ - private function buildImageContent(File $file): array { + private function buildImageContent(File $file, string $fileType): array { if (!$this->openAiSettingsService->getMultimodalImageEnabled()) { throw new UserFacingProcessingException( 'Image attachments are disabled', @@ -142,7 +163,6 @@ private function buildImageContent(File $file): array { $this->l10n->t('Image attachments are unsupported.'), ); } - $fileType = $file->getMimeType(); if ($this->isUsingOpenAi() && !in_array($fileType, self::VALID_IMAGE_MIME_TYPES, true)) { throw new UserFacingProcessingException( 'Invalid input file type for OpenAI ' . $fileType, @@ -162,7 +182,7 @@ private function buildImageContent(File $file): array { /** * @return list */ - private function buildAudioContent(File $file): array { + private function buildAudioContent(File $file, string $fileType): array { if (!$this->openAiSettingsService->getMultimodalAudioEnabled()) { throw new UserFacingProcessingException( 'Audio attachments are disabled', @@ -171,7 +191,6 @@ private function buildAudioContent(File $file): array { $this->l10n->t('Audio attachments are unsupported.'), ); } - $fileType = $file->getMimeType(); if (!array_key_exists($fileType, self::SUPPORTED_INPUT_AUDIO_FORMATS)) { throw new UserFacingProcessingException( @@ -194,7 +213,7 @@ private function buildAudioContent(File $file): array { /** * @return list */ - private function buildVideoContent(File $file): array { + private function buildVideoContent(File $file, string $fileType): array { if (!$this->openAiSettingsService->getMultimodalVideoEnabled()) { throw new UserFacingProcessingException( 'Video attachments are disabled', @@ -203,7 +222,6 @@ private function buildVideoContent(File $file): array { $this->l10n->t('Video attachments are unsupported.'), ); } - $fileType = $file->getMimeType(); return [[ 'type' => 'video_url', 'video_url' => [ @@ -215,7 +233,7 @@ private function buildVideoContent(File $file): array { /** * @return list */ - private function buildDocumentContent(File $file): array { + private function buildDocumentContent(File $file, string $fileType): array { if (!$this->openAiSettingsService->getMultimodalDocumentEnabled()) { // Fallback to image if documents are not supported if ($this->openAiSettingsService->getMultimodalImageEnabled()) { @@ -228,7 +246,6 @@ private function buildDocumentContent(File $file): array { $this->l10n->t('Document attachments are unsupported.'), ); } - $fileType = $file->getMimeType(); return [[ 'type' => 'file', 'file' => [ @@ -241,8 +258,7 @@ private function buildDocumentContent(File $file): array { /** * @return list */ - private function buildTextContent(File $file): array { - $fileType = $file->getMimeType(); + private function buildTextContent(File $file, string $fileType): array { // Sanity check that this isn't a binary if (!str_starts_with($fileType, 'text/') && !in_array($fileType, self::VALID_TEXT_MIME_TYPES, true)) { throw new UserFacingProcessingException( diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php index 93e62d50..b3b3fd6f 100644 --- a/tests/unit/Providers/OpenAiProviderTest.php +++ b/tests/unit/Providers/OpenAiProviderTest.php @@ -103,6 +103,7 @@ protected function setUp(): void { $this->openAiSettingsService, $this->createMock(\OCP\Files\IRootFolder::class), $this->createMock(\Psr\Log\LoggerInterface::class), + $this->createMock(\OCP\TaskProcessing\IManager::class), ), $this->createMock(\OCP\Notification\IManager::class), \OCP\Server::get(QuotaRuleService::class), diff --git a/tests/unit/Quota/QuotaTest.php b/tests/unit/Quota/QuotaTest.php index 3698f769..fed0baac 100644 --- a/tests/unit/Quota/QuotaTest.php +++ b/tests/unit/Quota/QuotaTest.php @@ -93,6 +93,7 @@ protected function setUp(): void { $this->openAiSettingsService, $this->createMock(\OCP\Files\IRootFolder::class), $this->createMock(LoggerInterface::class), + $this->createMock(\OCP\TaskProcessing\IManager::class), ), $this->notificationManager, \OCP\Server::get(QuotaRuleService::class), diff --git a/tests/unit/Service/ServiceOverrideTest.php b/tests/unit/Service/ServiceOverrideTest.php index 206dcd42..830616a4 100644 --- a/tests/unit/Service/ServiceOverrideTest.php +++ b/tests/unit/Service/ServiceOverrideTest.php @@ -96,6 +96,7 @@ protected function setUp(): void { $this->openAiSettingsService, $this->createMock(\OCP\Files\IRootFolder::class), $this->createMock(\Psr\Log\LoggerInterface::class), + $this->createMock(\OCP\TaskProcessing\IManager::class), ), $this->createMock(\OCP\Notification\IManager::class), \OCP\Server::get(QuotaRuleService::class), From 1902b7fe609cd1b4a113aba859063eb55e6207e2 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Thu, 16 Jul 2026 08:38:54 -0400 Subject: [PATCH 07/10] Add more validation and handle null userid Signed-off-by: Lukas Schaefer --- lib/Service/OpenAiAPIService.php | 16 ++++++++++++++++ lib/Service/OpenAiFileService.php | 7 ++++--- .../MultimodalChatWithToolsProvider.php | 4 ++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 5341bcb6..012b1be8 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -733,7 +733,23 @@ private function buildChatCompletionRequestParams( if (isset($message['content']) && is_array($message['content'])) { $content = []; foreach ($message['content'] as $item) { + if (!isset($item['type'])) { + throw new UserFacingProcessingException( + 'Invalid message history content', + 0, + null, + $this->l10n->t('Invalid message history content'), + ); + } if ($item['type'] === 'file') { + if (!isset($item['file_id'])) { + throw new UserFacingProcessingException( + 'Invalid message history content', + 0, + null, + $this->l10n->t('Invalid message history content'), + ); + } $content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null)); } else { $content[] = $item; diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index 5dce19fb..3c3b84fd 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -82,13 +82,13 @@ public function __construct( * Builds file content from a file ID within a given user folder. * * @param int $fileId The ID of the file to build content from. - * @param string $userId The user ID. + * @param ?string $userId The user ID. * @param ?int $taskId The ID of the task * @return list> Content parts suitable for OpenAI chat message content. * @throws ProcessingException * @throws UserFacingProcessingException */ - public function buildFileContentFromId(int $fileId, string $userId, ?int $taskId): array { + public function buildFileContentFromId(int $fileId, ?string $userId, ?int $taskId): array { $file = null; if ($taskId !== null) { $task = $this->taskProcessingManager->getUserTask($taskId, $userId); @@ -102,7 +102,8 @@ public function buildFileContentFromId(int $fileId, string $userId, ?int $taskId if ($file === null) { $file = $this->rootFolder->getFirstNodeByIdInPath($fileId, '/' . $this->rootFolder->getAppDataDirectoryName() . '/'); } - } else { + // If the userId is not specified they don't have a user folder + } elseif ($userId !== null) { $userFolder = $this->rootFolder->getUserFolder($userId); $file = $userFolder->getFirstNodeById($fileId); } diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php index 8624c972..27857352 100644 --- a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php +++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php @@ -214,12 +214,12 @@ public function process( foreach ($returnValue['images'] as $image) { if ($image['type'] === 'image_url') { $url = $image['image_url']['url']; - $base64Str = explode(',', $url)[1] ?? ''; + $base64Str = explode(',', $url)[1] ?? throw new ProcessingException('Invalid image URL in multimodal chat: ' . $url); $image = base64_decode($base64Str); $image = $this->watermarkingService->markImage($image); $attachments[] = $image; } else { - $this->logger->warning('Encountered an unknown image type in multimodal chat: ' . $image['type']); + throw new ProcessingException('Unknown image type in multimodal chat: ' . $image['type']); } } From 986e5839c2ecd49861ecc94e3b9eb29f89bfb464 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 20 Jul 2026 09:17:47 -0400 Subject: [PATCH 08/10] add filesize limit for analyzeimage back and skip historical file content of unsupported files Signed-off-by: Lukas Schaefer --- lib/Service/OpenAiAPIService.php | 7 ++++++- lib/TaskProcessing/AnalyzeImagesProvider.php | 20 +++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 012b1be8..1a6c849c 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -750,7 +750,12 @@ private function buildChatCompletionRequestParams( $this->l10n->t('Invalid message history content'), ); } - $content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null)); + // If the history contains a file that isn't supported anymore we should skip it so the chat isn't broken + try { + $content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null)); + } catch (ProcessingException|UserFacingProcessingException $e) { + $this->logger->warning('Could not build file content from id: ' . $item['file_id'] . '. Error: ' . $e->getMessage(), ['app' => Application::APP_ID]); + } } else { $content[] = $item; } diff --git a/lib/TaskProcessing/AnalyzeImagesProvider.php b/lib/TaskProcessing/AnalyzeImagesProvider.php index df822c12..1ba992e3 100644 --- a/lib/TaskProcessing/AnalyzeImagesProvider.php +++ b/lib/TaskProcessing/AnalyzeImagesProvider.php @@ -105,7 +105,10 @@ public function getOptionalOutputShapeEnumValues(): array { } public function process( - ?string $userId, array $input, callable $reportProgress, SynchronousProviderOptions $options = new SynchronousProviderOptions(), + ?string $userId, + array $input, + callable $reportProgress, + SynchronousProviderOptions $options = new SynchronousProviderOptions(), ): array { $reportOutput = $options->getReportIntermediateOutput(); $preferStreaming = $options->getPreferStreaming(); @@ -136,6 +139,21 @@ public function process( if (isset($input['max_tokens']) && is_int($input['max_tokens'])) { $maxTokens = $input['max_tokens']; } + $fileSizeTotal = array_reduce( + $images, + function ($carry, $file) { + return $carry + (method_exists($file, 'getSize') ? $file->getSize() : 0); + }, + 0 + ); + if ($fileSizeTotal > 50 * 1000 * 1000) { + throw new UserFacingProcessingException( + 'Filesize of input files too large. Max is 50MB', + 0, + null, + $this->l->t('The total size of the input files is too large. A maximum of 50MB is allowed.'), + ); + } try { $systemPrompt = 'Take the user\'s question and answer it based on the provided images. Ensure that the answer matches the language of the user\'s question.'; From 6c1ebebd822eb033e04725b8101f02863066737b Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Tue, 21 Jul 2026 16:09:03 -0400 Subject: [PATCH 09/10] drop imagick parsing of pdfs Signed-off-by: Lukas Schaefer --- lib/Service/OpenAiAPIService.php | 1 + lib/Service/OpenAiFileService.php | 76 +-------------------- tests/unit/Providers/OpenAiProviderTest.php | 1 - tests/unit/Quota/QuotaTest.php | 1 - tests/unit/Service/ServiceOverrideTest.php | 1 - 5 files changed, 2 insertions(+), 78 deletions(-) diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 1a6c849c..4a1a5e9f 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -29,6 +29,7 @@ use OCP\IL10N; use OCP\Lock\LockedException; use OCP\Notification\IManager as INotificationManager; +use OCP\TaskProcessing\Exception\ProcessingException; use OCP\TaskProcessing\Exception\UserFacingProcessingException; use OCP\TaskProcessing\ShapeEnumValue; use Psr\Log\LoggerInterface; diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index 3c3b84fd..d1977bca 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -9,7 +9,6 @@ namespace OCA\OpenAi\Service; -use Imagick; use OCA\OpenAi\AppInfo\Application; use OCP\Files\File; use OCP\Files\IRootFolder; @@ -17,8 +16,6 @@ use OCP\TaskProcessing\Exception\ProcessingException; use OCP\TaskProcessing\Exception\UserFacingProcessingException; use OCP\TaskProcessing\IManager as ITaskProcessingManager; -use Psr\Log\LoggerInterface; -use RuntimeException; class OpenAiFileService { private const MAX_FILE_SIZE_BYTES = 50_000_000; @@ -73,7 +70,6 @@ public function __construct( private IL10N $l10n, private OpenAiSettingsService $openAiSettingsService, private IRootFolder $rootFolder, - private LoggerInterface $logger, private ITaskProcessingManager $taskProcessingManager, ) { } @@ -232,14 +228,10 @@ private function buildVideoContent(File $file, string $fileType): array { } /** - * @return list + * @return list */ private function buildDocumentContent(File $file, string $fileType): array { if (!$this->openAiSettingsService->getMultimodalDocumentEnabled()) { - // Fallback to image if documents are not supported - if ($this->openAiSettingsService->getMultimodalImageEnabled()) { - return $this->buildImageFromFile($file); - } throw new UserFacingProcessingException( 'Document attachments are disabled', 0, @@ -279,70 +271,4 @@ private function isUsingOpenAi(): bool { $serviceUrl = $this->openAiSettingsService->getServiceUrl(); return $serviceUrl === '' || $serviceUrl === Application::OPENAI_API_BASE_URL; } - - /** - * @return list - */ - private function buildImageFromFile(File $file): array { - if (!extension_loaded('imagick')) { - throw new RuntimeException('Imagick extension not available can not process PDF'); - } - if (empty(Imagick::queryFormats('PDF'))) { - throw new RuntimeException('Imagick has no PDF support (Ghostscript missing or blocked by policy.xml)'); - } - $this->logger->debug('Building image from PDF file: {file}', ['file' => $file->getPath()]); - - $pdfContent = $file->getContent(); - - // pingImageBlob avoids rasterizing every page into the pixel cache - $probe = new Imagick(); - try { - $probe->pingImageBlob($pdfContent); - $pageCount = $probe->getNumberImages(); - } finally { - $probe->clear(); - $probe->destroy(); - } - - // Limit pages to avoid overwhelming Imagick memory and the API - $pages = min(10, $pageCount); - $images = []; - - $document = new Imagick(); - try { - // Keep resolution low 72 is still readable just very pixelated - $document->setResolution(72, 72); - $document->readImageBlob($pdfContent); - - for ($i = 0; $i < $pages; $i++) { - $document->setIteratorIndex($i); - $page = $document->getImage(); - $flat = null; - try { - $page->setBackgroundColor('white'); - $flat = $page->mergeImageLayers(Imagick::LAYERMETHOD_FLATTEN); - $flat->setImageFormat('jpeg'); - $flat->setImageCompressionQuality(85); - $images[] = [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => 'data:image/jpeg;base64,' . base64_encode($flat->getImageBlob()), - ], - ]; - } finally { - $page->clear(); - $page->destroy(); - if ($flat instanceof Imagick) { - $flat->clear(); - $flat->destroy(); - } - } - } - } finally { - $document->clear(); - $document->destroy(); - } - - return $images; - } } diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php index b3b3fd6f..194861fa 100644 --- a/tests/unit/Providers/OpenAiProviderTest.php +++ b/tests/unit/Providers/OpenAiProviderTest.php @@ -102,7 +102,6 @@ protected function setUp(): void { $this->createMock(\OCP\IL10N::class), $this->openAiSettingsService, $this->createMock(\OCP\Files\IRootFolder::class), - $this->createMock(\Psr\Log\LoggerInterface::class), $this->createMock(\OCP\TaskProcessing\IManager::class), ), $this->createMock(\OCP\Notification\IManager::class), diff --git a/tests/unit/Quota/QuotaTest.php b/tests/unit/Quota/QuotaTest.php index fed0baac..a68bed11 100644 --- a/tests/unit/Quota/QuotaTest.php +++ b/tests/unit/Quota/QuotaTest.php @@ -92,7 +92,6 @@ protected function setUp(): void { $this->createMock(IL10N::class), $this->openAiSettingsService, $this->createMock(\OCP\Files\IRootFolder::class), - $this->createMock(LoggerInterface::class), $this->createMock(\OCP\TaskProcessing\IManager::class), ), $this->notificationManager, diff --git a/tests/unit/Service/ServiceOverrideTest.php b/tests/unit/Service/ServiceOverrideTest.php index 830616a4..a7d8cfa7 100644 --- a/tests/unit/Service/ServiceOverrideTest.php +++ b/tests/unit/Service/ServiceOverrideTest.php @@ -95,7 +95,6 @@ protected function setUp(): void { $this->createMock(\OCP\IL10N::class), $this->openAiSettingsService, $this->createMock(\OCP\Files\IRootFolder::class), - $this->createMock(\Psr\Log\LoggerInterface::class), $this->createMock(\OCP\TaskProcessing\IManager::class), ), $this->createMock(\OCP\Notification\IManager::class), From 0f848f8d0b9d3f0d7ce8026b0448ae96c43bfd0a Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Thu, 23 Jul 2026 09:13:47 -0400 Subject: [PATCH 10/10] Add note about real urls Signed-off-by: Lukas Schaefer --- lib/TaskProcessing/MultimodalChatWithToolsProvider.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php index 27857352..36201610 100644 --- a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php +++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php @@ -212,6 +212,7 @@ public function process( // Handle image output foreach ($returnValue['images'] as $image) { + // Currently only data URIs have been seen, but if a real URL is found we should support it if ($image['type'] === 'image_url') { $url = $image['image_url']['url']; $base64Str = explode(',', $url)[1] ?? throw new ProcessingException('Invalid image URL in multimodal chat: ' . $url);