+
@@ -47,6 +48,7 @@
+
diff --git a/src/components/AdminSettings.vue b/src/components/AdminSettings.vue
index 01551409..73f99bba 100644
--- a/src/components/AdminSettings.vue
+++ b/src/components/AdminSettings.vue
@@ -326,6 +326,34 @@
{{ t('integration_openai', 'Use "{newParam}" parameter instead of the deprecated "{deprecatedParam}"', { newParam: 'max_completion_tokens', deprecatedParam: 'max_tokens' }) }}
+
+ {{ t('integration_openai', 'Multimodal LLM Support') }}
+
+
+ {{ t('integration_openai', 'Multimodal LLM Support allows you to enable or disable the use of images, audio, video and document attachments in the LLM.') }}
+
+
+
+ {{ t('integration_openai', 'Image attachments') }}
+
+
+ {{ t('integration_openai', 'Audio attachments') }}
+
+
+ {{ t('integration_openai', 'Video attachments') }}
+
+
+ {{ t('integration_openai', 'Document attachments') }}
+
+
@@ -633,11 +661,6 @@
@update:model-value="onCheckboxChanged($event, 'tts_provider_enabled', false)">
{{ t('integration_openai', 'Text-to-speech provider') }}
-
- {{ t('integration_openai', 'Analyze image provider') }}
-
diff --git a/tests/stubs/oc_hooks_emitter.php b/tests/stubs/oc_hooks_emitter.php
new file mode 100644
index 00000000..c631dc00
--- /dev/null
+++ b/tests/stubs/oc_hooks_emitter.php
@@ -0,0 +1,29 @@
+openAiSettingsService,
$this->streamingService,
+ new OpenAiFileService(
+ $this->createMock(\OCP\IL10N::class),
+ $this->openAiSettingsService,
+ $this->createMock(\OCP\Files\IRootFolder::class),
+ $this->createMock(\OCP\TaskProcessing\IManager::class),
+ ),
$this->createMock(\OCP\Notification\IManager::class),
\OCP\Server::get(QuotaRuleService::class),
$clientService,
@@ -236,6 +244,7 @@ public function testCreateStreamedChatCompletionReturnsStructuredChatResult(): v
'reasoning_messages' => [],
'tool_calls' => [],
'audio_messages' => [],
+ 'images' => [],
], $generator->getReturn());
$usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_TEXT);
@@ -294,6 +303,7 @@ public function testCreateStreamedChatCompletionCanYieldStructuredReasoningChunk
'reasoning_messages' => ['Thinking... done.'],
'tool_calls' => [],
'audio_messages' => [],
+ 'images' => [],
], $generator->getReturn());
$usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_TEXT);
@@ -1096,4 +1106,114 @@ public function testCreateChatCompletionExtractsMistralTypedTextContent(): void
$this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1);
}
+ public function testMultimodalChatWithToolsProvider(): void {
+ $provider = new MultimodalChatWithToolsProvider(
+ $this->openAiApiService,
+ $this->openAiSettingsService,
+ $this->createMock(\OCP\IL10N::class),
+ $this->createMock(\Psr\Log\LoggerInterface::class),
+ \OCP\Server::get(WatermarkingService::class),
+ );
+
+ $prompt = 'What is in this image?';
+ $systemPrompt = 'You are a helpful assistant.';
+ $toolsJson = '[{"type":"function","function":{"name":"get_weather","description":"Get the weather","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}}]';
+ $history = [
+ json_encode(['role' => 'user', 'content' => 'Hello']),
+ json_encode(['role' => 'assistant', 'content' => 'Hi there!']),
+ ];
+ $imageContent = 'fake-jpeg-bytes';
+ $stream = fopen('php://temp', 'r+');
+ if ($stream === false) {
+ throw new \RuntimeException('Could not open temp stream');
+ }
+ fwrite($stream, $imageContent);
+ rewind($stream);
+
+ $file = $this->createMock(\OCP\Files\File::class);
+ $file->method('isReadable')->willReturn(true);
+ $file->method('getSize')->willReturn(strlen($imageContent));
+ $file->method('getMimeType')->willReturn('image/jpeg');
+ $file->method('fopen')->with('rb')->willReturn($stream);
+
+ $response = '{
+ "id": "chatcmpl-123",
+ "object": "chat.completion",
+ "created": 1677652288,
+ "model": "gpt-4.1-mini",
+ "system_fingerprint": "fp_44709d6fcb",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": "This is a test response."
+ },
+ "finish_reason": "stop"
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 9,
+ "completion_tokens": 12,
+ "total_tokens": 21
+ }
+ }';
+
+ $url = self::OPENAI_API_BASE . 'chat/completions';
+ $options = ['timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, 'headers' => ['User-Agent' => Application::USER_AGENT, 'Authorization' => self::AUTHORIZATION_HEADER, 'Content-Type' => 'application/json']];
+ $options['body'] = json_encode([
+ 'model' => Application::DEFAULT_COMPLETION_MODEL_ID,
+ 'messages' => [
+ ['role' => 'system', 'content' => $systemPrompt],
+ ['role' => 'user', 'content' => 'Hello'],
+ ['role' => 'assistant', 'content' => 'Hi there!'],
+ [
+ 'role' => 'user',
+ 'content' => [
+ [
+ 'type' => 'image_url',
+ 'image_url' => [
+ 'url' => 'data:image/jpeg;base64,' . base64_encode($imageContent),
+ ],
+ ],
+ [
+ 'type' => 'text',
+ 'text' => $prompt,
+ ],
+ ],
+ ],
+ ],
+ 'n' => 1,
+ 'stream' => false,
+ 'max_completion_tokens' => Application::DEFAULT_MAX_NUM_OF_TOKENS,
+ 'tools' => json_decode($toolsJson),
+ 'user' => self::TEST_USER1,
+ ]);
+
+ $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class);
+ $iResponse->method('getBody')->willReturn($response);
+ $iResponse->method('getStatusCode')->willReturn(200);
+ $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json');
+
+ $this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse);
+
+ $result = $provider->process(self::TEST_USER1, [
+ 'input' => $prompt,
+ 'system_prompt' => $systemPrompt,
+ 'tool_message' => '',
+ 'tools' => $toolsJson,
+ 'history' => $history,
+ 'input_attachments' => [$file],
+ ], fn () => true, new SynchronousProviderOptions(preferStreaming: false));
+
+ $this->assertEquals('This is a test response.', $result['output']);
+ $this->assertEquals([], $result['output_attachments']);
+ $this->assertEquals('', $result['reasoning']);
+ $this->assertEquals('', $result['tool_calls']);
+
+ $usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_TEXT);
+ $this->assertEquals(21, $usage);
+ $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1);
+ }
+
}
diff --git a/tests/unit/Quota/QuotaTest.php b/tests/unit/Quota/QuotaTest.php
index f4c9cb48..a68bed11 100644
--- a/tests/unit/Quota/QuotaTest.php
+++ b/tests/unit/Quota/QuotaTest.php
@@ -16,6 +16,7 @@
use OCA\OpenAi\Db\EntityType;
use OCA\OpenAi\Db\QuotaUsageMapper;
use OCA\OpenAi\Service\OpenAiAPIService;
+use OCA\OpenAi\Service\OpenAiFileService;
use OCA\OpenAi\Service\OpenAiSettingsService;
use OCA\OpenAi\Service\QuotaRuleService;
use OCA\OpenAi\Service\StreamingService;
@@ -87,6 +88,12 @@ protected function setUp(): void {
new StreamingService(
$this->createMock(IL10N::class),
),
+ new OpenAiFileService(
+ $this->createMock(IL10N::class),
+ $this->openAiSettingsService,
+ $this->createMock(\OCP\Files\IRootFolder::class),
+ $this->createMock(\OCP\TaskProcessing\IManager::class),
+ ),
$this->notificationManager,
\OCP\Server::get(QuotaRuleService::class),
\OCP\Server::get(IClientService::class),
diff --git a/tests/unit/Service/ServiceOverrideTest.php b/tests/unit/Service/ServiceOverrideTest.php
index 9fc7fa72..a7d8cfa7 100644
--- a/tests/unit/Service/ServiceOverrideTest.php
+++ b/tests/unit/Service/ServiceOverrideTest.php
@@ -16,6 +16,7 @@
use OCA\OpenAi\Db\QuotaUsageMapper;
use OCA\OpenAi\Service\ChunkService;
use OCA\OpenAi\Service\OpenAiAPIService;
+use OCA\OpenAi\Service\OpenAiFileService;
use OCA\OpenAi\Service\OpenAiSettingsService;
use OCA\OpenAi\Service\QuotaRuleService;
use OCA\OpenAi\Service\StreamingService;
@@ -90,6 +91,12 @@ protected function setUp(): void {
new StreamingService(
$this->createMock(\OCP\IL10N::class),
),
+ new OpenAiFileService(
+ $this->createMock(\OCP\IL10N::class),
+ $this->openAiSettingsService,
+ $this->createMock(\OCP\Files\IRootFolder::class),
+ $this->createMock(\OCP\TaskProcessing\IManager::class),
+ ),
$this->createMock(\OCP\Notification\IManager::class),
\OCP\Server::get(QuotaRuleService::class),
$clientService,