From dd144a3c041c9cd1b9c8942b1c4ed18969dedeed Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Thu, 21 May 2026 16:18:29 +0200 Subject: [PATCH 1/5] feat: support ocr for images using llm Signed-off-by: Lukas Schaefer --- lib/AppInfo/Application.php | 1 + lib/TaskProcessing/ImageToTextOcrProvider.php | 192 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 lib/TaskProcessing/ImageToTextOcrProvider.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 7854fa4b..27be6312 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -149,6 +149,7 @@ public function register(IRegistrationContext $context): void { $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider::class); } if ($this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1') === '1') { + $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ImageToTextOcrProvider::class); $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\AnalyzeImagesProvider::class); } } diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php new file mode 100644 index 00000000..44e7a23b --- /dev/null +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -0,0 +1,192 @@ +openAiAPIService->getServiceName(); + } + + public function getTaskTypeId(): string { + return ImageToTextOpticalCharacterRecognition::ID; + } + + public function getExpectedRuntime(): int { + return $this->openAiAPIService->getExpTextProcessingTime(); + } + + public function getInputShapeEnumValues(): array { + return []; + } + + public function getInputShapeDefaults(): array { + return []; + } + + public function getOptionalInputShape(): array { + return [ + 'max_tokens' => new ShapeDescriptor( + $this->l->t('Maximum output words'), + $this->l->t('The maximum number of words/tokens that can be generated in the output.'), + EShapeType::Number + ), + 'model' => new ShapeDescriptor( + $this->l->t('Model'), + $this->l->t('The model used to generate the output'), + EShapeType::Enum + ), + ]; + } + + public function getOptionalInputShapeEnumValues(): array { + return [ + 'model' => $this->openAiAPIService->getModelEnumValues($this->userId), + ]; + } + + public function getOptionalInputShapeDefaults(): array { + $adminModel = $this->openAiSettingsService->getAdminDefaultCompletionModelId(); + return [ + 'max_tokens' => $this->openAiSettingsService->getMaxTokens(), + 'model' => $adminModel, + ]; + } + + public function getOutputShapeEnumValues(): array { + return []; + } + + public function getOptionalOutputShape(): array { + return []; + } + + public function getOptionalOutputShapeEnumValues(): array { + return []; + } + + public function process(?string $userId, array $input, callable $reportProgress): array { + if (!$this->openAiAPIService->isUsingOpenAi() && !$this->openAiSettingsService->getChatEndpointEnabled()) { + throw new RuntimeException('Must support chat completion endpoint'); + } + + if (!isset($input['input']) || !is_array($input['input'])) { + throw new RuntimeException('Invalid file list'); + } + if (count($input['input']) === 0) { + throw new RuntimeException('Invalid file list'); + } + if (count($input['input']) > 500) { + throw new RuntimeException('Too many files given. Max is 500'); + } + + if (isset($input['model']) && is_string($input['model'])) { + $model = $input['model']; + } else { + $model = $this->openAiSettingsService->getAdminDefaultCompletionModelId(); + } + + $maxTokens = null; + if (isset($input['max_tokens']) && is_int($input['max_tokens'])) { + $maxTokens = $input['max_tokens']; + } + + $fileSize = 0; + $outputs = []; + $systemPrompt = 'Extract all visible text from the image. Return only the extracted text without additional commentary. Preserve the original language of the text.'; + $userPrompt = 'Extract all text from this image.'; + + foreach ($input['input'] as $i => $file) { + if (!$file instanceof File || !$file->isReadable()) { + throw new RuntimeException('Invalid input file'); + } + $fileSize += intval($file->getSize()); + if ($fileSize > 50 * 1000 * 1000) { + throw new UserFacingProcessingException('Filesize of input files too large. Max is 50MB', userFacingMessage: $this->l->t('Filesize of input files too large. Max is 50MB')); + } + + $fileType = $file->getMimeType(); + if (!str_starts_with($fileType, 'image/')) { + throw new UserFacingProcessingException('Only supports image file types' . $fileType, userFacingMessage: $this->l->t('Only supports image file types')); + } + if ($this->openAiAPIService->isUsingOpenAi()) { + $validFileTypes = [ + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + ]; + if (!in_array($fileType, $validFileTypes, true)) { + throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType); + } + } + + $inputFile = base64_encode(stream_get_contents($file->fopen('rb'))); + $history = [ + json_encode([ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:' . $fileType . ';base64,' . $inputFile, + ], + ], + ], + ]), + ]; + + try { + $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $messages = $completion['messages']; + + if (count($messages) === 0) { + throw new RuntimeException('No result in OpenAI/LocalAI response.'); + } + + $outputs[] = array_pop($messages); + $reportProgress(($i + 1) / count($input['input'])); + } catch (\Exception $e) { + $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + } + } + + return ['output' => $outputs]; + } +} From 18ca8704e534c678fac87319644f8b983fb5fdc3 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 6 Jul 2026 17:12:04 -0400 Subject: [PATCH 2/5] Support pdfs by converting them to images Signed-off-by: Lukas Schaefer --- lib/TaskProcessing/ImageToTextOcrProvider.php | 157 +++++++++++++----- 1 file changed, 118 insertions(+), 39 deletions(-) diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php index 44e7a23b..3bc91f46 100644 --- a/lib/TaskProcessing/ImageToTextOcrProvider.php +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -9,11 +9,11 @@ namespace OCA\OpenAi\TaskProcessing; +use Imagick; use OCA\OpenAi\AppInfo\Application; use OCA\OpenAi\Service\OpenAiAPIService; use OCA\OpenAi\Service\OpenAiSettingsService; use OCP\Files\File; -use OCP\IAppConfig; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Exception\UserFacingProcessingException; @@ -30,7 +30,6 @@ public function __construct( private OpenAiSettingsService $openAiSettingsService, private IL10N $l, private LoggerInterface $logger, - private IAppConfig $appConfig, private ?string $userId, ) { } @@ -128,10 +127,12 @@ public function process(?string $userId, array $input, callable $reportProgress) $fileSize = 0; $outputs = []; + $fileCount = (float)count($input['input']); $systemPrompt = 'Extract all visible text from the image. Return only the extracted text without additional commentary. Preserve the original language of the text.'; $userPrompt = 'Extract all text from this image.'; - foreach ($input['input'] as $i => $file) { + $fileIndex = 0; + foreach ($input['input'] as $file) { if (!$file instanceof File || !$file->isReadable()) { throw new RuntimeException('Invalid input file'); } @@ -141,52 +142,130 @@ public function process(?string $userId, array $input, callable $reportProgress) } $fileType = $file->getMimeType(); - if (!str_starts_with($fileType, 'image/')) { - throw new UserFacingProcessingException('Only supports image file types' . $fileType, userFacingMessage: $this->l->t('Only supports image file types')); - } - if ($this->openAiAPIService->isUsingOpenAi()) { - $validFileTypes = [ - 'image/jpeg', - 'image/png', - 'image/gif', - 'image/webp', - ]; - if (!in_array($fileType, $validFileTypes, true)) { - throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType); - } - } - $inputFile = base64_encode(stream_get_contents($file->fopen('rb'))); - $history = [ - json_encode([ - 'role' => 'user', - 'content' => [ - [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => 'data:' . $fileType . ';base64,' . $inputFile, + if ($fileType === 'application/pdf') { + $outputForFile = ''; + $imagickProbe = $this->getImagickProbe($file); + $pagesRead = 0; + foreach ($this->imagickPdfToJpegBase64($imagickProbe['image']) as $base64Image) { + $pagesRead++; + $history = [json_encode([ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:image/jpeg;base64,' . $base64Image, + ], ], ], - ], - ]), - ]; + ])]; + try { + $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $messages = $completion['messages']; - try { - $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); - $messages = $completion['messages']; + if (count($messages) === 0) { + $this->logger->warning('No result in OpenAI/LocalAI response.'); + $outputs[] = ''; + continue; + } - if (count($messages) === 0) { - throw new RuntimeException('No result in OpenAI/LocalAI response.'); + $outputForFile .= array_pop($messages) . "\n\n"; + $reportProgress(((float)$fileIndex + (float)($pagesRead / $imagickProbe['count'])) / $fileCount); + } catch (\Exception $e) { + $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + } + } + $outputs[] = $outputForFile; + $reportProgress(((float)$fileIndex + 1.0) / $fileCount); + } elseif (str_starts_with($fileType, 'image/')) { + if ($this->openAiAPIService->isUsingOpenAi()) { + $validFileTypes = [ + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + ]; + if (!in_array($fileType, $validFileTypes, true)) { + throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType); + } } - $outputs[] = array_pop($messages); - $reportProgress(($i + 1) / count($input['input'])); - } catch (\Exception $e) { - $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); - throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + $base64Image = base64_encode(stream_get_contents($file->fopen('rb'))); + $history = [ + json_encode([ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:' . $fileType . ';base64,' . $base64Image, + ], + ], + ], + ]), + ]; + try { + $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $messages = $completion['messages']; + $reportProgress(((float)$fileIndex + 1.0) / $fileCount); + + if (count($messages) === 0) { + $this->logger->warning('No result in OpenAI/LocalAI response.'); + $outputs[] = ''; + continue; + } + + $outputs[] = array_pop($messages); + } catch (\Exception $e) { + $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + } + } else { + throw new UserFacingProcessingException('Only supports image and pdf file types' . $fileType, userFacingMessage: $this->l->t('Only supports image and pdf file types')); } + $fileIndex++; } return ['output' => $outputs]; } + + /** + * @return array{count: int, image: Imagick} + */ + private function getImagickProbe(File $file): array { + if (!extension_loaded('imagick')) { + throw new RuntimeException('Imagick extension not available can not process PDF'); + } + if (empty(Imagick::queryFormats('PDF'))) { + throw new RuntimeException('Imagick has no PDF support (Ghostscript missing or blocked by policy.xml)'); + } + + $pdfContent = $file->getContent(); + $probe = new Imagick(); + $probe->setResolution(200, 200); + $probe->readImageBlob($pdfContent); + return ['count' => $probe->getNumberImages(), 'image' => $probe]; + } + /** + * @return \Generator + */ + private function imagickPdfToJpegBase64(Imagick $im, int $maxPages = 100): \Generator { + $pageCount = 0; + foreach ($im as $page) { + if ($pageCount >= $maxPages) { + break; + } + $page = $page->getImage(); + $page->setImageBackgroundColor('white'); + $page = $page->flattenImages(); + $page->setImageFormat('jpeg'); + $page->setImageCompressionQuality(85); + yield base64_encode($page->getImageBlob()); + $page->clear(); + $pageCount++; + } + $im->clear(); + } } From fa9dd37e50492a1330f1c7aeefda0f7d1236806a Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 6 Jul 2026 18:23:51 -0400 Subject: [PATCH 3/5] Add batching to requests Signed-off-by: Lukas Schaefer --- lib/TaskProcessing/ImageToTextOcrProvider.php | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php index 3bc91f46..d21f5197 100644 --- a/lib/TaskProcessing/ImageToTextOcrProvider.php +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -143,23 +143,29 @@ public function process(?string $userId, array $input, callable $reportProgress) $fileType = $file->getMimeType(); + // handle pdf files by converting to jpeg and then passing to the api if ($fileType === 'application/pdf') { $outputForFile = ''; $imagickProbe = $this->getImagickProbe($file); $pagesRead = 0; - foreach ($this->imagickPdfToJpegBase64($imagickProbe['image']) as $base64Image) { - $pagesRead++; - $history = [json_encode([ - 'role' => 'user', - 'content' => [ - [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => 'data:image/jpeg;base64,' . $base64Image, + $history = []; + foreach ($this->imagickPdfToJpegBase64($imagickProbe['image'], 5) as $base64Image) { + $pagesRead += count($base64Image); + $history = []; + // Batches 5 in one request to speed up the process + foreach ($base64Image as $image) { + $history[] = json_encode([ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:image/jpeg;base64,' . $image, + ], ], ], - ], - ])]; + ]); + } try { $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); $messages = $completion['messages']; @@ -249,10 +255,11 @@ private function getImagickProbe(File $file): array { return ['count' => $probe->getNumberImages(), 'image' => $probe]; } /** - * @return \Generator + * @return \Generator> */ - private function imagickPdfToJpegBase64(Imagick $im, int $maxPages = 100): \Generator { + private function imagickPdfToJpegBase64(Imagick $im, int $batchSize, int $maxPages = 500): \Generator { $pageCount = 0; + $batch = []; foreach ($im as $page) { if ($pageCount >= $maxPages) { break; @@ -262,9 +269,17 @@ private function imagickPdfToJpegBase64(Imagick $im, int $maxPages = 100): \Gene $page = $page->flattenImages(); $page->setImageFormat('jpeg'); $page->setImageCompressionQuality(85); - yield base64_encode($page->getImageBlob()); + $batch[] = base64_encode($page->getImageBlob()); $page->clear(); $pageCount++; + // Read in batchs to avoid memory issues + if (count($batch) >= $batchSize) { + yield $batch; + $batch = []; + } + } + if (count($batch) > 0) { + yield $batch; } $im->clear(); } From 6950b70037c97d6f45794f76a8401a5438d07071 Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Mon, 20 Jul 2026 12:36:18 +0200 Subject: [PATCH 4/5] add streaming support to ocr provider Signed-off-by: Julien Veyssier --- lib/TaskProcessing/ImageToTextOcrProvider.php | 86 +++++++++++++++++-- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php index d21f5197..7b91157a 100644 --- a/lib/TaskProcessing/ImageToTextOcrProvider.php +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -16,14 +16,17 @@ use OCP\Files\File; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; +use OCP\TaskProcessing\Exception\ProcessingException; use OCP\TaskProcessing\Exception\UserFacingProcessingException; -use OCP\TaskProcessing\ISynchronousProvider; +use OCP\TaskProcessing\IProvider; +use OCP\TaskProcessing\ISynchronousOptionsAwareProvider; use OCP\TaskProcessing\ShapeDescriptor; +use OCP\TaskProcessing\SynchronousProviderOptions; use OCP\TaskProcessing\TaskTypes\ImageToTextOpticalCharacterRecognition; use Psr\Log\LoggerInterface; use RuntimeException; -class ImageToTextOcrProvider implements ISynchronousProvider { +class ImageToTextOcrProvider implements IProvider, ISynchronousOptionsAwareProvider { public function __construct( private OpenAiAPIService $openAiAPIService, @@ -99,7 +102,12 @@ public function getOptionalOutputShapeEnumValues(): array { return []; } - public function process(?string $userId, array $input, callable $reportProgress): array { + public function process( + ?string $userId, array $input, callable $reportProgress, SynchronousProviderOptions $options = new SynchronousProviderOptions(), + ): array { + $reportOutput = $options->getReportIntermediateOutput(); + $preferStreaming = $options->getPreferStreaming(); + if (!$this->openAiAPIService->isUsingOpenAi() && !$this->openAiSettingsService->getChatEndpointEnabled()) { throw new RuntimeException('Must support chat completion endpoint'); } @@ -127,6 +135,7 @@ public function process(?string $userId, array $input, callable $reportProgress) $fileSize = 0; $outputs = []; + $streamedOutputs = []; $fileCount = (float)count($input['input']); $systemPrompt = 'Extract all visible text from the image. Return only the extracted text without additional commentary. Preserve the original language of the text.'; $userPrompt = 'Extract all text from this image.'; @@ -140,6 +149,7 @@ public function process(?string $userId, array $input, callable $reportProgress) if ($fileSize > 50 * 1000 * 1000) { throw new UserFacingProcessingException('Filesize of input files too large. Max is 50MB', userFacingMessage: $this->l->t('Filesize of input files too large. Max is 50MB')); } + $streamedOutputs[] = ''; $fileType = $file->getMimeType(); @@ -167,8 +177,39 @@ public function process(?string $userId, array $input, callable $reportProgress) ]); } try { - $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); - $messages = $completion['messages']; + if ($preferStreaming) { + $chunks = $this->openAiAPIService->createStreamedChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $time = microtime(true); + foreach ($chunks as $chunk) { + if (!in_array($chunk['kind'] ?? null, ['content'], true)) { + continue; + } + $streamedOutputs[$fileIndex] .= $chunk['text']; + // we don't report more often than every 250ms + if (microtime(true) - $time >= 0.25) { + $running = $reportOutput([ + 'output' => $streamedOutputs, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + $time = microtime(true); + } + } + if ($streamedOutputs[$fileIndex] !== '') { + $running = $reportOutput([ + 'output' => $streamedOutputs, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + } + $returnValue = $chunks->getReturn(); + $messages = $returnValue['messages']; + } else { + $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $messages = $completion['messages']; + } if (count($messages) === 0) { $this->logger->warning('No result in OpenAI/LocalAI response.'); @@ -213,8 +254,39 @@ public function process(?string $userId, array $input, callable $reportProgress) ]), ]; try { - $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); - $messages = $completion['messages']; + if ($preferStreaming) { + $chunks = $this->openAiAPIService->createStreamedChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $time = microtime(true); + foreach ($chunks as $chunk) { + if (!in_array($chunk['kind'] ?? null, ['content'], true)) { + continue; + } + $streamedOutputs[$fileIndex] .= $chunk['text']; + // we don't report more often than every 250ms + if (microtime(true) - $time >= 0.25) { + $running = $reportOutput([ + 'output' => $streamedOutputs, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + $time = microtime(true); + } + } + if ($streamedOutputs[$fileIndex] !== '') { + $running = $reportOutput([ + 'output' => $streamedOutputs, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + } + $returnValue = $chunks->getReturn(); + $messages = $returnValue['messages']; + } else { + $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); + $messages = $completion['messages']; + } $reportProgress(((float)$fileIndex + 1.0) / $fileCount); if (count($messages) === 0) { From 4af658985123902a38787c8ba91559e24e44e946 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Fri, 24 Jul 2026 12:06:33 -0400 Subject: [PATCH 5/5] Use OpenAIFileService for ocr provider too Signed-off-by: Lukas Schaefer --- lib/TaskProcessing/ImageToTextOcrProvider.php | 272 +++++------------- 1 file changed, 65 insertions(+), 207 deletions(-) diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php index 7b91157a..d7113640 100644 --- a/lib/TaskProcessing/ImageToTextOcrProvider.php +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -9,11 +9,9 @@ namespace OCA\OpenAi\TaskProcessing; -use Imagick; use OCA\OpenAi\AppInfo\Application; use OCA\OpenAi\Service\OpenAiAPIService; use OCA\OpenAi\Service\OpenAiSettingsService; -use OCP\Files\File; use OCP\IL10N; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Exception\ProcessingException; @@ -24,7 +22,6 @@ use OCP\TaskProcessing\SynchronousProviderOptions; use OCP\TaskProcessing\TaskTypes\ImageToTextOpticalCharacterRecognition; use Psr\Log\LoggerInterface; -use RuntimeException; class ImageToTextOcrProvider implements IProvider, ISynchronousOptionsAwareProvider { @@ -109,19 +106,18 @@ public function process( $preferStreaming = $options->getPreferStreaming(); if (!$this->openAiAPIService->isUsingOpenAi() && !$this->openAiSettingsService->getChatEndpointEnabled()) { - throw new RuntimeException('Must support chat completion endpoint'); + throw new ProcessingException('Must support chat completion endpoint'); } if (!isset($input['input']) || !is_array($input['input'])) { - throw new RuntimeException('Invalid file list'); + throw new ProcessingException('Invalid file list'); } if (count($input['input']) === 0) { - throw new RuntimeException('Invalid file list'); - } - if (count($input['input']) > 500) { - throw new RuntimeException('Too many files given. Max is 500'); + throw new ProcessingException('Invalid file list'); } + $files = $input['input']; + if (isset($input['model']) && is_string($input['model'])) { $model = $input['model']; } else { @@ -133,226 +129,88 @@ public function process( $maxTokens = $input['max_tokens']; } - $fileSize = 0; + $fileSizeTotal = array_reduce( + $files, + function ($carry, $file) { + return $carry + (method_exists($file, 'getSize') ? $file->getSize() : 0); + }, + 0 + ); + if ($fileSizeTotal > 50 * 1000 * 1000) { + throw new UserFacingProcessingException( + 'Filesize of input files too large. Max is 50MB', + 0, + null, + $this->l->t('The total size of the input files is too large. A maximum of 50MB is allowed.'), + ); + } + $outputs = []; $streamedOutputs = []; - $fileCount = (float)count($input['input']); - $systemPrompt = 'Extract all visible text from the image. Return only the extracted text without additional commentary. Preserve the original language of the text.'; - $userPrompt = 'Extract all text from this image.'; + $fileCount = (float)count($files); + $systemPrompt = 'Extract all visible text from the file. Return only the extracted text without additional commentary. Preserve the original language of the text.'; + $userPrompt = 'Extract all text from this file.'; $fileIndex = 0; - foreach ($input['input'] as $file) { - if (!$file instanceof File || !$file->isReadable()) { - throw new RuntimeException('Invalid input file'); - } - $fileSize += intval($file->getSize()); - if ($fileSize > 50 * 1000 * 1000) { - throw new UserFacingProcessingException('Filesize of input files too large. Max is 50MB', userFacingMessage: $this->l->t('Filesize of input files too large. Max is 50MB')); - } + foreach ($files as $file) { $streamedOutputs[] = ''; - - $fileType = $file->getMimeType(); - - // handle pdf files by converting to jpeg and then passing to the api - if ($fileType === 'application/pdf') { - $outputForFile = ''; - $imagickProbe = $this->getImagickProbe($file); - $pagesRead = 0; - $history = []; - foreach ($this->imagickPdfToJpegBase64($imagickProbe['image'], 5) as $base64Image) { - $pagesRead += count($base64Image); - $history = []; - // Batches 5 in one request to speed up the process - foreach ($base64Image as $image) { - $history[] = json_encode([ - 'role' => 'user', - 'content' => [ - [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => 'data:image/jpeg;base64,' . $image, - ], - ], - ], - ]); - } - try { - if ($preferStreaming) { - $chunks = $this->openAiAPIService->createStreamedChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); - $time = microtime(true); - foreach ($chunks as $chunk) { - if (!in_array($chunk['kind'] ?? null, ['content'], true)) { - continue; - } - $streamedOutputs[$fileIndex] .= $chunk['text']; - // we don't report more often than every 250ms - if (microtime(true) - $time >= 0.25) { - $running = $reportOutput([ - 'output' => $streamedOutputs, - ]); - if (!$running) { - throw new ProcessingException('OpenAI/LocalAI task cancelled'); - } - $time = microtime(true); - } - } - if ($streamedOutputs[$fileIndex] !== '') { - $running = $reportOutput([ - 'output' => $streamedOutputs, - ]); - if (!$running) { - throw new ProcessingException('OpenAI/LocalAI task cancelled'); - } - } - $returnValue = $chunks->getReturn(); - $messages = $returnValue['messages']; - } else { - $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); - $messages = $completion['messages']; - } - - if (count($messages) === 0) { - $this->logger->warning('No result in OpenAI/LocalAI response.'); - $outputs[] = ''; + try { + if ($preferStreaming) { + $chunks = $this->openAiAPIService->createStreamedChatCompletion( + $userId, $model, $userPrompt, $systemPrompt, null, 1, $maxTokens, null, null, null, [$file], + ); + $time = microtime(true); + foreach ($chunks as $chunk) { + if (!in_array($chunk['kind'] ?? null, ['content'], true)) { continue; } - - $outputForFile .= array_pop($messages) . "\n\n"; - $reportProgress(((float)$fileIndex + (float)($pagesRead / $imagickProbe['count'])) / $fileCount); - } catch (\Exception $e) { - $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); - throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); - } - } - $outputs[] = $outputForFile; - $reportProgress(((float)$fileIndex + 1.0) / $fileCount); - } elseif (str_starts_with($fileType, 'image/')) { - if ($this->openAiAPIService->isUsingOpenAi()) { - $validFileTypes = [ - 'image/jpeg', - 'image/png', - 'image/gif', - 'image/webp', - ]; - if (!in_array($fileType, $validFileTypes, true)) { - throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType); - } - } - - $base64Image = base64_encode(stream_get_contents($file->fopen('rb'))); - $history = [ - json_encode([ - 'role' => 'user', - 'content' => [ - [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => 'data:' . $fileType . ';base64,' . $base64Image, - ], - ], - ], - ]), - ]; - try { - if ($preferStreaming) { - $chunks = $this->openAiAPIService->createStreamedChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); - $time = microtime(true); - foreach ($chunks as $chunk) { - if (!in_array($chunk['kind'] ?? null, ['content'], true)) { - continue; - } - $streamedOutputs[$fileIndex] .= $chunk['text']; - // we don't report more often than every 250ms - if (microtime(true) - $time >= 0.25) { - $running = $reportOutput([ - 'output' => $streamedOutputs, - ]); - if (!$running) { - throw new ProcessingException('OpenAI/LocalAI task cancelled'); - } - $time = microtime(true); - } - } - if ($streamedOutputs[$fileIndex] !== '') { + $streamedOutputs[$fileIndex] .= $chunk['text']; + // we don't report more often than every 250ms + if (microtime(true) - $time >= 0.25) { $running = $reportOutput([ 'output' => $streamedOutputs, ]); if (!$running) { throw new ProcessingException('OpenAI/LocalAI task cancelled'); } + $time = microtime(true); } - $returnValue = $chunks->getReturn(); - $messages = $returnValue['messages']; - } else { - $completion = $this->openAiAPIService->createChatCompletion($userId, $model, $userPrompt, $systemPrompt, $history, 1, $maxTokens); - $messages = $completion['messages']; } - $reportProgress(((float)$fileIndex + 1.0) / $fileCount); - - if (count($messages) === 0) { - $this->logger->warning('No result in OpenAI/LocalAI response.'); - $outputs[] = ''; - continue; + if ($streamedOutputs[$fileIndex] !== '') { + $running = $reportOutput([ + 'output' => $streamedOutputs, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } } + $returnValue = $chunks->getReturn(); + $messages = $returnValue['messages']; + } else { + $completion = $this->openAiAPIService->createChatCompletion( + $userId, $model, $userPrompt, $systemPrompt, null, 1, $maxTokens, null, null, null, [$file], + ); + $messages = $completion['messages']; + } + $reportProgress(((float)$fileIndex + 1.0) / $fileCount); - $outputs[] = array_pop($messages); - } catch (\Exception $e) { - $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); - throw new RuntimeException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + if (count($messages) === 0) { + $this->logger->warning('No result in OpenAI/LocalAI response.'); + $outputs[] = ''; + $fileIndex++; + continue; } - } else { - throw new UserFacingProcessingException('Only supports image and pdf file types' . $fileType, userFacingMessage: $this->l->t('Only supports image and pdf file types')); + + $outputs[] = array_pop($messages); + } catch (UserFacingProcessingException|ProcessingException $e) { + throw $e; + } catch (\Throwable $e) { + $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new ProcessingException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); } $fileIndex++; } return ['output' => $outputs]; } - - /** - * @return array{count: int, image: Imagick} - */ - private function getImagickProbe(File $file): array { - if (!extension_loaded('imagick')) { - throw new RuntimeException('Imagick extension not available can not process PDF'); - } - if (empty(Imagick::queryFormats('PDF'))) { - throw new RuntimeException('Imagick has no PDF support (Ghostscript missing or blocked by policy.xml)'); - } - - $pdfContent = $file->getContent(); - $probe = new Imagick(); - $probe->setResolution(200, 200); - $probe->readImageBlob($pdfContent); - return ['count' => $probe->getNumberImages(), 'image' => $probe]; - } - /** - * @return \Generator> - */ - private function imagickPdfToJpegBase64(Imagick $im, int $batchSize, int $maxPages = 500): \Generator { - $pageCount = 0; - $batch = []; - foreach ($im as $page) { - if ($pageCount >= $maxPages) { - break; - } - $page = $page->getImage(); - $page->setImageBackgroundColor('white'); - $page = $page->flattenImages(); - $page->setImageFormat('jpeg'); - $page->setImageCompressionQuality(85); - $batch[] = base64_encode($page->getImageBlob()); - $page->clear(); - $pageCount++; - // Read in batchs to avoid memory issues - if (count($batch) >= $batchSize) { - yield $batch; - $batch = []; - } - } - if (count($batch) > 0) { - yield $batch; - } - $im->clear(); - } }