diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 7854fa4b..27be6312 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -149,6 +149,7 @@ public function register(IRegistrationContext $context): void { $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider::class); } if ($this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1') === '1') { + $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ImageToTextOcrProvider::class); $context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\AnalyzeImagesProvider::class); } } diff --git a/lib/TaskProcessing/ImageToTextOcrProvider.php b/lib/TaskProcessing/ImageToTextOcrProvider.php new file mode 100644 index 00000000..d7113640 --- /dev/null +++ b/lib/TaskProcessing/ImageToTextOcrProvider.php @@ -0,0 +1,216 @@ +openAiAPIService->getServiceName(); + } + + public function getTaskTypeId(): string { + return ImageToTextOpticalCharacterRecognition::ID; + } + + public function getExpectedRuntime(): int { + return $this->openAiAPIService->getExpTextProcessingTime(); + } + + public function getInputShapeEnumValues(): array { + return []; + } + + public function getInputShapeDefaults(): array { + return []; + } + + public function getOptionalInputShape(): array { + return [ + 'max_tokens' => new ShapeDescriptor( + $this->l->t('Maximum output words'), + $this->l->t('The maximum number of words/tokens that can be generated in the output.'), + EShapeType::Number + ), + 'model' => new ShapeDescriptor( + $this->l->t('Model'), + $this->l->t('The model used to generate the output'), + EShapeType::Enum + ), + ]; + } + + public function getOptionalInputShapeEnumValues(): array { + return [ + 'model' => $this->openAiAPIService->getModelEnumValues($this->userId), + ]; + } + + public function getOptionalInputShapeDefaults(): array { + $adminModel = $this->openAiSettingsService->getAdminDefaultCompletionModelId(); + return [ + 'max_tokens' => $this->openAiSettingsService->getMaxTokens(), + 'model' => $adminModel, + ]; + } + + public function getOutputShapeEnumValues(): array { + return []; + } + + public function getOptionalOutputShape(): array { + return []; + } + + public function getOptionalOutputShapeEnumValues(): array { + return []; + } + + public function process( + ?string $userId, array $input, callable $reportProgress, SynchronousProviderOptions $options = new SynchronousProviderOptions(), + ): array { + $reportOutput = $options->getReportIntermediateOutput(); + $preferStreaming = $options->getPreferStreaming(); + + if (!$this->openAiAPIService->isUsingOpenAi() && !$this->openAiSettingsService->getChatEndpointEnabled()) { + throw new ProcessingException('Must support chat completion endpoint'); + } + + if (!isset($input['input']) || !is_array($input['input'])) { + throw new ProcessingException('Invalid file list'); + } + if (count($input['input']) === 0) { + throw new ProcessingException('Invalid file list'); + } + + $files = $input['input']; + + if (isset($input['model']) && is_string($input['model'])) { + $model = $input['model']; + } else { + $model = $this->openAiSettingsService->getAdminDefaultCompletionModelId(); + } + + $maxTokens = null; + if (isset($input['max_tokens']) && is_int($input['max_tokens'])) { + $maxTokens = $input['max_tokens']; + } + + $fileSizeTotal = array_reduce( + $files, + function ($carry, $file) { + return $carry + (method_exists($file, 'getSize') ? $file->getSize() : 0); + }, + 0 + ); + if ($fileSizeTotal > 50 * 1000 * 1000) { + throw new UserFacingProcessingException( + 'Filesize of input files too large. Max is 50MB', + 0, + null, + $this->l->t('The total size of the input files is too large. A maximum of 50MB is allowed.'), + ); + } + + $outputs = []; + $streamedOutputs = []; + $fileCount = (float)count($files); + $systemPrompt = 'Extract all visible text from the file. Return only the extracted text without additional commentary. Preserve the original language of the text.'; + $userPrompt = 'Extract all text from this file.'; + + $fileIndex = 0; + foreach ($files as $file) { + $streamedOutputs[] = ''; + try { + if ($preferStreaming) { + $chunks = $this->openAiAPIService->createStreamedChatCompletion( + $userId, $model, $userPrompt, $systemPrompt, null, 1, $maxTokens, null, null, null, [$file], + ); + $time = microtime(true); + foreach ($chunks as $chunk) { + if (!in_array($chunk['kind'] ?? null, ['content'], true)) { + continue; + } + $streamedOutputs[$fileIndex] .= $chunk['text']; + // we don't report more often than every 250ms + if (microtime(true) - $time >= 0.25) { + $running = $reportOutput([ + 'output' => $streamedOutputs, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + $time = microtime(true); + } + } + if ($streamedOutputs[$fileIndex] !== '') { + $running = $reportOutput([ + 'output' => $streamedOutputs, + ]); + if (!$running) { + throw new ProcessingException('OpenAI/LocalAI task cancelled'); + } + } + $returnValue = $chunks->getReturn(); + $messages = $returnValue['messages']; + } else { + $completion = $this->openAiAPIService->createChatCompletion( + $userId, $model, $userPrompt, $systemPrompt, null, 1, $maxTokens, null, null, null, [$file], + ); + $messages = $completion['messages']; + } + $reportProgress(((float)$fileIndex + 1.0) / $fileCount); + + if (count($messages) === 0) { + $this->logger->warning('No result in OpenAI/LocalAI response.'); + $outputs[] = ''; + $fileIndex++; + continue; + } + + $outputs[] = array_pop($messages); + } catch (UserFacingProcessingException|ProcessingException $e) { + throw $e; + } catch (\Throwable $e) { + $this->logger->warning('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new ProcessingException('OpenAI/LocalAI\'s OCR failed with: ' . $e->getMessage()); + } + $fileIndex++; + } + + return ['output' => $outputs]; + } +}