Add audio translation task type and provider

composer.json

@@ -48,7 +48,7 @@
 		"optimize-autoloader": true,
 		"classmap-authoritative": false,
 		"platform": {
-			"php": "8.2"
+			"php": "8.3"
 		}
 	},
 	"extra": {

lib/AppInfo/Application.php

@@ -11,6 +11,8 @@
 use OCA\OpenAi\Notification\Notifier;
 use OCA\OpenAi\OldProcessing\Translation\TranslationProvider as OldTranslationProvider;
 use OCA\OpenAi\TaskProcessing\AudioToAudioChatProvider;
+use OCA\OpenAi\TaskProcessing\AudioToAudioTranslateProvider;
+use OCA\OpenAi\TaskProcessing\AudioToAudioTranslateTaskType;
 use OCA\OpenAi\TaskProcessing\AudioToTextEnhancedProvider;
 use OCA\OpenAi\TaskProcessing\AudioToTextProvider;
 use OCA\OpenAi\TaskProcessing\ChangeToneProvider;
@@ -29,7 +31,6 @@
 use OCA\OpenAi\TaskProcessing\TranslateProvider;
 use OCP\AppFramework\App;
 use OCP\AppFramework\Bootstrap\IBootContext;
-
 use OCP\AppFramework\Bootstrap\IBootstrap;
 use OCP\AppFramework\Bootstrap\IRegistrationContext;
 use OCP\IAppConfig;
@@ -75,7 +76,6 @@ class Application extends App implements IBootstrap {
 		self::QUOTA_TYPE_IMAGE => 0, // 0 = unlimited
 		self::QUOTA_TYPE_TRANSCRIPTION => 0, // 0 = unlimited
 		self::QUOTA_TYPE_SPEECH => 0, // 0 = unlimited
-
 	];
 
 	public const MODELS_CACHE_KEY = 'models';
@@ -103,11 +103,19 @@ public function register(IRegistrationContext $context): void {
 			$context->registerTranslationProvider(OldTranslationProvider::class);
 		}
 
+		$translationProviderEnabled = $this->appConfig->getValueString(Application::APP_ID, 'translation_provider_enabled', '1') === '1';
+		$sttProviderEnabled = $this->appConfig->getValueString(Application::APP_ID, 'stt_provider_enabled', '1') === '1';
+		$ttsProviderEnabled = $this->appConfig->getValueString(Application::APP_ID, 'tts_provider_enabled', '1') === '1';
+
 		// Task processing
-		if ($this->appConfig->getValueString(Application::APP_ID, 'translation_provider_enabled', '1') === '1') {
+		if ($translationProviderEnabled) {
 			$context->registerTaskProcessingProvider(TranslateProvider::class);
 		}
-		if ($this->appConfig->getValueString(Application::APP_ID, 'stt_provider_enabled', '1') === '1') {
+		if ($translationProviderEnabled && $sttProviderEnabled && $ttsProviderEnabled) {
+			$context->registerTaskProcessingTaskType(AudioToAudioTranslateTaskType::class);
+			$context->registerTaskProcessingProvider(AudioToAudioTranslateProvider::class);
+		}
+		if ($sttProviderEnabled) {
 			$context->registerTaskProcessingProvider(AudioToTextProvider::class);
 			if (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToTextReformatParagraphs')) {
 				$context->registerTaskProcessingProvider(AudioToTextEnhancedProvider::class);

lib/Controller/ConfigController.php

@@ -13,7 +13,6 @@
 use OCP\AppFramework\Controller;
 use OCP\AppFramework\Http;
 use OCP\AppFramework\Http\Attribute\NoAdminRequired;
-
 use OCP\AppFramework\Http\Attribute\PasswordConfirmationRequired;
 use OCP\AppFramework\Http\DataResponse;
 use OCP\IRequest;

lib/Db/QuotaRuleMapper.php

@@ -64,7 +64,6 @@ public function getRule(int $quotaType, string $userId, array $groups): QuotaRul
 						$qb->expr()->eq('u.entity_type', $qb->createNamedParameter(EntityType::GROUP->value, IQueryBuilder::PARAM_INT)),
 						$qb->expr()->in('u.entity_id', $qb->createNamedParameter($groups, IQueryBuilder::PARAM_STR_ARRAY))
 					),
-
 				)
 			)->orderBy('r.priority', 'ASC')
 			->setMaxResults(1);

lib/Migration/Version030102Date20241003155512.php

@@ -6,6 +6,7 @@
  * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
+
 namespace OCA\OpenAi\Migration;
 
 use Closure;

lib/Migration/Version030103Date20241009172829.php

@@ -6,6 +6,7 @@
  * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
+
 namespace OCA\OpenAi\Migration;
 
 use Closure;

lib/Notification/Notifier.php

@@ -12,7 +12,6 @@
 use OCP\L10N\IFactory;
 use OCP\Notification\IAction;
 use OCP\Notification\INotification;
-
 use OCP\Notification\INotifier;
 use OCP\Notification\UnknownNotificationException;
 

lib/Service/OpenAiSettingsService.php

@@ -81,7 +81,6 @@ class OpenAiSettingsService {
 		'stt_language' => 'string',
 	];
 
-
 	public function __construct(
 		private IConfig $config,
 		private IAppConfig $appConfig,
@@ -633,7 +632,6 @@ public function getUserConfig(string $userId): array {
 			'use_basic_auth' => $this->getUseBasicAuth(),
 			'is_custom_service' => $isCustomService,
 			'stt_language' => $this->getUserSTTLanguage($userId)
-
 		];
 	}
 

lib/Service/TranslateService.php

@@ -10,8 +10,42 @@
 namespace OCA\OpenAi\Service;
 
 use OCA\OpenAi\AppInfo\Application;
+use OCP\ICacheFactory;
+use Psr\Log\LoggerInterface;
 
 class TranslateService {
+	public const SYSTEM_PROMPT = 'You are a translations expert that ONLY outputs a valid JSON with the translated text in the following format: { "translation": "<translated text>" } .';
+	public const JSON_RESPONSE_FORMAT = [
+		'response_format' => [
+			'type' => 'json_schema',
+			'json_schema' => [
+				'name' => 'TranslationResponse',
+				'description' => 'A JSON object containing the translated text',
+				'strict' => true,
+				'schema' => [
+					'type' => 'object',
+					'properties' => [
+						'translation' => [
+							'type' => 'string',
+							'description' => 'The translated text',
+						],
+					],
+					'required' => ['translation'],
+					'additionalProperties' => false,
+				],
+			],
+		],
+	];
+
+	public function __construct(
+		private OpenAiSettingsService $openAiSettingsService,
+		private LoggerInterface $logger,
+		private OpenAiAPIService $openAiAPIService,
+		private ChunkService $chunkService,
+		private ICacheFactory $cacheFactory,
+	) {
+	}
+
 	/**
 	 * @return array<array{code: string, name: string}>
 	 */
@@ -30,4 +64,82 @@ public static function getStaticLanguages(): array {
 	public static function getCoreLanguagesByCode(): array {
 		return array_column(Application::LANGUAGE_CODES_AND_ENDONYMS, 1, 0);
 	}
+
+	public function translate(
+		string $inputText, string $sourceLanguageCode, string $targetLanguageCode, string $model, ?int $maxTokens,
+		?string $userId, ?callable $reportProgress = null, bool $preferStreaming = false, ?callable $reportOutput = null,
+	): string {
+		$chunks = $this->chunkService->chunkSplitPrompt($inputText, true, $maxTokens);
+		$translation = '';
+		$increase = 1.0 / (float)count($chunks);
+		$progress = 0.0;
+
+		$coreLanguages = self::getCoreLanguagesByCode();
+
+		$fromLanguage = $sourceLanguageCode;
+		$toLanguage = $coreLanguages[$targetLanguageCode] ?? $targetLanguageCode;
+
+		if ($sourceLanguageCode !== 'detect_language') {
+			$fromLanguage = $coreLanguages[$sourceLanguageCode] ?? $sourceLanguageCode;
+			$promptStart = 'Translate the following text from ' . $fromLanguage . ' to ' . $toLanguage . ': ';
+		} else {
+			$promptStart = 'Translate the following text to ' . $toLanguage . ': ';
+		}
+
+		$cache = $this->cacheFactory->createDistributed('integration_openai');
+		foreach ($chunks as $chunk) {
+			$progress += $increase;
+			$cacheKey = $sourceLanguageCode . '/' . $targetLanguageCode . '/' . md5($chunk);
+
+			if ($cached = $cache->get($cacheKey)) {
+				$this->logger->debug('Using cached translation', ['cached' => $cached, 'cacheKey' => $cacheKey]);
+				$translation .= $cached;
+				if ($reportProgress !== null) {
+					$reportProgress($progress);
+				}
+				if ($preferStreaming && $reportOutput !== null) {
+					$reportOutput($translation);
+				}
+				continue;
+			}
+			$prompt = $promptStart . PHP_EOL . PHP_EOL . $chunk;
+
+			if ($this->openAiAPIService->isUsingOpenAi() || $this->openAiSettingsService->getChatEndpointEnabled()) {
+				$completionsObj = $this->openAiAPIService->createChatCompletion(
+					$userId, $model, $prompt, TranslateService::SYSTEM_PROMPT, null, 1, $maxTokens, TranslateService::JSON_RESPONSE_FORMAT
+				);
+				$completions = $completionsObj['messages'];
+			} else {
+				$completions = $this->openAiAPIService->createCompletion(
+					$userId, $prompt . PHP_EOL . TranslateService::SYSTEM_PROMPT . PHP_EOL . PHP_EOL, 1, $model, $maxTokens
+				);
+			}
+
+			if ($reportProgress !== null) {
+				$reportProgress($progress);
+			}
+
+			if (count($completions) === 0) {
+				$this->logger->error('Empty translation response received for chunk');
+				continue;
+			}
+
+			$completion = array_pop($completions);
+			$decodedCompletion = json_decode($completion, true);
+			if (
+				!isset($decodedCompletion['translation'])
+				|| !is_string($decodedCompletion['translation'])
+				|| empty($decodedCompletion['translation'])
+			) {
+				$this->logger->error('Invalid translation response received for chunk', ['response' => $completion]);
+				continue;
+			}
+			$translation .= $decodedCompletion['translation'];
+			if ($preferStreaming && $reportOutput !== null) {
+				$reportOutput($translation);
+			}
+			$cache->set($cacheKey, $decodedCompletion['translation']);
+		}
+		return $translation;
+	}
 }

lib/Service/WatermarkingService.php

@@ -123,7 +123,7 @@ public function markAudio(string $audio): string {
 
 			return $newAudio;
 		} catch (\Throwable $e) {
-			$this->logger->warning('Could not add AI watermark to AI generated image', ['exception' => $e]);
+			$this->logger->warning('Could not add AI watermark to AI generated audio', ['exception' => $e]);
 			return $audio;
 		}
 	}

lib/TaskProcessing/AnalyzeImagesProvider.php

@@ -60,7 +60,6 @@ public function getInputShapeDefaults(): array {
 		return [];
 	}
 
-
 	public function getOptionalInputShape(): array {
 		return [
 			'max_tokens' => new ShapeDescriptor(
@@ -160,7 +159,6 @@ public function process(
 			]);
 		}
 
-
 		if (!isset($input['input']) || !is_string($input['input'])) {
 			throw new RuntimeException('Invalid prompt');
 		}

lib/TaskProcessing/AudioToAudioChatProvider.php

@@ -69,7 +69,6 @@ public function getInputShapeDefaults(): array {
 		return [];
 	}
 
-
 	public function getOptionalInputShape(): array {
 		$isUsingOpenAi = $this->openAiAPIService->isUsingOpenAi();
 		$ois = [
@@ -199,7 +198,6 @@ public function process(?string $userId, array $input, callable $reportProgress)
 				: $this->openAiSettingsService->getAdminDefaultCompletionModelId();
 		}
 
-
 		if (isset($input['voice']) && is_string($input['voice'])) {
 			$outputVoice = $input['voice'];
 		} else {