feat: support multimodal chat

lib/Listener/BeforeTemplateRenderedListener.php

@@ -76,6 +76,11 @@ public function handle(Event $event): void {
 			$this->initialStateService->provideInitialState('autoplay_audio_chat', $autoplayAudioChat);
 			$agencyAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes());
 			$this->initialStateService->provideInitialState('agency_available', $agencyAvailable);
+
+			$multimodalChatAvailable = $agencyAvailable
+				? (class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes()))
+				: (class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypes()));
+			$this->initialStateService->provideInitialState('multimodal_chat_available', $multimodalChatAvailable);
 		}
 		if (class_exists(\OCA\Viewer\Event\LoadViewer::class)) {
 			$this->eventDispatcher->dispatchTyped(new \OCA\Viewer\Event\LoadViewer());

lib/Listener/ChattyLLMTaskListener.php

@@ -82,6 +82,10 @@ public function handle(Event $event): void {
 				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\AudioToAudioChat::ID;
 			$isAgencyAudioChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentAudioInteraction')
 				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID;
+			$isMultimodalChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')
+				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID;
+			$isMultimodalAgencyChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction')
+				&& $taskTypeId === \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID;
 
 			$taskOutput = $task->getOutput();
 
@@ -128,6 +132,17 @@ public function handle(Event $event): void {
 				// the task is not an audio one, but we might still need to Tts the answer
 				// if it is a response to a ContextAgentInteraction confirmation that was asked about an audio message
 				$this->runTtsIfNeeded($sessionId, $message, $taskTypeId, $task->getUserId());
+				if ($isMultimodalChat || $isMultimodalAgencyChat) {
+					$attachments = $taskOutput['output_attachments'] ?? [];
+					$attachments = array_map(function ($attachment) use ($task) {
+						return [
+							'type' => 'File',
+							'file_id' => $attachment,
+							'ocp_task_id' => $task->getId(),
+						];
+					}, $attachments);
+					$message->setAttachments(json_encode($attachments));
+				}
 			}
 			try {
 				$this->messageMapper->insert($message);
@@ -138,7 +153,7 @@ public function handle(Event $event): void {
 			$session = $this->sessionMapper->getUserSession($task->getUserId(), $sessionId);
 
 			// store the conversation token and the actions if we are using the agency feature
-			if ($isAgency || $isAgencyAudioChat) {
+			if ($isAgency || $isAgencyAudioChat || $isMultimodalAgencyChat) {
 				$conversationToken = ($taskOutput['conversation_token'] ?? null) ?: null;
 				$pendingActions = ($taskOutput['actions'] ?? null) ?: null;
 				$session->setAgencyConversationToken($conversationToken);

lib/Service/AssistantService.php

@@ -772,7 +772,7 @@ private function extractFileIdsFromTask(Task $task): array {
 				/** @var int|list<int> $inputSlot */
 				$inputSlot = $task->getInput()[$key];
 				if (is_array($inputSlot)) {
-					$ids += $inputSlot;
+					$ids = array_merge($ids, $inputSlot);
 				} else {
 					$ids[] = $inputSlot;
 				}
@@ -784,14 +784,14 @@ private function extractFileIdsFromTask(Task $task): array {
 					/** @var int|list<int> $outputSlot */
 					$outputSlot = $task->getOutput()[$key];
 					if (is_array($outputSlot)) {
-						$ids += $outputSlot;
+						$ids = array_merge($ids, $outputSlot);
 					} else {
 						$ids[] = $outputSlot;
 					}
 				}
 			}
 		}
-		return array_values($ids);
+		return $ids;
 	}
 
 	/**

lib/Service/ChatService.php

@@ -40,6 +40,7 @@ public function __construct(
 		private readonly IManager $taskProcessingManager,
 		private readonly LoggerInterface $logger,
 		private readonly ITimeFactory $timeFactory,
+		private readonly AssistantService $assistantService,
 	) {
 	}
 
@@ -399,6 +400,13 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
 				// audio agency
 				$fileId = $audioAttachment['file_id'];
 				$taskId = $this->scheduleAgencyAudioTask($userId, $fileId, $agencyConfirm, $lastConversationToken, $sessionId, $lastUserMessage->getId());
+			} elseif ($this->isMultimodalContextAgentAvailable()) {
+				// multimodal agency
+				$prompt = $lastUserMessage->getContent();
+				$inputAttachments = array_map(static function (array $attachment) {
+					return $attachment['file_id'];
+				}, $lastAttachments);
+				$taskId = $this->scheduleAgencyMultimodalTask($userId, $prompt, $agencyConfirm, $lastConversationToken, $sessionId, $inputAttachments);
 			} else {
 				// classic agency
 				$prompt = $lastUserMessage->getContent();
@@ -446,14 +454,40 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
 				$fileId = $audioAttachment['file_id'];
 				$taskId = $this->scheduleAudioChatTask($userId, $fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId());
 			} else {
-				// for a text chat task, let's only use text in the history
-				$history = array_map(static function (Message $message) {
-					return json_encode([
-						'role' => $message->getRole(),
-						'content' => $message->getContent(),
-					]);
-				}, $history);
-				$taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $history, $sessionId);
+				if ($this->isMultimodalChatAvailable()) {
+					// for a multimodal chat also attachments need to be added to the history
+					$historyMessages = array_map(static function (Message $message) {
+						$attachments = $message->jsonSerialize()['attachments'];
+						$content = array_map(static function (array $attachment) {
+							$newAttachment = ['type' => 'file', 'file_id' => $attachment['file_id']];
+							if (isset($attachment['ocp_task_id'])) {
+								$newAttachment['ocp_task_id'] = $attachment['ocp_task_id'];
+							}
+							return $newAttachment;
+						}, $attachments);
+						$content[] = [
+							'type' => 'text',
+							'text' => $message->getContent(),
+						];
+						return json_encode([
+							'role' => $message->getRole(),
+							'content' => $content,
+						]);
+					}, $history);
+					$lastAttachments = array_map(static function (array $attachment) {
+						return $attachment['file_id'];
+					}, $lastAttachments);
+					$taskId = $this->scheduleMultimodalChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, $lastAttachments);
+				} else {
+					// for a text chat task only use text in the history
+					$historyMessages = array_map(static function (Message $message) {
+						return json_encode([
+							'role' => $message->getRole(),
+							'content' => $message->getContent(),
+						]);
+					}, $history);
+					$taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId);
+				}
 			}
 		}
 		return $taskId;
@@ -567,6 +601,20 @@ public function isContextAgentAudioAvailable(): bool {
 		return in_array(\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
 	}
 
+	public function isMultimodalChatAvailable(): bool {
+		if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')) {
+			return false;
+		}
+		return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
+	}
+
+	public function isMultimodalContextAgentAvailable(): bool {
+		if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction')) {
+			return false;
+		}
+		return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
+	}
+
 	private function getAudioHistory(array $history): array {
 		// history is a list of JSON strings
 		// The content is the remote audio ID (or the transcription as fallback)
@@ -682,6 +730,49 @@ private function scheduleLLMChatTask(
 		return $task->getId() ?? 0;
 	}
 
+	/**
+	 * Schedule a Multimodal Chat task
+	 *
+	 * @throws BadRequestException
+	 * @throws InternalException
+	 */
+	private function scheduleMultimodalChatTask(
+		?string $userId,
+		string $content,
+		string $systemPrompt,
+		array $history,
+		int $sessionId,
+		array $attachmentsHistory,
+	): int {
+		$customId = 'chatty-llm:' . $sessionId;
+		$this->checkIfSessionIsThinking($userId, $customId);
+		$input = [
+			'input' => $content,
+			'system_prompt' => $systemPrompt,
+			'history' => $history,
+			'input_attachments' => $attachmentsHistory,
+			'tools' => '[]', // Empty tools as there is not a non tools version
+			'tool_message' => '',
+		];
+		/** @psalm-suppress UndefinedClass */
+		$task = new Task(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $input, Application::APP_ID . ':chatty-llm', $userId, $customId);
+		/** @psalm-suppress UndefinedMethod */
+		$task->setPreferStreaming(true);
+		try {
+			$this->taskProcessingManager->scheduleTask($task);
+		} catch (PreConditionNotMetException $e) {
+			throw new BadRequestException('pre_condition_not_met', previous: $e);
+		} catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) {
+			throw new BadRequestException('unauthorized', previous: $e);
+		} catch (ValidationException $e) {
+			throw new BadRequestException('validation_failed', previous: $e);
+		} catch (\OCP\TaskProcessing\Exception\Exception $e) {
+			$this->logger->error($e->getMessage(), ['exception' => $e]);
+			throw new InternalException(previous: $e);
+		}
+		return $task->getId() ?? 0;
+	}
+
 	/**
 	 * Schedule an agency chat task
 	 *
@@ -731,6 +822,58 @@ private function scheduleAgencyTask(
 		return $task->getId() ?? 0;
 	}
 
+	/**
+	 * Schedule a multimodal agency chat task
+	 *
+	 * @param list<int> $inputAttachments
+	 * @throws BadRequestException
+	 * @throws InternalException
+	 */
+	private function scheduleAgencyMultimodalTask(
+		?string $userId,
+		string $content,
+		int $confirmation,
+		string $conversationToken,
+		int $sessionId,
+		array $inputAttachments,
+	): int {
+		$customId = 'chatty-llm:' . $sessionId;
+		$this->checkIfSessionIsThinking($userId, $customId);
+		$taskInput = [
+			'input' => $content,
+			'input_attachments' => $inputAttachments,
+			'confirmation' => $confirmation,
+			'conversation_token' => $conversationToken,
+		];
+		/** @psalm-suppress UndefinedClass */
+		if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID]['optionalInputShape']['memories'])) {
+			$taskInput['memories'] = $this->sessionSummaryService->getMemories($userId);
+		}
+		/** @psalm-suppress UndefinedClass */
+		$task = new Task(
+			\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID,
+			$taskInput,
+			Application::APP_ID . ':chatty-llm',
+			$userId,
+			$customId
+		);
+		/** @psalm-suppress UndefinedMethod */
+		$task->setPreferStreaming(true);
+		try {
+			$this->taskProcessingManager->scheduleTask($task);
+		} catch (PreConditionNotMetException $e) {
+			throw new BadRequestException('pre_condition_not_met', previous: $e);
+		} catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) {
+			throw new BadRequestException('unauthorized', previous: $e);
+		} catch (ValidationException $e) {
+			throw new BadRequestException('validation_failed', previous: $e);
+		} catch (\OCP\TaskProcessing\Exception\Exception $e) {
+			$this->logger->error($e->getMessage(), ['exception' => $e]);
+			throw new InternalException(previous: $e);
+		}
+		return $task->getId() ?? 0;
+	}
+
 	/**
 	 * Schedule an audio chat task
 	 * @throws BadRequestException

src/components/ChattyLLM/ChattyLLMInputForm.vue

@@ -665,16 +665,20 @@ export default {
 			return session.timestamp ? (' ' + moment(session.timestamp * 1000).format('LLL')) : t('assistant', 'Untitled conversation')
 		},
 
-		async handleSubmit(event) {
+		async handleSubmit(attachedFileIds) {
 			if (this.chatContent.trim() === '') {
 				console.debug('empty message')
 				return
 			}
+			const attachments = attachedFileIds.map(fileId => ({ type: SHAPE_TYPE_NAMES.File, file_id: fileId }))
 
 			const role = Roles.HUMAN
 			const content = this.chatContent.trim()
 			const timestamp = +new Date() / 1000 | 0
 			console.debug('[Assistant] submit text', content)
+			if (attachedFileIds.length > 0) {
+				console.debug('[Assistant] submit attachments', attachments)
+			}
 
 			if (this.active === null) {
 				await this.newSession()
@@ -686,10 +690,10 @@ export default {
 				this.active.agencyAnswered = true
 			}
 
-			this.messages.push({ role, content, timestamp, session_id: this.active.id })
+			this.messages.push({ role, content, timestamp, session_id: this.active.id, attachments })
 			this.chatContent = ''
 			this.scrollToLastMessage()
-			await this.newMessage(role, content, timestamp, this.active.id)
+			await this.newMessage(role, content, timestamp, this.active.id, attachments)
 		},
 
 		async handleSubmitAudio(fileId) {

src/components/ChattyLLM/DocumentPreviews.vue

@@ -0,0 +1,82 @@
+<!--
+  - SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+  - SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+<template>
+	<div class="document-previews">
+		<div class="document-previews__list">
+			<div v-for="fileId in fileIds"
+				:key="fileId"
+				class="document-previews__item">
+				<FileDisplay :file-id="fileId"
+					:task-id="null" />
+				<NcButton class="document-previews__delete"
+					variant="tertiary"
+					:aria-label="t('assistant', 'Remove this media')"
+					@click="$emit('delete', fileId)">
+					<template #icon>
+						<TrashCanOutlineIcon :size="20" />
+					</template>
+				</NcButton>
+			</div>
+		</div>
+	</div>
+</template>
+
+<script>
+import TrashCanOutlineIcon from 'vue-material-design-icons/TrashCanOutline.vue'
+
+import NcButton from '@nextcloud/vue/components/NcButton'
+
+import FileDisplay from '../fields/FileDisplay.vue'
+
+export default {
+	name: 'DocumentPreviews',
+
+	components: {
+		FileDisplay,
+		NcButton,
+		TrashCanOutlineIcon,
+	},
+
+	props: {
+		fileIds: {
+			type: Array,
+			required: true,
+		},
+	},
+
+	emits: [
+		'delete',
+	],
+}
+</script>
+
+<style lang="scss" scoped>
+.document-previews {
+	overflow-x: auto;
+
+	&__list {
+		display: flex;
+		flex-direction: row;
+		gap: 8px;
+	}
+
+	&__item {
+		position: relative;
+		padding: 8px;
+		border-radius: var(--border-radius-large);
+		background-color: var(--color-main-background);
+
+		&:hover {
+			background-color: var(--color-primary-element-light-hover);
+		}
+	}
+
+	&__delete {
+		position: absolute;
+		right: 0;
+		bottom: 0;
+	}
+}
+</style>