diff --git a/lib/Listener/BeforeTemplateRenderedListener.php b/lib/Listener/BeforeTemplateRenderedListener.php index 2c4fd6f0..3a4f75bd 100644 --- a/lib/Listener/BeforeTemplateRenderedListener.php +++ b/lib/Listener/BeforeTemplateRenderedListener.php @@ -76,6 +76,11 @@ public function handle(Event $event): void { $this->initialStateService->provideInitialState('autoplay_audio_chat', $autoplayAudioChat); $agencyAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes()); $this->initialStateService->provideInitialState('agency_available', $agencyAvailable); + + $multimodalChatAvailable = $agencyAvailable + ? (class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes())) + : (class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypes())); + $this->initialStateService->provideInitialState('multimodal_chat_available', $multimodalChatAvailable); } if (class_exists(\OCA\Viewer\Event\LoadViewer::class)) { $this->eventDispatcher->dispatchTyped(new \OCA\Viewer\Event\LoadViewer()); diff --git a/lib/Listener/ChattyLLMTaskListener.php b/lib/Listener/ChattyLLMTaskListener.php index 9f860948..388f720d 100644 --- a/lib/Listener/ChattyLLMTaskListener.php +++ b/lib/Listener/ChattyLLMTaskListener.php @@ -82,6 +82,10 @@ public function handle(Event $event): void { && $taskTypeId === \OCP\TaskProcessing\TaskTypes\AudioToAudioChat::ID; $isAgencyAudioChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentAudioInteraction') && $taskTypeId === \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID; + $isMultimodalChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') + && $taskTypeId === \OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID; + $isMultimodalAgencyChat = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction') + && $taskTypeId === \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID; $taskOutput = $task->getOutput(); @@ -128,6 +132,17 @@ public function handle(Event $event): void { // the task is not an audio one, but we might still need to Tts the answer // if it is a response to a ContextAgentInteraction confirmation that was asked about an audio message $this->runTtsIfNeeded($sessionId, $message, $taskTypeId, $task->getUserId()); + if ($isMultimodalChat || $isMultimodalAgencyChat) { + $attachments = $taskOutput['output_attachments'] ?? []; + $attachments = array_map(function ($attachment) use ($task) { + return [ + 'type' => 'File', + 'file_id' => $attachment, + 'ocp_task_id' => $task->getId(), + ]; + }, $attachments); + $message->setAttachments(json_encode($attachments)); + } } try { $this->messageMapper->insert($message); @@ -138,7 +153,7 @@ public function handle(Event $event): void { $session = $this->sessionMapper->getUserSession($task->getUserId(), $sessionId); // store the conversation token and the actions if we are using the agency feature - if ($isAgency || $isAgencyAudioChat) { + if ($isAgency || $isAgencyAudioChat || $isMultimodalAgencyChat) { $conversationToken = ($taskOutput['conversation_token'] ?? null) ?: null; $pendingActions = ($taskOutput['actions'] ?? null) ?: null; $session->setAgencyConversationToken($conversationToken); diff --git a/lib/Service/AssistantService.php b/lib/Service/AssistantService.php index 28330bd1..537827cf 100644 --- a/lib/Service/AssistantService.php +++ b/lib/Service/AssistantService.php @@ -772,7 +772,7 @@ private function extractFileIdsFromTask(Task $task): array { /** @var int|list $inputSlot */ $inputSlot = $task->getInput()[$key]; if (is_array($inputSlot)) { - $ids += $inputSlot; + $ids = array_merge($ids, $inputSlot); } else { $ids[] = $inputSlot; } @@ -784,14 +784,14 @@ private function extractFileIdsFromTask(Task $task): array { /** @var int|list $outputSlot */ $outputSlot = $task->getOutput()[$key]; if (is_array($outputSlot)) { - $ids += $outputSlot; + $ids = array_merge($ids, $outputSlot); } else { $ids[] = $outputSlot; } } } } - return array_values($ids); + return $ids; } /** diff --git a/lib/Service/ChatService.php b/lib/Service/ChatService.php index 0a5986a5..edf656f2 100644 --- a/lib/Service/ChatService.php +++ b/lib/Service/ChatService.php @@ -40,6 +40,7 @@ public function __construct( private readonly IManager $taskProcessingManager, private readonly LoggerInterface $logger, private readonly ITimeFactory $timeFactory, + private readonly AssistantService $assistantService, ) { } @@ -399,6 +400,13 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ // audio agency $fileId = $audioAttachment['file_id']; $taskId = $this->scheduleAgencyAudioTask($userId, $fileId, $agencyConfirm, $lastConversationToken, $sessionId, $lastUserMessage->getId()); + } elseif ($this->isMultimodalContextAgentAvailable()) { + // multimodal agency + $prompt = $lastUserMessage->getContent(); + $inputAttachments = array_map(static function (array $attachment) { + return $attachment['file_id']; + }, $lastAttachments); + $taskId = $this->scheduleAgencyMultimodalTask($userId, $prompt, $agencyConfirm, $lastConversationToken, $sessionId, $inputAttachments); } else { // classic agency $prompt = $lastUserMessage->getContent(); @@ -446,14 +454,40 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ $fileId = $audioAttachment['file_id']; $taskId = $this->scheduleAudioChatTask($userId, $fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId()); } else { - // for a text chat task, let's only use text in the history - $history = array_map(static function (Message $message) { - return json_encode([ - 'role' => $message->getRole(), - 'content' => $message->getContent(), - ]); - }, $history); - $taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $history, $sessionId); + if ($this->isMultimodalChatAvailable()) { + // for a multimodal chat also attachments need to be added to the history + $historyMessages = array_map(static function (Message $message) { + $attachments = $message->jsonSerialize()['attachments']; + $content = array_map(static function (array $attachment) { + $newAttachment = ['type' => 'file', 'file_id' => $attachment['file_id']]; + if (isset($attachment['ocp_task_id'])) { + $newAttachment['ocp_task_id'] = $attachment['ocp_task_id']; + } + return $newAttachment; + }, $attachments); + $content[] = [ + 'type' => 'text', + 'text' => $message->getContent(), + ]; + return json_encode([ + 'role' => $message->getRole(), + 'content' => $content, + ]); + }, $history); + $lastAttachments = array_map(static function (array $attachment) { + return $attachment['file_id']; + }, $lastAttachments); + $taskId = $this->scheduleMultimodalChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, $lastAttachments); + } else { + // for a text chat task only use text in the history + $historyMessages = array_map(static function (Message $message) { + return json_encode([ + 'role' => $message->getRole(), + 'content' => $message->getContent(), + ]); + }, $history); + $taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId); + } } } return $taskId; @@ -567,6 +601,20 @@ public function isContextAgentAudioAvailable(): bool { return in_array(\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds()); } + public function isMultimodalChatAvailable(): bool { + if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')) { + return false; + } + return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypeIds()); + } + + public function isMultimodalContextAgentAvailable(): bool { + if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction')) { + return false; + } + return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds()); + } + private function getAudioHistory(array $history): array { // history is a list of JSON strings // The content is the remote audio ID (or the transcription as fallback) @@ -682,6 +730,49 @@ private function scheduleLLMChatTask( return $task->getId() ?? 0; } + /** + * Schedule a Multimodal Chat task + * + * @throws BadRequestException + * @throws InternalException + */ + private function scheduleMultimodalChatTask( + ?string $userId, + string $content, + string $systemPrompt, + array $history, + int $sessionId, + array $attachmentsHistory, + ): int { + $customId = 'chatty-llm:' . $sessionId; + $this->checkIfSessionIsThinking($userId, $customId); + $input = [ + 'input' => $content, + 'system_prompt' => $systemPrompt, + 'history' => $history, + 'input_attachments' => $attachmentsHistory, + 'tools' => '[]', // Empty tools as there is not a non tools version + 'tool_message' => '', + ]; + /** @psalm-suppress UndefinedClass */ + $task = new Task(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $input, Application::APP_ID . ':chatty-llm', $userId, $customId); + /** @psalm-suppress UndefinedMethod */ + $task->setPreferStreaming(true); + try { + $this->taskProcessingManager->scheduleTask($task); + } catch (PreConditionNotMetException $e) { + throw new BadRequestException('pre_condition_not_met', previous: $e); + } catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) { + throw new BadRequestException('unauthorized', previous: $e); + } catch (ValidationException $e) { + throw new BadRequestException('validation_failed', previous: $e); + } catch (\OCP\TaskProcessing\Exception\Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + throw new InternalException(previous: $e); + } + return $task->getId() ?? 0; + } + /** * Schedule an agency chat task * @@ -731,6 +822,55 @@ private function scheduleAgencyTask( return $task->getId() ?? 0; } + /** + * Schedule a multimodal agency chat task + * + * @param list $inputAttachments + * @throws BadRequestException + * @throws InternalException + */ + private function scheduleAgencyMultimodalTask( + ?string $userId, + string $content, + int $confirmation, + string $conversationToken, + int $sessionId, + array $inputAttachments, + ): int { + $customId = 'chatty-llm:' . $sessionId; + $this->checkIfSessionIsThinking($userId, $customId); + $taskInput = [ + 'input' => $content, + 'input_attachments' => $inputAttachments, + 'confirmation' => $confirmation, + 'conversation_token' => $conversationToken, + ]; + $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); + /** @psalm-suppress UndefinedClass */ + $task = new Task( + \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, + $taskInput, + Application::APP_ID . ':chatty-llm', + $userId, + $customId + ); + /** @psalm-suppress UndefinedMethod */ + $task->setPreferStreaming(true); + try { + $this->taskProcessingManager->scheduleTask($task); + } catch (PreConditionNotMetException $e) { + throw new BadRequestException('pre_condition_not_met', previous: $e); + } catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) { + throw new BadRequestException('unauthorized', previous: $e); + } catch (ValidationException $e) { + throw new BadRequestException('validation_failed', previous: $e); + } catch (\OCP\TaskProcessing\Exception\Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + throw new InternalException(previous: $e); + } + return $task->getId() ?? 0; + } + /** * Schedule an audio chat task * @throws BadRequestException diff --git a/src/components/ChattyLLM/ChattyLLMInputForm.vue b/src/components/ChattyLLM/ChattyLLMInputForm.vue index da2b310c..0d93d451 100644 --- a/src/components/ChattyLLM/ChattyLLMInputForm.vue +++ b/src/components/ChattyLLM/ChattyLLMInputForm.vue @@ -665,16 +665,20 @@ export default { return session.timestamp ? (' ' + moment(session.timestamp * 1000).format('LLL')) : t('assistant', 'Untitled conversation') }, - async handleSubmit(event) { + async handleSubmit(attachedFileIds) { if (this.chatContent.trim() === '') { console.debug('empty message') return } + const attachments = attachedFileIds.map(fileId => ({ type: SHAPE_TYPE_NAMES.File, file_id: fileId })) const role = Roles.HUMAN const content = this.chatContent.trim() const timestamp = +new Date() / 1000 | 0 console.debug('[Assistant] submit text', content) + if (attachedFileIds.length > 0) { + console.debug('[Assistant] submit attachments', attachments) + } if (this.active === null) { await this.newSession() @@ -686,10 +690,10 @@ export default { this.active.agencyAnswered = true } - this.messages.push({ role, content, timestamp, session_id: this.active.id }) + this.messages.push({ role, content, timestamp, session_id: this.active.id, attachments }) this.chatContent = '' this.scrollToLastMessage() - await this.newMessage(role, content, timestamp, this.active.id) + await this.newMessage(role, content, timestamp, this.active.id, attachments) }, async handleSubmitAudio(fileId) { diff --git a/src/components/ChattyLLM/DocumentPreviews.vue b/src/components/ChattyLLM/DocumentPreviews.vue new file mode 100644 index 00000000..06e5192a --- /dev/null +++ b/src/components/ChattyLLM/DocumentPreviews.vue @@ -0,0 +1,82 @@ + + + + + + diff --git a/src/components/ChattyLLM/InputArea.vue b/src/components/ChattyLLM/InputArea.vue index a40c24c4..8855e476 100644 --- a/src/components/ChattyLLM/InputArea.vue +++ b/src/components/ChattyLLM/InputArea.vue @@ -4,53 +4,98 @@ --> @@ -187,11 +306,16 @@ export default { .input-area { display: flex; - flex-direction: row; - justify-content: space-between; - align-items: end; + flex-direction: column; gap: 4px; + &__row { + display: flex; + flex-direction: row; + align-items: end; + gap: 4px; + } + :deep(&__thinking > div) { font-style: italic; animation: breathing 2s linear infinite normal; diff --git a/src/components/ChattyLLM/Message.vue b/src/components/ChattyLLM/Message.vue index fa05f801..a17e5bef 100644 --- a/src/components/ChattyLLM/Message.vue +++ b/src/components/ChattyLLM/Message.vue @@ -102,7 +102,15 @@ :autoplay="message.autoPlay" :file-id="a.file_id" :task-id="message.role === 'human' ? undefined : (a.ocp_task_id ?? message.ocp_task_id)" - :is-output="message.role === 'assistant'" /> + :is-output="isOutput" /> + @@ -126,6 +134,7 @@ import { showSuccess } from '@nextcloud/dialogs' import { generateOcsUrl } from '@nextcloud/router' import axios from '@nextcloud/axios' import { SHAPE_TYPE_NAMES } from '../../constants.js' +import FileDisplay from '../fields/FileDisplay.vue' const PLAIN_URL_PATTERN = /(?:\s|^|\()((?:https?:\/\/)(?:[-A-Z0-9+_.]+(?::[0-9]+)?(?:\/[-A-Z0-9+&@#%?=~_|!:,.;()]*)*))(?:\s|$|\))/ig const MARKDOWN_LINK_PATTERN = /\[[-A-Z0-9+&@#%?=~_|!:,.;()]+\]\(((?:https?:\/\/)(?:[-A-Z0-9+_.]+(?::[0-9]+)?(?:\/[-A-Z0-9+&@#%?=~_|!:,.;]*)*))\)/ig @@ -135,6 +144,7 @@ export default { components: { AudioDisplay, + FileDisplay, AssistantIcon, NcAvatar, @@ -213,6 +223,12 @@ export default { audioAttachments() { return this.message.attachments?.filter(a => a.type === SHAPE_TYPE_NAMES.Audio) ?? [] }, + fileAttachments() { + return this.message.attachments?.filter(a => a.type === SHAPE_TYPE_NAMES.File) ?? [] + }, + isOutput() { + return this.message.role === 'assistant' + }, }, watch: { @@ -273,6 +289,24 @@ export default { } return this.informationSourceNames[source] ? this.informationSourceNames[source] : source }, + onPreviewClick(file) { + // do not open input media files in the viewer + if (file.file_id === null || !this.isOutput) { + return + } + + const url = generateOcsUrl('/apps/assistant/api/v1/task/{taskId}/file/{fileId}/save', { + taskId: file.ocp_task_id ?? this.message.ocp_task_id, + fileId: file.file_id, + }) + return axios.post(url).then(response => { + const savedPath = response.data.ocs.data.path + console.debug('[assistant] view output file', savedPath) + OCA.Viewer.open({ path: savedPath }) + }).catch(error => { + console.error(error) + }) + }, }, } diff --git a/src/constants.js b/src/constants.js index 75034955..d71a33b7 100644 --- a/src/constants.js +++ b/src/constants.js @@ -4,6 +4,7 @@ */ export const MAX_TEXT_INPUT_LENGTH = 64_000 +export const MAX_ATTACHED_FILES = 10 export const TASK_STATUS_INT = { cancelled: 5, diff --git a/tests/psalm-baseline.xml b/tests/psalm-baseline.xml index 14dd255c..4284f4aa 100644 --- a/tests/psalm-baseline.xml +++ b/tests/psalm-baseline.xml @@ -42,4 +42,12 @@ + + + + + + ]]> + +