From 44553564653c1bf8d26ea92aa172b6b741de022c Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 13 Jul 2026 14:29:26 +0200 Subject: [PATCH 1/3] build: add dependency and update CMake configuration for LLM extension --- packages/react-native-executorch/android/CMakeLists.txt | 2 ++ packages/react-native-executorch/package.json | 3 +++ yarn.lock | 8 ++++++++ 3 files changed, 13 insertions(+) diff --git a/packages/react-native-executorch/android/CMakeLists.txt b/packages/react-native-executorch/android/CMakeLists.txt index a14fb9625b..d65862e860 100644 --- a/packages/react-native-executorch/android/CMakeLists.txt +++ b/packages/react-native-executorch/android/CMakeLists.txt @@ -26,6 +26,7 @@ find_package(ReactAndroid REQUIRED CONFIG) file(GLOB CORE_SOURCES ${CPP_DIR}/core/*.cpp) file(GLOB MATH_SOURCES ${CPP_DIR}/extensions/math/*.cpp) file(GLOB NLP_SOURCES ${CPP_DIR}/extensions/nlp/*.cpp) +file(GLOB LLM_SOURCES ${CPP_DIR}/extensions/llm/*.cpp) file(GLOB OPENCV_SOURCES ${CPP_DIR}/extensions/cv/*.cpp) set(RNE_SOURCES @@ -33,6 +34,7 @@ set(RNE_SOURCES ${CORE_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} + ${LLM_SOURCES} cpp-adapter.cpp ) diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json index ad2b4b435e..f8eb1be3d7 100644 --- a/packages/react-native-executorch/package.json +++ b/packages/react-native-executorch/package.json @@ -82,6 +82,9 @@ "publishConfig": { "registry": "https://registry.npmjs.org/" }, + "dependencies": { + "@huggingface/jinja": "^0.5.9" + }, "peerDependencies": { "react": "*", "react-native": "*", diff --git a/yarn.lock b/yarn.lock index bc05759538..63f74c76bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3362,6 +3362,13 @@ __metadata: languageName: node linkType: hard +"@huggingface/jinja@npm:^0.5.9": + version: 0.5.9 + resolution: "@huggingface/jinja@npm:0.5.9" + checksum: 10/8147f05df29b609ebb923f9e294f485897c5b5b92cddb1e41d8469b2992def242d9b536b8e9ca450a24a88fae0df6764986ab1248033f34edd4f5eb9dc2c4d78 + languageName: node + linkType: hard + "@humanwhocodes/config-array@npm:^0.13.0": version: 0.13.0 resolution: "@humanwhocodes/config-array@npm:0.13.0" @@ -12898,6 +12905,7 @@ __metadata: resolution: "react-native-executorch@workspace:packages/react-native-executorch" dependencies: "@babel/core": "npm:^7.25.1" + "@huggingface/jinja": "npm:^0.5.9" "@react-native/babel-preset": "npm:0.83.6" "@react-native/metro-config": "npm:^0.86.0" "@types/react": "npm:^19.1.12" From 963d532b786aa2fdf6a3f4b8bfd4f29bed5c3521 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 13 Jul 2026 14:31:41 +0200 Subject: [PATCH 2/3] feat: implement native JSI wrapper for TextLLMRunner --- .../cpp/RnExecutorch.cpp | 2 + .../cpp/extensions/llm/install.cpp | 14 ++ .../cpp/extensions/llm/install.h | 7 + .../cpp/extensions/llm/llm_runner.cpp | 234 ++++++++++++++++++ .../cpp/extensions/llm/llm_runner.h | 29 +++ 5 files changed, 286 insertions(+) create mode 100644 packages/react-native-executorch/cpp/extensions/llm/install.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/llm/install.h create mode 100644 packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/llm/llm_runner.h diff --git a/packages/react-native-executorch/cpp/RnExecutorch.cpp b/packages/react-native-executorch/cpp/RnExecutorch.cpp index 064d20e808..97c528f2ad 100644 --- a/packages/react-native-executorch/cpp/RnExecutorch.cpp +++ b/packages/react-native-executorch/cpp/RnExecutorch.cpp @@ -1,6 +1,7 @@ #include "RnExecutorch.h" #include "core/install.h" +#include "extensions/llm/install.h" #include "extensions/math/install.h" #include "extensions/nlp/install.h" @@ -20,6 +21,7 @@ void install(jsi::Runtime &jsiRuntime) { #endif rnexecutorch::extensions::math::install(jsiRuntime, module); rnexecutorch::extensions::nlp::install(jsiRuntime, module); + rnexecutorch::extensions::llm::install(jsiRuntime, module); jsiRuntime.global().setProperty(jsiRuntime, "__rnexecutorch_jsi__", std::move(module)); } diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.cpp b/packages/react-native-executorch/cpp/extensions/llm/install.cpp new file mode 100644 index 0000000000..f30a2bd80a --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/install.cpp @@ -0,0 +1,14 @@ +#include "install.h" +#include "llm_runner.h" + +namespace rnexecutorch::extensions::llm { +namespace jsi = facebook::jsi; + +void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) { + jsi::Object llmModule = jsi::Object(rt); + + install_createLLMRunner(rt, llmModule); + + module.setProperty(rt, "llm", llmModule); +} +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.h b/packages/react-native-executorch/cpp/extensions/llm/install.h new file mode 100644 index 0000000000..89d3de5e6b --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/install.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +namespace rnexecutorch::extensions::llm { +void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp new file mode 100644 index 0000000000..e9e07c5775 --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -0,0 +1,234 @@ +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include "llm_runner.h" +#include "core/conversions.h" +#include +#include +#include +#include + +namespace rnexecutorch::extensions::llm { +namespace jsi = facebook::jsi; +namespace conversions = rnexecutorch::core::conversions; + +namespace { +jsi::Object statsToJSI(jsi::Runtime &rt, const executorch::extension::llm::Stats &stats) { + jsi::Object obj(rt); + obj.setProperty(rt, "numPromptTokens", static_cast(stats.num_prompt_tokens)); + obj.setProperty(rt, "numGeneratedTokens", static_cast(stats.num_generated_tokens)); + obj.setProperty(rt, "firstTokenMs", static_cast(stats.first_token_ms)); + obj.setProperty(rt, "inferenceStartMs", static_cast(stats.inference_start_ms)); + obj.setProperty(rt, "inferenceEndMs", static_cast(stats.inference_end_ms)); + obj.setProperty(rt, "modelLoadStartMs", static_cast(stats.model_load_start_ms)); + obj.setProperty(rt, "modelLoadEndMs", static_cast(stats.model_load_end_ms)); + return obj; +} +} // namespace + +LLMRunnerHostObject::LLMRunnerHostObject(const std::string &modelPath, + const std::string &tokenizerPath) + : modelPath_(modelPath), + tokenizerPath_(tokenizerPath) { + auto tokenizer = executorch::extension::llm::load_tokenizer(tokenizerPath); + if (!tokenizer) { + throw std::runtime_error("LLMRunner: Failed to load runner tokenizer at path: " + tokenizerPath); + } + + runner_ = executorch::extension::llm::create_text_llm_runner(modelPath, std::move(tokenizer)); + if (!runner_) { + throw std::runtime_error("LLMRunner: Failed to create text llm runner"); + } + + auto loadError = runner_->load(); + if (loadError != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(loadError); + throw std::runtime_error("LLMRunner: Failed to load model: " + errorMsg); + } +} + +jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { + auto nameStr = name.utf8(rt); + + if (nameStr == "modelPath") { + return jsi::String::createFromUtf8(rt, modelPath_); + } + + if (nameStr == "tokenizerPath") { + return jsi::String::createFromUtf8(rt, tokenizerPath_); + } + + if (nameStr == "generate") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count < 1) { + throw jsi::JSError(rt, "LLMRunner.generate: Usage: generate(prompt, config?, onToken?)"); + } + + std::string prompt = conversions::asType(rt, "LLMRunner.generate: prompt", args[0]); + + executorch::extension::llm::GenerationConfig config; + if (count > 1 && !args[1].isUndefined() && !args[1].isNull()) { + auto configObj = conversions::asType(rt, "LLMRunner.generate: config", args[1]); + if (auto echoOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "echo")) { + config.echo = *echoOpt; + } + if (auto ignoreEosOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "ignoreEos")) { + config.ignore_eos = *ignoreEosOpt; + } + if (auto maxNewTokensOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "maxNewTokens")) { + config.max_new_tokens = *maxNewTokensOpt; + } + if (auto tempOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "temperature")) { + config.temperature = *tempOpt; + } + } + + std::function tokenCallback; + if (count > 2 && !args[2].isUndefined() && !args[2].isNull()) { + auto tokenFn = std::make_shared(conversions::asType(rt, "LLMRunner.generate: onToken", args[2])); + tokenCallback = [&rt, tokenFn](const std::string &token) { + tokenFn->call(rt, jsi::String::createFromUtf8(rt, token)); + }; + } + + auto finalStats = std::make_shared(); + auto statsCallback = [finalStats](const executorch::extension::llm::Stats &stats) { + finalStats->num_prompt_tokens = stats.num_prompt_tokens; + finalStats->num_generated_tokens = stats.num_generated_tokens; + finalStats->first_token_ms = stats.first_token_ms; + finalStats->inference_start_ms = stats.inference_start_ms; + finalStats->inference_end_ms = stats.inference_end_ms; + finalStats->model_load_start_ms = stats.model_load_start_ms; + finalStats->model_load_end_ms = stats.model_load_end_ms; + finalStats->aggregate_sampling_time_ms = stats.aggregate_sampling_time_ms; + }; + + // Hold the lock for the whole call so dispose() cannot free the + // runner mid-generation (dispose blocks on this lock until we + // return). try_to_lock: only one prefill/generate may run at a + // time, so fail fast instead of queuing. stop() is lock-free and + // can still interrupt us. + std::unique_lock lock(self->mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + throw jsi::JSError(rt, "LLMRunner.generate: Runner is already in use"); + } + if (!self->runner_) { + throw jsi::JSError(rt, "LLMRunner.generate: Runner has been disposed"); + } + auto error = self->runner_->generate(prompt, config, tokenCallback, statsCallback); + + if (error != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(error); + throw jsi::JSError(rt, "LLMRunner.generate: Failed to generate: " + errorMsg); + } + + return statsToJSI(rt, *finalStats); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "generate"), 1, fnBody); + } + + if (nameStr == "prefill") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count < 1) { + throw jsi::JSError(rt, "LLMRunner.prefill: Usage: prefill(prompt)"); + } + + std::string prompt = conversions::asType(rt, "LLMRunner.prefill: prompt", args[0]); + + // Lock held for the whole call, same as generate(). + std::unique_lock lock(self->mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + throw jsi::JSError(rt, "LLMRunner.prefill: Runner is already in use"); + } + if (!self->runner_) { + throw jsi::JSError(rt, "LLMRunner.prefill: Runner has been disposed"); + } + auto result = self->runner_->prefill({executorch::extension::llm::make_text_input(prompt)}); + if (result.error() != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(result.error()); + throw jsi::JSError(rt, "LLMRunner.prefill: Failed: " + errorMsg); + } + + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "prefill"), 1, fnBody); + } + + if (nameStr == "stop") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t /*count*/) -> jsi::Value { + // Intentionally no mutex here: stop() is designed to be called + // concurrently to interrupt an in-progress generate(). Taking the + // lock would block until generate() finishes, defeating the point. + // runner_ is only cleared by dispose() on this same (JS) thread, + // so reading it lock-free here is safe. + if (!self->runner_) { + throw jsi::JSError(rt, "LLMRunner.stop: Runner has been disposed"); + } + self->runner_->stop(); + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "stop"), 0, fnBody); + } + + if (nameStr == "dispose") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { + if (count != 0) { + throw jsi::JSError(rt, "dispose: Usage: dispose()"); + } + + // Signal stop before locking so any in-progress generate() exits + // quickly; we then block on the lock until it returns and clear + // the runner, which frees the model. Idempotent: a second + // dispose() finds a null runner_ and is a no-op. + if (self->runner_) { + self->runner_->stop(); + } + + std::unique_lock lock(self->mutex_); + self->runner_ = nullptr; + + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, fnBody); + } + + return jsi::Value::undefined(); +} + +std::vector LLMRunnerHostObject::getPropertyNames(jsi::Runtime &rt) { + std::vector properties; + properties.push_back(jsi::PropNameID::forAscii(rt, "modelPath")); + properties.push_back(jsi::PropNameID::forAscii(rt, "tokenizerPath")); + properties.push_back(jsi::PropNameID::forAscii(rt, "prefill")); + properties.push_back(jsi::PropNameID::forAscii(rt, "generate")); + properties.push_back(jsi::PropNameID::forAscii(rt, "stop")); + properties.push_back(jsi::PropNameID::forAscii(rt, "dispose")); + return properties; +} + +void install_createLLMRunner(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "createLLMRunner"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 2) { + throw jsi::JSError(rt, "createLLMRunner: Usage: createLLMRunner(modelPath, tokenizerPath)"); + } + + auto modelPath = conversions::asType(rt, "createLLMRunner: modelPath", args[0]); + auto tokenizerPath = conversions::asType(rt, "createLLMRunner: tokenizerPath", args[1]); + + try { + auto runnerInstance = std::make_shared(modelPath, tokenizerPath); + return jsi::Object::createFromHostObject(rt, runnerInstance); + } catch (const std::exception &e) { + throw jsi::JSError(rt, std::format("createLLMRunner: {}", e.what())); + } + }; + + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, fnBody)); +} +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h new file mode 100644 index 0000000000..b5205d02bf --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -0,0 +1,29 @@ +#pragma once + +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include +#include +#include +#include +#include + +namespace rnexecutorch::extensions::llm { +class LLMRunnerHostObject : public facebook::jsi::HostObject, public std::enable_shared_from_this { +public: + LLMRunnerHostObject(const std::string &modelPath, const std::string &tokenizerPath); + + facebook::jsi::Value get(facebook::jsi::Runtime &rt, const facebook::jsi::PropNameID &name) override; + std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; + +private: + std::unique_ptr runner_; + std::mutex mutex_; + std::string modelPath_; + std::string tokenizerPath_; +}; + +void install_createLLMRunner(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::llm From 3599f2c3ada2186f4cce53e3d9aa492b81179925 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 13 Jul 2026 14:38:59 +0200 Subject: [PATCH 3/3] feat: implement TypeScript wrapper and hook for LLM Chat Session --- .../src/extensions/llm/index.ts | 3 + .../src/extensions/llm/jinja.ts | 37 +++ .../src/extensions/llm/llmRunner.ts | 73 ++++ .../extensions/llm/tasks/llmChatSession.ts | 162 +++++++++ .../src/extensions/llm/tokenizerConfig.ts | 39 +++ .../src/hooks/useLLMChatSession.ts | 111 +++++++ packages/react-native-executorch/src/index.ts | 3 + .../react-native-executorch/src/models.ts | 314 ++++++++++++++++++ 8 files changed, 742 insertions(+) create mode 100644 packages/react-native-executorch/src/extensions/llm/index.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/jinja.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/llmRunner.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts create mode 100644 packages/react-native-executorch/src/hooks/useLLMChatSession.ts diff --git a/packages/react-native-executorch/src/extensions/llm/index.ts b/packages/react-native-executorch/src/extensions/llm/index.ts new file mode 100644 index 0000000000..5d5ce8f45e --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/index.ts @@ -0,0 +1,3 @@ +export * from './llmRunner'; +export * from './jinja'; +export * from './tokenizerConfig'; diff --git a/packages/react-native-executorch/src/extensions/llm/jinja.ts b/packages/react-native-executorch/src/extensions/llm/jinja.ts new file mode 100644 index 0000000000..615ccf3f4c --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/jinja.ts @@ -0,0 +1,37 @@ +import { Template } from '@huggingface/jinja'; + +import type { ChatFormatter } from './tasks/llmChatSession'; + +/** Configuration options for the Jinja chat formatter. */ +export type JinjaFormatterOptions = { + readonly bosToken?: string; + readonly extraContext?: Record; +}; + +/** + * Creates a chat formatter function that renders messages using a Jinja template. + * @param chatTemplate The Jinja template string (e.g. from tokenizer_config.json). + * @param options Jinja formatter options. + * @returns A ChatFormatter function. + */ +export function createJinjaChatFormatter( + chatTemplate: string, + options: JinjaFormatterOptions = {} +): ChatFormatter { + const { bosToken = '', extraContext } = options; + const template = new Template(chatTemplate); + + return (message, { isFirst }) => { + const isGenerationPrompt = message.role === 'assistant' && message.content === ''; + return template.render({ + // Only the first prefill of a conversation should carry the BOS token; + // later turns append to the model's existing KV cache. + // eslint-disable-next-line camelcase + bos_token: isFirst ? bosToken : '', + // eslint-disable-next-line camelcase + add_generation_prompt: isGenerationPrompt, + messages: isGenerationPrompt ? [] : [{ role: message.role, content: message.content }], + ...extraContext, + }); + }; +} diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts new file mode 100644 index 0000000000..b8a9d68bf5 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -0,0 +1,73 @@ +import { rnexecutorchJsi } from '../../native/bridge'; + +declare const llmRunnerBrand: unique symbol; + +/** Configuration options for LLM text generation. */ +export type GenerationConfig = { + readonly echo?: boolean; + readonly ignoreEos?: boolean; + readonly maxNewTokens?: number; + readonly temperature?: number; +}; + +/** Execution and performance statistics for a generation call. */ +export type GenerationStats = { + readonly numPromptTokens: number; + readonly numGeneratedTokens: number; + readonly firstTokenMs: number; + readonly inferenceStartMs: number; + readonly inferenceEndMs: number; + readonly modelLoadStartMs: number; + readonly modelLoadEndMs: number; +}; + +/** Handle to a native ExecuTorch text LLM runner. */ +export type LLMRunner = { + /** Path to the local model file. */ + readonly modelPath: string; + + /** Path to the local tokenizer configuration file. */ + readonly tokenizerPath: string; + + /** Disposes the native LLM runner and releases the loaded model memory. */ + dispose(): void; + + /** + * Prefills the runner with a prompt to build up the KV cache. + * @param prompt The prefill text prompt. + */ + prefill(prompt: string): void; + + /** Interrupts and stops any active generation call on this runner. */ + stop(): void; + + /** + * Generates text continuation from a prompt. + * @param prompt The text prompt to generate continuation for. + * @param config Generation configuration options. + * @param onToken Callback function triggered whenever a new token is generated. + * @returns Generation performance statistics. + */ + generate( + prompt: string, + config?: GenerationConfig, + onToken?: (token: string) => void + ): GenerationStats; + + /** + * Prevents plain JS objects from being cast as LLMRunners. + * @internal + */ + readonly [llmRunnerBrand]: never; +}; + +/** + * Creates a native ExecuTorch Text LLM runner instance. + * @param modelPath Path to the local .pte model file. + * @param tokenizerPath Path to the local tokenizer configuration file (e.g. tokenizer.json). + * @returns A native LLMRunner instance. + */ +export function createLLMRunner(modelPath: string, tokenizerPath: string): LLMRunner { + 'worklet'; + return rnexecutorchJsi.llm.createLLMRunner(modelPath, tokenizerPath) as LLMRunner; +} diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts new file mode 100644 index 0000000000..b32c1068e6 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -0,0 +1,162 @@ +import { scheduleOnRN, type WorkletRuntime } from 'react-native-worklets'; + +import { wrapAsync } from '../../../core/runtime'; +import { + createLLMRunner, + type LLMRunner, + type GenerationConfig, + type GenerationStats, +} from '../llmRunner'; +import { createJinjaChatFormatter } from '../jinja'; +import type { TokenizerChatConfig } from '../tokenizerConfig'; + +export type { GenerationConfig, GenerationStats }; + +/** Message interface for chat history and inputs. */ +export type ChatMessage = { + readonly role: 'system' | 'user' | 'assistant'; + readonly content: string; +}; + +/** Interface for converting a ChatMessage history turn into raw prompt text. */ +export type ChatFormatter = ( + message: ChatMessage, + options: { readonly isFirst: boolean } +) => string; + +/** Model path configuration for LLM chat. */ +export type LLMModel = { + readonly modelPath: string; + readonly tokenizerPath: string; + readonly tokenizerConfigPath: string; +}; + +/** Custom generation and state options for an LLM chat session. */ +export type LLMChatSessionOptions = { + readonly initialMessages?: readonly ChatMessage[]; + readonly generationConfig?: GenerationConfig; + readonly stopTokens?: readonly string[]; +}; + +/** Config package passed to instantiate an LLM chat session. */ +export type LLMChatSessionConfig = { + readonly model: Omit & { tokenizerConfig: TokenizerChatConfig }; + readonly options?: LLMChatSessionOptions; +}; + +/** Return wrapper holding generated response text and performance stats. */ +export type GenerationResult = { + readonly response: string; + readonly stats: GenerationStats; +}; + +/** Orchestrator interface for active LLM chat sessions. */ +export type LLMChatSession = { + dispose(): void; + sendMessage( + message: string, + onToken?: (token: string) => void, + genConfig?: GenerationConfig + ): Promise; + getHistory(): readonly ChatMessage[]; + stop(): void; +}; + +type SessionState = { + history: ChatMessage[]; +}; + +function generateChatTurn( + nativeRunner: LLMRunner, + prompt: string, + options: { + readonly genConfig: GenerationConfig; + readonly stopTokens: readonly string[]; + readonly onToken?: (token: string) => void; + } +): GenerationResult { + 'worklet'; + const { genConfig, stopTokens, onToken } = options; + + let response = ''; + + const callback = (token: string) => { + if (stopTokens.includes(token)) return; + response += token; + if (onToken) scheduleOnRN(onToken, token); + }; + + const stats = nativeRunner.generate(prompt, genConfig, callback); + + return { response, stats }; +} + +/** + * Instantiates an LLM chat session using background thread execution. + * @param config Chat session configuration and settings. + * @param runtime The worklet runtime thread to run native generation on. + * @returns A Promise resolving to an LLMChatSession instance. + */ +export async function createLLMChatSession( + config: LLMChatSessionConfig, + runtime?: WorkletRuntime +): Promise { + const { model, options } = config; + const { modelPath, tokenizerPath, tokenizerConfig } = model; + + const initialMessages = options?.initialMessages ?? []; + const defaultGenerationConfig = options?.generationConfig; + + const { chatTemplate, bosToken, eosToken } = tokenizerConfig; + + const format = createJinjaChatFormatter(chatTemplate, { bosToken }); + const stopTokens = [...(options?.stopTokens ?? []), ...(eosToken ? [eosToken] : [])]; + + const state: SessionState = { history: [] }; + const nativeRunner = await wrapAsync(createLLMRunner, runtime)(modelPath, tokenizerPath); + const prefill = wrapAsync(nativeRunner.prefill, runtime); + + for (const msg of initialMessages) { + const fmtMsg = format(msg, { isFirst: state.history.length === 0 }); + if (fmtMsg.length > 0) { + await prefill(fmtMsg); + } + state.history.push(msg); + } + + const stop = () => nativeRunner.stop(); + const dispose = () => nativeRunner.dispose(); + const runGeneration = wrapAsync(generateChatTurn, runtime); + + const sendMessage = async ( + message: string, + onToken?: (token: string) => void, + genConfig?: GenerationConfig + ): Promise => { + const userMsg: ChatMessage = { role: 'user', content: message }; + const assistantHeader: ChatMessage = { role: 'assistant', content: '' }; + + const fmtUserMsg = format(userMsg, { isFirst: state.history.length === 0 }); + const fmtAssistantHeader = format(assistantHeader, { isFirst: false }); + + state.history.push(userMsg); + + const prompt = fmtUserMsg + fmtAssistantHeader; + const { response, stats } = await runGeneration(nativeRunner, prompt, { + genConfig: { ...defaultGenerationConfig, ...genConfig }, + stopTokens, + onToken, + }); + + state.history.push({ role: 'assistant', content: response }); + + return { response, stats }; + }; + + return { + stop, + dispose, + sendMessage, + getHistory: () => state.history, + }; +} diff --git a/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts new file mode 100644 index 0000000000..829e2b20bc --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts @@ -0,0 +1,39 @@ +/** Model chat template configuration resolved from tokenizer config file. */ +export type TokenizerChatConfig = { + readonly chatTemplate: string; + readonly bosToken?: string; + readonly eosToken?: string; +}; + +function resolveToken(token: unknown): string | undefined { + if (typeof token === 'string') return token; + if (token && typeof token === 'object' && typeof (token as any).content === 'string') { + return (token as any).content; + } + return undefined; +} + +/** + * Parses raw JSON configuration from `tokenizer_config.json` into a normalized format. + * @param config Raw JSON object from tokenizer_config.json. + * @returns A parsed TokenizerChatConfig object. + */ +export function parseTokenizerConfig(config: any): TokenizerChatConfig { + let chatTemplate = config?.chat_template; + + // Some models ship multiple named templates as `[{ name, template }]`. + if (Array.isArray(chatTemplate)) { + const entry = chatTemplate.find((t) => t?.name === 'default') ?? chatTemplate[0]; + chatTemplate = entry?.template; + } + + if (typeof chatTemplate !== 'string') { + throw new Error('tokenizer_config.json does not contain a string `chat_template`'); + } + + return { + chatTemplate, + bosToken: resolveToken(config?.bos_token), + eosToken: resolveToken(config?.eos_token), + }; +} diff --git a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts new file mode 100644 index 0000000000..8a9fcce64e --- /dev/null +++ b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts @@ -0,0 +1,111 @@ +import { useEffect, useState } from 'react'; +import RNFS from 'react-native-fs'; + +import { useModel } from './useModel'; +import { useResourceDownload } from './useResourceDownload'; +import { + createLLMChatSession, + type LLMModel, + type LLMChatSessionOptions, + type LLMChatSessionConfig, +} from '../extensions/llm/tasks/llmChatSession'; +import { parseTokenizerConfig, type TokenizerChatConfig } from '../extensions/llm/tokenizerConfig'; + +/** + * Custom React hook to resolve and parse the tokenizer chat template config. + * @category Hooks + * @param source Remote URL or local path to the tokenizer config. + * @param options Config options. + * @returns Object containing parsed config, downloadProgress, and any download/parsing error. + */ +export function useTokenizerConfig(source: string, options?: { preventLoad?: boolean }) { + const { localPath, downloadProgress, downloadError } = useResourceDownload( + source, + options?.preventLoad + ); + const [config, setConfig] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + setConfig(null); + setError(null); + if (!localPath) return; + + let isMounted = true; + RNFS.readFile(localPath, 'utf8') + .then((text) => { + if (isMounted) setConfig(parseTokenizerConfig(JSON.parse(text))); + }) + .catch((e) => { + if (isMounted) setError(e instanceof Error ? e : new Error(String(e))); + }); + return () => { + isMounted = false; + }; + }, [localPath]); + + return { config, downloadProgress, error: downloadError || error }; +} + +/** + * React hook to manage downloading, caching, loading, and interacting with an LLM Chat Session model. + * @category Hooks + * @param model Configuration defining model, tokenizer, and tokenizer template paths. + * @param options Chat session options and preventLoad flag. + * @returns Object containing chat session state, sendMessage function, stop function, and errors. + */ +export function useLLMChatSession( + model: LLMModel, + options?: LLMChatSessionOptions & { preventLoad?: boolean } +) { + const { + localPath: localModelPath, + downloadProgress: modelProgress, + downloadError: modelError, + } = useResourceDownload(model.modelPath, options?.preventLoad); + + const { localPath: localTokenizerPath, downloadError: tokenizerError } = useResourceDownload( + model.tokenizerPath, + options?.preventLoad + ); + + const { config: tokenizerConfig, error: configError } = useTokenizerConfig( + model.tokenizerConfigPath, + { preventLoad: options?.preventLoad } + ); + + const downloadProgress = modelProgress; + const downloadError = modelError || tokenizerError || configError; + + const sessionOptions = options + ? { + initialMessages: options.initialMessages, + generationConfig: options.generationConfig, + stopTokens: options.stopTokens, + } + : undefined; + + let sessionConfig: LLMChatSessionConfig | null = null; + if (localModelPath && localTokenizerPath && tokenizerConfig) + sessionConfig = { + model: { modelPath: localModelPath, tokenizerPath: localTokenizerPath, tokenizerConfig }, + options: sessionOptions, + }; + + const { model: session, error: loadError } = useModel(createLLMChatSession, sessionConfig, [ + localModelPath, + localTokenizerPath, + tokenizerConfig, + ]); + + return { + isReady: !!session, + downloadProgress, + error: downloadError || loadError, + localModelPath, + localTokenizerPath, + sendMessage: session?.sendMessage, + getHistory: session?.getHistory, + stop: session?.stop, + }; +} diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index bc416044a9..0133e54d3a 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -6,6 +6,7 @@ export * from './hooks/useInstanceSegmenter'; export * from './hooks/useKeypointDetector'; export * from './hooks/useObjectDetector'; export * from './hooks/useTokenizer'; +export * from './hooks/useLLMChatSession'; export * from './hooks/useTextEmbedder'; export * from './hooks/useImageEmbedder'; export * from './hooks/useResourceDownload'; @@ -25,6 +26,7 @@ export * from './extensions/cv/tasks/objectDetection'; export * from './extensions/cv/tasks/imageEmbedding'; export * from './extensions/nlp/tasks/tokenization'; export * from './extensions/nlp/tasks/textEmbedding'; +export * from './extensions/llm/tasks/llmChatSession'; // Core primitives — for library builders and power users export { tensor } from './core/tensor'; @@ -48,6 +50,7 @@ export { defaultWorkletRuntime, wrapAsync } from './core/runtime'; export * as math from './extensions/math'; export * as cv from './extensions/cv'; export * as nlp from './extensions/nlp'; +export * as llm from './extensions/llm'; // Utils export * from './utils'; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 239655b8e9..73403efd88 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -6,6 +6,7 @@ import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetect import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation'; import type { ImageEmbedderModel } from './extensions/cv/tasks/imageEmbedding'; import type { TextEmbedderModel } from './extensions/nlp/tasks/textEmbedding'; +import type { LLMModel } from './extensions/llm/tasks/llmChatSession'; import { IMAGENET_NORM, IMAGENET1K_LABELS, @@ -595,6 +596,213 @@ const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_INT8: ImageEmbedderModel = { // ============================================================================= const ALL_MINILM_L6_V2_TOKENIZER = `${BASE_URL}-all-MiniLM-L6-v2/${VERSION_TAG}/tokenizer.json`; +// ============================================================================= +// LLMs +// ============================================================================= +const LFM2_5_BASE_URL = `${BASE_URL}-lfm-2.5/${VERSION_TAG}`; + +const LFM2_5_1_2B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_8da4w.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, +}; +const LFM2_5_1_2B_XNNPACK_FP16: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_fp16.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, +}; +const LFM2_5_350M_XNNPACK_8DA4W: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/350m/xnnpack/lfm_2_5_350m_xnnpack_8da4w.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, +}; +const LFM2_5_350M_XNNPACK_FP16: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/350m/xnnpack/lfm_2_5_350m_xnnpack_fp16.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, +}; + +const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; + +const BIELIK_V3_1_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${BIELIK_V3_1_5B_BASE_URL}/xnnpack/bielik_v3_0_1_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer_config.json`, +}; +const BIELIK_V3_1_5B_XNNPACK_FP16: LLMModel = { + modelPath: `${BIELIK_V3_1_5B_BASE_URL}/xnnpack/bielik_v3_0_1_5b_xnnpack_fp16.pte`, + tokenizerPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer_config.json`, +}; + +const LLAMA3_2_BASE_URL = `${BASE_URL}-llama-3.2/${VERSION_TAG}`; + +const LLAMA3_2_3B_SPINQUANT: LLMModel = { + modelPath: `${LLAMA3_2_BASE_URL}/3b/xnnpack/llama_3_2_3b_xnnpack_spinquant.pte`, + tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, +}; +const LLAMA3_2_3B_BF16: LLMModel = { + modelPath: `${LLAMA3_2_BASE_URL}/3b/xnnpack/llama_3_2_3b_xnnpack_bf16.pte`, + tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, +}; +const LLAMA3_2_1B_SPINQUANT: LLMModel = { + modelPath: `${LLAMA3_2_BASE_URL}/1b/xnnpack/llama_3_2_1b_xnnpack_spinquant.pte`, + tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, +}; +const LLAMA3_2_1B_BF16: LLMModel = { + modelPath: `${LLAMA3_2_BASE_URL}/1b/xnnpack/llama_3_2_1b_xnnpack_bf16.pte`, + tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, +}; + +const SMOLLM2_BASE_URL = `${BASE_URL}-smolLm-2/${VERSION_TAG}`; + +const SMOLLM2_135M_8DA4W: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/135m/xnnpack/smollm2_135m_xnnpack_8da4w.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_135M_BF16: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/135m/xnnpack/smollm2_135m_xnnpack_bf16.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_360M_8DA4W: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/360m/xnnpack/smollm2_360m_xnnpack_8da4w.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_360M_BF16: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/360m/xnnpack/smollm2_360m_xnnpack_bf16.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_1_7B_8DA4W: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/1_7b/xnnpack/smollm2_1_7b_xnnpack_8da4w.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_1_7B_BF16: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/1_7b/xnnpack/smollm2_1_7b_xnnpack_bf16.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; + +const HAMMER2_1_BASE_URL = `${BASE_URL}-hammer-2.1/${VERSION_TAG}`; + +const HAMMER2_1_0_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_0_5B_XNNPACK_BF16: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_bf16.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_1_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_1_5B_XNNPACK_BF16: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_bf16.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_3B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/3b/xnnpack/hammer_2_1_3b_xnnpack_8da4w.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_3B_XNNPACK_BF16: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/3b/xnnpack/hammer_2_1_3b_xnnpack_bf16.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; + +const PHI4_MINI_BASE_URL = `${BASE_URL}-phi-4-mini/${VERSION_TAG}`; + +const PHI4_MINI_XNNPACK_8DA4W: LLMModel = { + modelPath: `${PHI4_MINI_BASE_URL}/xnnpack/phi_4_mini_xnnpack_8da4w.pte`, + tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, +}; +const PHI4_MINI_XNNPACK_BF16: LLMModel = { + modelPath: `${PHI4_MINI_BASE_URL}/xnnpack/phi_4_mini_xnnpack_bf16.pte`, + tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, +}; + +const QWEN2_5_BASE_URL = `${BASE_URL}-qwen-2.5/${VERSION_TAG}`; + +const QWEN2_5_0_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_0_5B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_1_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_1_5B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_3B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/3b/xnnpack/qwen_2_5_3b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_3B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/3b/xnnpack/qwen_2_5_3b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; + +const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${VERSION_TAG}`; + +const QWEN3_0_6B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/0_6b/xnnpack/qwen_3_0_6b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_0_6B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/0_6b/xnnpack/qwen_3_0_6b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_1_7B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/1_7b/xnnpack/qwen_3_1_7b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_1_7B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/1_7b/xnnpack/qwen_3_1_7b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_4B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/4b/xnnpack/qwen_3_4b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_4B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/4b/xnnpack/qwen_3_4b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; + /** * Registry of pre-configured ExecuTorch models. * @@ -799,6 +1007,112 @@ export const models = { tokenizer: { ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER, }, + llm: { + LFM2_5: { + '1_2B': { + ...LFM2_5_1_2B_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_1_2B_XNNPACK_8DA4W, + XNNPACK_FP16: LFM2_5_1_2B_XNNPACK_FP16, + }, + '350M': { + ...LFM2_5_350M_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_350M_XNNPACK_8DA4W, + XNNPACK_FP16: LFM2_5_350M_XNNPACK_FP16, + }, + }, + BIELIK_V3: { + '1_5B': { + ...BIELIK_V3_1_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: BIELIK_V3_1_5B_XNNPACK_8DA4W, + XNNPACK_FP16: BIELIK_V3_1_5B_XNNPACK_FP16, + }, + }, + LLAMA3_2: { + '1B': { + ...LLAMA3_2_1B_SPINQUANT, + XNNPACK_SPINQUANT: LLAMA3_2_1B_SPINQUANT, + XNNPACK_BF16: LLAMA3_2_1B_BF16, + }, + '3B': { + ...LLAMA3_2_3B_SPINQUANT, + XNNPACK_SPINQUANT: LLAMA3_2_3B_SPINQUANT, + XNNPACK_BF16: LLAMA3_2_3B_BF16, + }, + }, + SMOLLM2: { + '135M': { + ...SMOLLM2_135M_8DA4W, + XNNPACK_8DA4W: SMOLLM2_135M_8DA4W, + XNNPACK_BF16: SMOLLM2_135M_BF16, + }, + '360M': { + ...SMOLLM2_360M_8DA4W, + XNNPACK_8DA4W: SMOLLM2_360M_8DA4W, + XNNPACK_BF16: SMOLLM2_360M_BF16, + }, + '1_7B': { + ...SMOLLM2_1_7B_8DA4W, + XNNPACK_8DA4W: SMOLLM2_1_7B_8DA4W, + XNNPACK_BF16: SMOLLM2_1_7B_BF16, + }, + }, + HAMMER2_1: { + '0_5B': { + ...HAMMER2_1_0_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: HAMMER2_1_0_5B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_0_5B_XNNPACK_BF16, + }, + '1_5B': { + ...HAMMER2_1_1_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: HAMMER2_1_1_5B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_1_5B_XNNPACK_BF16, + }, + '3B': { + ...HAMMER2_1_3B_XNNPACK_8DA4W, + XNNPACK_8DA4W: HAMMER2_1_3B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_3B_XNNPACK_BF16, + }, + }, + PHI4_MINI: { + ...PHI4_MINI_XNNPACK_8DA4W, + XNNPACK_8DA4W: PHI4_MINI_XNNPACK_8DA4W, + XNNPACK_BF16: PHI4_MINI_XNNPACK_BF16, + }, + QWEN2_5: { + '0_5B': { + ...QWEN2_5_0_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN2_5_0_5B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_0_5B_XNNPACK_BF16, + }, + '1_5B': { + ...QWEN2_5_1_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN2_5_1_5B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_1_5B_XNNPACK_BF16, + }, + '3B': { + ...QWEN2_5_3B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN2_5_3B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_3B_XNNPACK_BF16, + }, + }, + QWEN3: { + '0_6B': { + ...QWEN3_0_6B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN3_0_6B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_0_6B_XNNPACK_BF16, + }, + '1_7B': { + ...QWEN3_1_7B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN3_1_7B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_1_7B_XNNPACK_BF16, + }, + '4B': { + ...QWEN3_4B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN3_4B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_4B_XNNPACK_BF16, + }, + }, + }, textEmbeddings: { ALL_MINILM_L6_V2: { ...ALL_MINILM_L6_V2_EMBEDDINGS,