From e24b2f230818ba48ea9fa0a874d77afdad6ae9d8 Mon Sep 17 00:00:00 2001 From: benITo47 Date: Mon, 20 Jul 2026 17:09:09 +0200 Subject: [PATCH 01/13] [RNE Rewrite] feat(cv): native OCR + image ops (CRAFT/DBNet decode, CTC, rectifyQuad, warpByGrid, rotate) --- .../cpp/core/model.cpp | 229 +++++++++- .../react-native-executorch/cpp/core/model.h | 5 +- .../cpp/core/tensor_helpers.cpp | 36 ++ .../cpp/core/tensor_helpers.h | 28 ++ .../cpp/extensions/cv/box_ops.cpp | 51 +-- .../cpp/extensions/cv/image_ops.cpp | 247 +++++++++++ .../cpp/extensions/cv/image_ops.h | 3 + .../cpp/extensions/cv/install.cpp | 8 + .../cpp/extensions/cv/ocr_ops.cpp | 396 ++++++++++++++++++ .../cpp/extensions/cv/ocr_ops.h | 9 + .../cpp/extensions/cv/utils.h | 2 + 11 files changed, 985 insertions(+), 29 deletions(-) create mode 100644 packages/react-native-executorch/cpp/extensions/cv/ocr_ops.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/cv/ocr_ops.h diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 5b9503cb82..dbacdd1aaa 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "dtype.h" @@ -188,6 +189,141 @@ parseDynamicInputShapes(executorch::extension::Module &module, const std::string return dynamicShapes; } + +/** + * Parses enumerated input-shape constraints for the inputs of a module method. + * + * Mirrors parseDynamicInputShapes for enumerated-shape backends (e.g. CoreML, + * which accepts only a fixed set of whole shapes rather than a continuous + * range). The .pte must export a companion method "get_enum_shapes_" + * that takes no arguments and returns one output per Tag::Tensor input; each + * output is a 2D int32 tensor of shape [numShapes, rank] whose rows are the + * legal whole shapes for that input. + * + * @return A vector aligned 1-to-1 with the method's inputs (empty entries for + * non-tensor inputs), or std::nullopt if the companion is not defined. + */ +std::optional> +parseEnumInputShapes(executorch::extension::Module &module, const std::string &methodName) { + using executorch::aten::ScalarType; + + const auto companion = std::format("get_enum_shapes_{}", methodName); + const auto ctx = companion + ": "; + + auto methodNames = unwrap(ctx + "failed to get method names", module.method_names()); + if (methodName == companion || !methodNames.contains(companion)) { + return std::nullopt; + } + + auto methodMeta = unwrap(std::format("{}failed to get meta for method '{}'", ctx, methodName), + module.method_meta(methodName)); + + size_t expectedTensorInputs = 0; + for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { + auto tag = unwrap(std::format("{}failed to get tag for input [{}]", ctx, i), methodMeta.input_tag(i)); + if (tag == executorch::runtime::Tag::Tensor) { + expectedTensorInputs++; + } + } + + auto result = unwrap(ctx + "failed to execute", module.execute(companion)); + if (result.size() != expectedTensorInputs) { + throw std::runtime_error(std::format("{}number of outputs returned ({}) does not match the number of " + "tensor inputs declared by method '{}' ({})", + ctx, result.size(), methodName, expectedTensorInputs)); + } + + std::vector enumeratedShapes; + enumeratedShapes.reserve(methodMeta.num_inputs()); + size_t tensorIndex = 0; + + for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { + auto tag = unwrap(std::format("{}failed to get tag for input [{}]", ctx, i), + methodMeta.input_tag(i)); + + if (tag != executorch::runtime::Tag::Tensor) { + // Emplace an empty set for non-tensor inputs to keep 1-to-1 index + // alignment between enumeratedShapes and the model's inputs vector. + enumeratedShapes.emplace_back(); + continue; + } + + const auto &out = result.at(tensorIndex); + if (!out.isTensor()) { + throw std::runtime_error(std::format("{}output[{}] is not a tensor", ctx, tensorIndex)); + } + + auto inputMeta = unwrap(std::format("{}failed to get tensor meta for input [{}]", ctx, i), + methodMeta.input_tensor_meta(i)); + const auto rank = inputMeta.sizes().size(); + const auto shapeTensor = out.toTensor(); + + if (shapeTensor.dim() != 2 || shapeTensor.size(1) != static_cast(rank) || + shapeTensor.scalar_type() != ScalarType::Int) { + throw std::runtime_error(std::format("{}output[{}] expected to be a 2D int32_t tensor of shape " + "[numShapes, {}]", + ctx, tensorIndex, rank)); + } + + const auto numShapes = shapeTensor.size(0); + const auto *data = shapeTensor.const_data_ptr(); + tensor::EnumeratedShapes shapes; + shapes.reserve(static_cast(numShapes)); + + for (ssize_t s = 0; s < numShapes; ++s) { + std::vector dims; + dims.reserve(rank); + for (size_t axis = 0; axis < rank; ++axis) { + const auto dim = data[static_cast(s) * rank + axis]; + if (dim < 1) { + throw std::runtime_error(std::format("{}output[{}], shape {} axis {} is invalid: " + "expected dim >= 1 but got {}", + ctx, tensorIndex, s, axis, dim)); + } + if (dim > inputMeta.sizes()[axis]) { + throw std::runtime_error(std::format("{}output[{}], shape {} axis {} ({}) exceeds " + "model metadata upper limit ({})", + ctx, tensorIndex, s, axis, dim, inputMeta.sizes()[axis])); + } + dims.push_back(dim); + } + shapes.push_back(std::move(dims)); + } + + enumeratedShapes.push_back(std::move(shapes)); + ++tensorIndex; + } + + return enumeratedShapes; +} + +// Encodes one resolved input constraint onto a JS object, discriminated by the +// field it sets: `dims` (dynamic ranges) or `shapes` (enumerated set). Overloaded +// on the ShapeConstraint alternatives so the accessor can std::visit them. +void encodeConstraint(jsi::Runtime &rt, jsi::Object &out, const tensor::SymbolicShape &shape) { + auto dims = jsi::Array(rt, shape.size()); + for (size_t axis = 0; axis < shape.size(); ++axis) { + const auto &range = std::get(shape[axis]); + auto dim = jsi::Object(rt); + dim.setProperty(rt, "min", static_cast(range.min)); + dim.setProperty(rt, "max", static_cast(range.max)); + dim.setProperty(rt, "step", static_cast(range.step.value_or(1))); + dims.setValueAtIndex(rt, axis, dim); + } + out.setProperty(rt, "dims", dims); +} + +void encodeConstraint(jsi::Runtime &rt, jsi::Object &out, const tensor::EnumeratedShapes &enumeratedShapes) { + auto shapes = jsi::Array(rt, enumeratedShapes.size()); + for (size_t s = 0; s < enumeratedShapes.size(); ++s) { + auto shapeArr = jsi::Array(rt, enumeratedShapes[s].size()); + for (size_t axis = 0; axis < enumeratedShapes[s].size(); ++axis) { + shapeArr.setValueAtIndex(rt, axis, static_cast(enumeratedShapes[s][axis])); + } + shapes.setValueAtIndex(rt, s, shapeArr); + } + out.setProperty(rt, "shapes", shapes); +} } // namespace namespace rnexecutorch::core::model { @@ -208,9 +344,30 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) auto methodNames = unwrap("Failed to get method names", etModule_->method_names()); for (const auto &methodName : methodNames) { auto dynamicShapes = parseDynamicInputShapes(*etModule_, methodName); + auto enumShapes = parseEnumInputShapes(*etModule_, methodName); + + if (dynamicShapes && enumShapes) { + throw std::runtime_error(std::format( + "Method '{}' declares both get_dynamic_dims_{} and get_enum_shapes_{} companions; " + "a method must use a single shape-constraint kind, not both.", + methodName, methodName, methodName)); + } + + std::vector constraints; if (dynamicShapes) { - dynamicInputShapes_.emplace(methodName, std::move(*dynamicShapes)); + constraints.reserve(dynamicShapes->size()); + for (auto &shape : *dynamicShapes) { + constraints.emplace_back(std::move(shape)); + } + } else if (enumShapes) { + constraints.reserve(enumShapes->size()); + for (auto &shape : *enumShapes) { + constraints.emplace_back(std::move(shape)); + } + } else { + continue; } + inputShapeConstraints_.emplace(methodName, std::move(constraints)); } } @@ -322,6 +479,66 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "getMethodMeta"), 1, fnBody); } + if (nameStr == "getInputShapeConstraints") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 1) { + throw jsi::JSError(rt, "getInputShapeConstraints: Usage: getInputShapeConstraints(methodName)"); + } + + std::unique_lock lock(self->mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + throw jsi::JSError(rt, "getInputShapeConstraints: Model is currently in use"); + } + + if (!self->etModule_) { + throw jsi::JSError(rt, "getInputShapeConstraints: Model has been disposed"); + } + + auto methodName = conversions::asType(rt, "getInputShapeConstraints: methodName", args[0]); + auto methodMeta = unwrap(rt, "getInputShapeConstraints", self->etModule_->method_meta(methodName)); + + const auto it = self->inputShapeConstraints_.find(methodName); + const std::vector *constraints = + it != self->inputShapeConstraints_.end() ? &it->second : nullptr; + + // One entry per tensor input, aligned with getMethodMeta().inputTensorMeta. + size_t numTensorInputs = 0; + for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { + auto ctx = std::format("getInputShapeConstraints: input tag [{}]", i); + if (unwrap(rt, ctx, methodMeta.input_tag(i)) == executorch::runtime::Tag::Tensor) { + ++numTensorInputs; + } + } + + auto jsArray = jsi::Array(rt, numTensorInputs); + size_t outIndex = 0; + for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { + auto ctx = std::format("getInputShapeConstraints: input [{}]", i); + if (unwrap(rt, ctx, methodMeta.input_tag(i)) != executorch::runtime::Tag::Tensor) { + continue; + } + + auto entry = jsi::Object(rt); + if (constraints != nullptr) { + std::visit([&](const auto &c) { encodeConstraint(rt, entry, c); }, (*constraints)[i]); + } else { + // No companion: report the single static shape. + auto tensorMeta = unwrap(rt, ctx, methodMeta.input_tensor_meta(i)); + auto shapeArr = jsi::Array(rt, tensorMeta.sizes().size()); + for (size_t axis = 0; axis < tensorMeta.sizes().size(); ++axis) { + shapeArr.setValueAtIndex(rt, axis, static_cast(tensorMeta.sizes()[axis])); + } + entry.setProperty(rt, "shape", shapeArr); + } + jsArray.setValueAtIndex(rt, outIndex, std::move(entry)); + ++outIndex; + } + return jsArray; + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "getInputShapeConstraints"), 1, fnBody); + } + if (nameStr == "execute") { auto self = shared_from_this(); auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { @@ -364,9 +581,12 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { auto expectedDtype = fromScalarType(rt, ctx, tensorMeta.scalar_type()); std::shared_ptr tensorHostObject; - if (self->dynamicInputShapes_.contains(methodName)) { - auto expectedShape = self->dynamicInputShapes_[methodName][i]; - tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, expectedShape); + if (self->inputShapeConstraints_.contains(methodName)) { + tensorHostObject = std::visit( + [&](const auto &constraint) { + return tensor::fromJs(rt, ctx, val, expectedDtype, constraint); + }, + self->inputShapeConstraints_[methodName][i]); } else { tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, tensorMeta.sizes()); } @@ -500,6 +720,7 @@ std::vector ModelHostObject::getPropertyNames(jsi::Ru properties.push_back(jsi::PropNameID::forAscii(rt, "path")); properties.push_back(jsi::PropNameID::forAscii(rt, "getMethodNames")); properties.push_back(jsi::PropNameID::forAscii(rt, "getMethodMeta")); + properties.push_back(jsi::PropNameID::forAscii(rt, "getInputShapeConstraints")); properties.push_back(jsi::PropNameID::forAscii(rt, "execute")); properties.push_back(jsi::PropNameID::forAscii(rt, "dispose")); return properties; diff --git a/packages/react-native-executorch/cpp/core/model.h b/packages/react-native-executorch/cpp/core/model.h index b196d48664..ab8fab3fa8 100644 --- a/packages/react-native-executorch/cpp/core/model.h +++ b/packages/react-native-executorch/cpp/core/model.h @@ -35,7 +35,10 @@ class ModelHostObject : public jsi::HostObject, std::unique_ptr etModule_; std::mutex mutex_; - std::unordered_map> dynamicInputShapes_; + // Per-method, per-input shape constraints (dynamic ranges or enumerated + // shapes), parsed from the model's companion methods at load. Absent for + // methods whose inputs are fully static. + std::unordered_map> inputShapeConstraints_; }; void install_loadModel(jsi::Runtime &rt, jsi::Object &module); diff --git a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp index 54c9e23064..69cd9cfca2 100644 --- a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp +++ b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp @@ -129,4 +129,40 @@ fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, return tensor; } + +std::shared_ptr +fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, + std::optional expectedDtype, const EnumeratedShapes &enumeratedShapes) { + + auto obj = conversions::asType(rt, name, value); + if (!obj.isHostObject(rt)) { + throw jsi::JSError(rt, name + " must be a Tensor"); + } + + auto tensor = obj.getHostObject(rt); + const auto &dtype = tensor->dtype_; + const auto &shape = tensor->shape_; + + if (expectedDtype && dtype != *expectedDtype) { + throw jsi::JSError(rt, std::format("{} must be of type {}", name, types::toString(*expectedDtype))); + } + + for (const auto &candidate : enumeratedShapes) { + if (candidate.size() != shape.size()) { + continue; + } + bool matches = true; + for (size_t axis = 0; axis < shape.size(); ++axis) { + if (candidate[axis] != shape[axis]) { + matches = false; + break; + } + } + if (matches) { + return tensor; + } + } + + throw jsi::JSError(rt, std::format("{} shape does not match any of the model's enumerated input shapes", name)); +} } // namespace rnexecutorch::core::tensor diff --git a/packages/react-native-executorch/cpp/core/tensor_helpers.h b/packages/react-native-executorch/cpp/core/tensor_helpers.h index 9004e6f262..f7b82d659e 100644 --- a/packages/react-native-executorch/cpp/core/tensor_helpers.h +++ b/packages/react-native-executorch/cpp/core/tensor_helpers.h @@ -31,6 +31,23 @@ struct RangeDim { using SymbolicShape = std::vector>; +/** + * A set of complete legal input shapes for a tensor input, used by + * enumerated-shape backends (e.g. CoreML) that accept only specific whole + * shapes rather than a continuous range. Unlike RangeDim, which constrains a + * single dimension, an enumerated set is a cross-dimensional constraint: a + * concrete shape must equal one entry exactly. + */ +using EnumeratedShapes = std::vector>; + +/** + * The resolved shape constraint for a single tensor input: either dynamic + * per-dimension ranges ({@link SymbolicShape}) or an enumerated set of whole + * shapes ({@link EnumeratedShapes}). A method's inputs are uniformly one kind or + * the other — a single method cannot mix both. + */ +using ShapeConstraint = std::variant; + /** * Tries to acquire a shared (read) lock on the underlying tensor resource. * Throws a facebook::jsi::JSError if the lock is currently held uniquely by @@ -88,6 +105,17 @@ std::shared_ptr fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, std::optional expectedDtype, const std::optional &expectedShape); +/** + * @overload + * + * Validates the tensor's concrete shape against an enumerated set: the shape + * must equal one of `enumeratedShapes` exactly (enumerated-shape backends). + * Throws a facebook::jsi::JSError listing the legal shapes on mismatch. + */ +std::shared_ptr +fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, + std::optional expectedDtype, const EnumeratedShapes &enumeratedShapes); + /** * @overload * diff --git a/packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp index 593c86bbd2..89d0aa73ff 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp +++ b/packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp @@ -71,6 +71,7 @@ std::array decodeToXyxy( case BoxFormat::CXCYWH: return {a - c / 2.0f, b - d / 2.0f, a + c / 2.0f, b + d / 2.0f}; } + throw std::invalid_argument("decodeToXyxy: unhandled box format"); } } // namespace @@ -127,21 +128,29 @@ void install_nms(jsi::Runtime &rt, jsi::Object &module) { std::vector> groups; std::vector suppressed(candidates.size(), false); + // Decode every candidate to xyxy once, not per pair in the O(n²) loop. + std::vector> decoded(candidates.size()); + for (size_t k = 0; k < candidates.size(); ++k) { + const std::int32_t idx = candidates[k].first; + decoded[k] = decodeToXyxy( + boxesPtr[idx * 4 + 0], + boxesPtr[idx * 4 + 1], + boxesPtr[idx * 4 + 2], + boxesPtr[idx * 4 + 3], + boxFormat); + } + const auto boxArea = [](const std::array &box) { + return (box[2] - box[0]) * (box[3] - box[1]); + }; + for (size_t i = 0; i < candidates.size(); ++i) { if (suppressed[i]) { continue; } - std::int32_t idxI = candidates[i].first; - - auto [xminA, yminA, xmaxA, ymaxA] = decodeToXyxy( - boxesPtr[idxI * 4 + 0], - boxesPtr[idxI * 4 + 1], - boxesPtr[idxI * 4 + 2], - boxesPtr[idxI * 4 + 3], - boxFormat); - - const float areaA = (xmaxA - xminA) * (ymaxA - yminA); + const std::int32_t idxI = candidates[i].first; + const auto &[aXmin, aYmin, aXmax, aYmax] = decoded[i]; + const float areaA = boxArea(decoded[i]); std::vector overlapping = {idxI}; @@ -150,21 +159,14 @@ void install_nms(jsi::Runtime &rt, jsi::Object &module) { continue; } - std::int32_t idxJ = candidates[j].first; - - auto [xminB, yminB, xmaxB, ymaxB] = decodeToXyxy( - boxesPtr[idxJ * 4 + 0], - boxesPtr[idxJ * 4 + 1], - boxesPtr[idxJ * 4 + 2], - boxesPtr[idxJ * 4 + 3], - boxFormat); - - const float areaB = (xmaxB - xminB) * (ymaxB - yminB); + const std::int32_t idxJ = candidates[j].first; + const auto &[bXmin, bYmin, bXmax, bYmax] = decoded[j]; + const float areaB = boxArea(decoded[j]); - const float interYMin = std::max(yminA, yminB); - const float interXMin = std::max(xminA, xminB); - const float interYMax = std::min(ymaxA, ymaxB); - const float interXMax = std::min(xmaxA, xmaxB); + const float interYMin = std::max(aYmin, bYmin); + const float interXMin = std::max(aXmin, bXmin); + const float interYMax = std::min(aYmax, bYmax); + const float interXMax = std::min(aXmax, bXmax); const float interH = std::max(0.0f, interYMax - interYMin); const float interW = std::max(0.0f, interXMax - interXMin); @@ -200,6 +202,7 @@ void install_nms(jsi::Runtime &rt, jsi::Object &module) { return resultGroups; } } + throw jsi::JSError(rt, "nms: unhandled nmsType"); }; module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody)); diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp index 25716c576c..db4c2fb61a 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp +++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp @@ -1,6 +1,7 @@ #include "image_ops.h" #include +#include #include #include #include @@ -432,4 +433,250 @@ void install_applyColormap(jsi::Runtime &rt, jsi::Object &module) { }; module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody)); } + +void install_rotate(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "rotate"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 3) { + throw jsi::JSError(rt, "Usage: rotate(src, dst, degCW)"); + } + + auto src = tensor::fromJs(rt, "rotate: src", args[0], std::nullopt, {"H", "W", "C"}); + auto dst = tensor::fromJs(rt, "rotate: dst", args[1], src->dtype_, {"H'", "W'", src->shape_[2]}); + tensor::checkNotSameTensor(rt, "rotate: src", src, "rotate: dst", dst); + + const auto degCW = conversions::asType(rt, "rotate: degCW", args[2]); + int rotateCode = 0; + if (degCW == 90) { + rotateCode = ::cv::ROTATE_90_CLOCKWISE; + } else if (degCW == 180) { + rotateCode = ::cv::ROTATE_180; + } else if (degCW == 270) { + rotateCode = ::cv::ROTATE_90_COUNTERCLOCKWISE; + } else { + throw jsi::JSError(rt, "rotate: degCW must be 90, 180, or 270"); + } + + const int32_t srcH = src->shape_[0]; + const int32_t srcW = src->shape_[1]; + const int32_t channels = src->shape_[2]; + // 90/270 transpose the axes; 180 preserves them. dst must be pre-sized to match, + // else cv::rotate reallocates off the tensor buffer and the result is lost. + const bool swap = degCW != 180; + const int32_t expH = swap ? srcW : srcH; + const int32_t expW = swap ? srcH : srcW; + if (dst->shape_[0] != expH || dst->shape_[1] != expW) { + throw jsi::JSError(rt, "rotate: dst must be sized [" + std::to_string(expH) + ", " + + std::to_string(expW) + ", C] for a " + std::to_string(degCW) + + " degree rotation"); + } + + auto srcLock = tensor::tryLockShared(rt, "rotate: src", src); + auto dstLock = tensor::tryLockUnique(rt, "rotate: dst", dst); + + int cvType{}; + try { + cvType = CV_MAKETYPE(dtypeToCvDepth(src->dtype_), channels); + } catch (const std::exception &e) { + throw jsi::JSError(rt, "rotate: " + std::string(e.what())); + } + + const ::cv::Mat srcMat(srcH, srcW, cvType, src->data_.get()); + ::cv::Mat dstMat(expH, expW, cvType, dst->data_.get()); + ::cv::rotate(srcMat, dstMat, rotateCode); + + return jsi::Value(rt, args[1]); + }; + + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody)); +} + +// ------------------------------- warpByGrid -------------------------------- +// Warp `src` through a backward sampling field (torch grid_sample step of a +// geometric dewarp) into `dst` via cv::remap. grid is [..,2,gH,gW], normalized +// to [-1,1] with align_corners=true (channel 0 = x, 1 = y). +void install_warpByGrid(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "warpByGrid"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value &, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 3) { + throw jsi::JSError(rt, "Usage: warpByGrid(src, grid, dst)"); + } + + using DType = rnexecutorch::core::types::DType; + auto src = tensor::fromJs(rt, "warpByGrid: src", args[0], DType::uint8, {"H", "W", "C"}); + auto grid = tensor::fromJs(rt, "warpByGrid: grid", args[1], DType::float32, std::nullopt); + auto dst = tensor::fromJs(rt, "warpByGrid: dst", args[2], DType::uint8, + {src->shape_[0], src->shape_[1], src->shape_[2]}); + tensor::checkNotSameTensor(rt, "warpByGrid: src", src, "warpByGrid: dst", dst); + tensor::checkNotSameTensor(rt, "warpByGrid: grid", grid, "warpByGrid: dst", dst); + + // grid is the torch grid_sample field [..,2,gH,gW], channel 0 = x, 1 = y, + // normalized to [-1,1] with align_corners=true. Its rank varies, so it can't + // be expressed as a single fromJs shape — check the [..,2,gH,gW] tail here. + const auto &gs = grid->shape_; + if (gs.size() < 3 || gs[gs.size() - 3] != 2) { + throw jsi::JSError(rt, "warpByGrid: grid must be [..,2,gH,gW]"); + } + + auto srcLock = tensor::tryLockShared(rt, "warpByGrid: src", src); + auto gridLock = tensor::tryLockShared(rt, "warpByGrid: grid", grid); + auto dstLock = tensor::tryLockUnique(rt, "warpByGrid: dst", dst); + + const int32_t h = src->shape_[0]; + const int32_t w = src->shape_[1]; + const int32_t channels = src->shape_[2]; + const int32_t gridH = gs[gs.size() - 2]; + const int32_t gridW = gs[gs.size() - 1]; + const int32_t plane = gridH * gridW; // grid is tiny (~45x45); no int32 overflow + const auto *g = reinterpret_cast(grid->data_.get()); + + // Bilinearly sample channel `c` of the low-res grid at fractional (gx, gy). + auto sampleGrid = [&](int32_t c, float gx, float gy) -> float { + const int32_t x0 = std::clamp(static_cast(std::floor(gx)), 0, gridW - 1); + const int32_t y0 = std::clamp(static_cast(std::floor(gy)), 0, gridH - 1); + const int32_t x1 = std::min(x0 + 1, gridW - 1); + const int32_t y1 = std::min(y0 + 1, gridH - 1); + const float dx = gx - static_cast(x0); + const float dy = gy - static_cast(y0); + const int32_t base = c * plane; + const float top = g[base + y0 * gridW + x0] + + (g[base + y0 * gridW + x1] - g[base + y0 * gridW + x0]) * dx; + const float bot = g[base + y1 * gridW + x0] + + (g[base + y1 * gridW + x1] - g[base + y1 * gridW + x0]) * dx; + return top + (bot - top) * dy; + }; + + ::cv::Mat mapX(h, w, CV_32F); + ::cv::Mat mapY(h, w, CV_32F); + for (int32_t oy = 0; oy < h; ++oy) { + const float gy = h > 1 ? (static_cast(oy) / static_cast(h - 1)) * + static_cast(gridH - 1) + : 0.0f; + auto *rowX = mapX.ptr(oy); + auto *rowY = mapY.ptr(oy); + for (int32_t ox = 0; ox < w; ++ox) { + const float gx = w > 1 ? (static_cast(ox) / static_cast(w - 1)) * + static_cast(gridW - 1) + : 0.0f; + const float nx = sampleGrid(0, gx, gy); // [-1,1] + const float ny = sampleGrid(1, gx, gy); + rowX[ox] = ((nx + 1.0f) / 2.0f) * static_cast(w - 1); + rowY[ox] = ((ny + 1.0f) / 2.0f) * static_cast(h - 1); + } + } + + const int cvType = CV_MAKETYPE(CV_8U, channels); + ::cv::Mat srcMat(h, w, cvType, src->data_.get()); + ::cv::Mat dstMat(h, w, cvType, dst->data_.get()); + try { + ::cv::remap(srcMat, dstMat, mapX, mapY, ::cv::INTER_LINEAR, ::cv::BORDER_REPLICATE); + } catch (const std::exception &e) { + throw jsi::JSError(rt, std::string("warpByGrid: OpenCV error: ") + e.what()); + } + return jsi::Value(rt, args[2]); + }; + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody)); +} + +// ------------------------------- rectifyQuad ---------------------------------- +// Perspective-crop an oriented quad of `src` into the `dst` canvas (crop + +// resize-to-height + pad/align). Used by the OCR recognizer to normalize a +// detected text box into the fixed recognizer canvas. +void install_rectifyQuad(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "rectifyQuad"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value &, const jsi::Value *args, + size_t count) -> jsi::Value { + if (count != 4) { + throw jsi::JSError(rt, "Usage: rectifyQuad(src, dst, quad, options)"); + } + using DType = rnexecutorch::core::types::DType; + auto src = tensor::fromJs(rt, "rectifyQuad: src", args[0], DType::uint8, {"H", "W", "C"}); + auto dst = tensor::fromJs(rt, "rectifyQuad: dst", args[1], DType::uint8, + {"H'", "W'", src->shape_[2]}); + tensor::checkNotSameTensor(rt, "rectifyQuad: src", src, "rectifyQuad: dst", dst); + const auto quadArr = conversions::asType(rt, "rectifyQuad: quad", args[2]); + const auto opts = conversions::asType(rt, "rectifyQuad: options", args[3]); + if (quadArr.length(rt) != 8) { + throw jsi::JSError(rt, "rectifyQuad: quad must have exactly 8 numbers (4 points)"); + } + + const int32_t channels = src->shape_[2]; + const int32_t recH = dst->shape_[0]; + const int32_t canvasW = dst->shape_[1]; + + const int32_t contentWidth = std::clamp( + conversions::getRequiredProperty(rt, "rectifyQuad: options", opts, "contentWidth"), + 1, canvasW); + const auto padMode = conversions::getRequiredProperty(rt, "rectifyQuad: options", opts, "padMode"); + const auto padValue = conversions::getRequiredProperty(rt, "rectifyQuad: options", opts, "padValue"); + const auto align = conversions::getRequiredProperty(rt, "rectifyQuad: options", opts, "align"); + // offsetX >= 0 places content at that x (overriding align); clear=false skips + // wiping dst first, so successive warps compose into one canvas (glyph strips). + const auto offsetXOpt = conversions::getRequiredProperty(rt, "rectifyQuad: options", opts, "offsetX"); + const auto clear = conversions::getRequiredProperty(rt, "rectifyQuad: options", opts, "clear"); + + std::array<::cv::Point2f, 4> quad; + for (std::size_t i = 0; i < 4; ++i) { + quad[i] = {conversions::asType(rt, "rectifyQuad: quad", quadArr.getValueAtIndex(rt, i * 2)), + conversions::asType(rt, "rectifyQuad: quad", quadArr.getValueAtIndex(rt, i * 2 + 1))}; + } + + auto srcLock = tensor::tryLockShared(rt, "rectifyQuad: src", src); + auto dstLock = tensor::tryLockUnique(rt, "rectifyQuad: dst", dst); + + const int cvType = CV_MAKETYPE(CV_8U, channels); + ::cv::Mat srcMat(src->shape_[0], src->shape_[1], cvType, src->data_.get()); + ::cv::Mat dstMat(recH, canvasW, cvType, dst->data_.get()); + + try { + const std::array<::cv::Point2f, 4> dstPts = { + ::cv::Point2f{0.0f, 0.0f}, + {static_cast(contentWidth), 0.0f}, + {static_cast(contentWidth), static_cast(recH)}, + {0.0f, static_cast(recH)}}; + const std::array<::cv::Point2f, 4> srcPts = {quad[0], quad[1], quad[2], quad[3]}; + ::cv::Mat m = ::cv::getPerspectiveTransform(srcPts.data(), dstPts.data()); + ::cv::Mat content; + ::cv::warpPerspective(srcMat, content, m, ::cv::Size(contentWidth, recH), + ::cv::INTER_CUBIC, ::cv::BORDER_REPLICATE); + + ::cv::Scalar padColor; + if (padMode == "cornerMean") { + const int patch = std::max(1, std::min(recH, contentWidth) / 30); + ::cv::Scalar acc(0, 0, 0, 0); + const std::array<::cv::Rect, 4> rects = { + ::cv::Rect(0, 0, patch, patch), + ::cv::Rect(contentWidth - patch, 0, patch, patch), + ::cv::Rect(0, recH - patch, patch, patch), + ::cv::Rect(contentWidth - patch, recH - patch, patch, patch)}; + for (const auto &r : rects) { + acc += ::cv::mean(content(r)); + } + padColor = acc / 4.0; + } else { + padColor = ::cv::Scalar::all(padValue); + } + + if (clear) { + dstMat.setTo(padColor); + } + int32_t offsetX = offsetXOpt; + if (offsetX < 0) { + offsetX = (align == "center") ? (canvasW - contentWidth) / 2 : 0; + } + if (offsetX < canvasW) { + const int32_t copyW = std::min(contentWidth, canvasW - offsetX); + content(::cv::Rect(0, 0, copyW, recH)) + .copyTo(dstMat(::cv::Rect(offsetX, 0, copyW, recH))); + } + } catch (const std::exception &e) { + throw jsi::JSError(rt, std::string("rectifyQuad: OpenCV error: ") + e.what()); + } + return jsi::Value(rt, args[1]); + }; + module.setProperty(rt, name, + jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), + 4, fnBody)); +} + } // namespace rnexecutorch::extensions::cv::image_ops diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.h b/packages/react-native-executorch/cpp/extensions/cv/image_ops.h index 893c0957e5..27419f4da1 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.h +++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.h @@ -9,4 +9,7 @@ void install_toChannelsFirst(facebook::jsi::Runtime &rt, facebook::jsi::Object & void install_toChannelsLast(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); void install_normalize(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); void install_applyColormap(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_rotate(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_warpByGrid(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_rectifyQuad(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); } // namespace rnexecutorch::extensions::cv::image_ops diff --git a/packages/react-native-executorch/cpp/extensions/cv/install.cpp b/packages/react-native-executorch/cpp/extensions/cv/install.cpp index 0540f45c9b..01fa8525f4 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/install.cpp +++ b/packages/react-native-executorch/cpp/extensions/cv/install.cpp @@ -1,6 +1,7 @@ #include "install.h" #include "box_ops.h" #include "image_ops.h" +#include "ocr_ops.h" namespace rnexecutorch::extensions::cv { namespace jsi = facebook::jsi; @@ -14,10 +15,17 @@ void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) { image_ops::install_toChannelsLast(rt, cvModule); image_ops::install_normalize(rt, cvModule); image_ops::install_applyColormap(rt, cvModule); + image_ops::install_rotate(rt, cvModule); + image_ops::install_warpByGrid(rt, cvModule); + image_ops::install_rectifyQuad(rt, cvModule); box_ops::install_nms(rt, cvModule); box_ops::install_restrictToBox(rt, cvModule); + ocr_ops::install_extractCraftTextBoxes(rt, cvModule); + ocr_ops::install_extractDbnetTextBoxes(rt, cvModule); + ocr_ops::install_ctcGreedyDecode(rt, cvModule); + module.setProperty(rt, "cv", cvModule); } } // namespace rnexecutorch::extensions::cv diff --git a/packages/react-native-executorch/cpp/extensions/cv/ocr_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/ocr_ops.cpp new file mode 100644 index 0000000000..dfd0a9fbf6 --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/cv/ocr_ops.cpp @@ -0,0 +1,396 @@ +#include "ocr_ops.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "core/conversions.h" +#include "core/dtype.h" +#include "core/tensor.h" +#include "core/tensor_helpers.h" + +namespace rnexecutorch::extensions::cv::ocr_ops { +namespace jsi = facebook::jsi; +namespace tensor = rnexecutorch::core::tensor; +namespace conversions = rnexecutorch::core::conversions; +using DType = rnexecutorch::core::types::DType; + +namespace { +// Drop CRAFT connected components smaller than this (detector-input px²). +constexpr int32_t kMinComponentArea = 10; + +// Axis-aligned box (p0 = min, p1 = max) plus the rotated-rect angle. Line grouping +// and de-skew live in TypeScript; native only produces these component boxes. +struct Box { + float x0{}, y0{}, x1{}, y1{}; + float angle = 0.0f; +}; + +using Quad = std::array<::cv::Point2f, 4>; + +// ------------------------------ CRAFT branch ------------------------------- +// All per-component work stays inside the component's (dilation-padded) rect — +// full-frame masks per component would make the decode O(components · H · W). +std::optional boxFromComponent(const ::cv::Mat &textMap, const ::cv::Mat &labels, + const ::cv::Mat &stats, int32_t i, int32_t imgW, int32_t imgH, + float lowTextThreshold) { + const int32_t area = stats.at(i, ::cv::CC_STAT_AREA); + if (area < kMinComponentArea) { + return std::nullopt; + } + const int32_t x = stats.at(i, ::cv::CC_STAT_LEFT); + const int32_t y = stats.at(i, ::cv::CC_STAT_TOP); + const int32_t w = stats.at(i, ::cv::CC_STAT_WIDTH); + const int32_t h = stats.at(i, ::cv::CC_STAT_HEIGHT); + const ::cv::Rect compRect(x, y, w, h); + ::cv::Mat mask = (labels(compRect) == i); + double maxVal = 0.0; + ::cv::minMaxLoc(textMap(compRect), nullptr, &maxVal, nullptr, nullptr, mask); + if (maxVal < static_cast(lowTextThreshold)) { + return std::nullopt; + } + + const auto dilationRadius = + static_cast(std::sqrt(static_cast(area) / std::max(w, h)) * 2); + const int32_t sx = std::max(x - dilationRadius, 0); + const int32_t ex = std::min(x + w + dilationRadius, imgW); + const int32_t sy = std::max(y - dilationRadius, 0); + const int32_t ey = std::min(y + h + dilationRadius, imgH); + ::cv::Mat segMap = ::cv::Mat::zeros(ey - sy, ex - sx, CV_8U); + segMap(::cv::Rect(x - sx, y - sy, w, h)).setTo(255, mask); + const int32_t kSize = 1 + dilationRadius; + ::cv::Mat kernel = ::cv::getStructuringElement(::cv::MORPH_RECT, ::cv::Size(kSize, kSize)); + ::cv::dilate(segMap, segMap, kernel, ::cv::Point(-1, -1), 1); + + std::vector> contours; + // The offset restores full-frame coordinates for the ROI-local contours. + ::cv::findContours(segMap, contours, ::cv::RETR_EXTERNAL, ::cv::CHAIN_APPROX_SIMPLE, + ::cv::Point(sx, sy)); + if (contours.empty()) { + return std::nullopt; + } + ::cv::RotatedRect rr = ::cv::minAreaRect(contours[0]); + std::array<::cv::Point2f, 4> v; + rr.points(v.data()); + Box box; + box.x0 = std::min({v[0].x, v[1].x, v[2].x, v[3].x}); + box.y0 = std::min({v[0].y, v[1].y, v[2].y, v[3].y}); + box.x1 = std::max({v[0].x, v[1].x, v[2].x, v[3].x}); + box.y1 = std::max({v[0].y, v[1].y, v[2].y, v[3].y}); + box.angle = rr.angle; + return box; +} + +// CRAFT text+affinity maps -> one component box each. charLevel=false ADDS +// affinity to link adjacent glyphs into line-regions (keeping the rotated-rect +// angle); charLevel=true SUBTRACTS it (+ erode/dilate) to break them into upright +// per-glyph boxes (angle 0) for the stacked-column reader. +std::vector componentBoxes(::cv::Mat &textMap, ::cv::Mat &affinityMap, float textThreshold, + float linkThreshold, float lowTextThreshold, bool charLevel) { + const int32_t imgH = textMap.rows; + const int32_t imgW = textMap.cols; + ::cv::Mat textScore; + ::cv::Mat affinityScore; + ::cv::threshold(textMap, textScore, static_cast(textThreshold), 1.0, ::cv::THRESH_BINARY); + ::cv::threshold(affinityMap, affinityScore, static_cast(linkThreshold), 1.0, + ::cv::THRESH_BINARY); + + ::cv::Mat comb; + if (charLevel) { + comb = textScore - affinityScore; // subtract to separate adjacent glyphs + ::cv::threshold(comb, comb, 0.0, 1.0, ::cv::THRESH_TOZERO); + ::cv::threshold(comb, comb, 1.0, 1.0, ::cv::THRESH_TRUNC); + ::cv::Mat kernel = ::cv::getStructuringElement(::cv::MORPH_RECT, ::cv::Size(3, 3)); + ::cv::erode(comb, comb, kernel, ::cv::Point(-1, -1), 1); + ::cv::dilate(comb, comb, kernel, ::cv::Point(-1, -1), 4); + } else { + comb = textScore + affinityScore; // add to link adjacent glyphs into lines + ::cv::threshold(comb, comb, 0.0, 1.0, ::cv::THRESH_BINARY); + } + + ::cv::Mat binary; + comb.convertTo(binary, CV_8UC1); + ::cv::Mat labels; + ::cv::Mat stats; + ::cv::Mat centroids; + const int32_t nLabels = ::cv::connectedComponentsWithStats(binary, labels, stats, centroids, 4); + + std::vector boxes; + boxes.reserve(static_cast(nLabels)); + for (int32_t i = 1; i < nLabels; ++i) { + auto box = boxFromComponent(textMap, labels, stats, i, imgW, imgH, lowTextThreshold); + if (box) { + if (charLevel) { + box->angle = 0.0f; // glyphs are read upright, never rotated + } + boxes.push_back(*box); + } + } + return boxes; +} + +// CRAFT half-res heatmap (text+affinity interleaved) -> component boxes in +// detector-input pixels; restoreRatio scales the half-res boxes back up. Line +// grouping and de-skew are done in TypeScript; charLevel yields upright per-glyph +// boxes. `data` points at heatW*heatH*2 floats. +std::vector extractCraft(float *data, int32_t heatW, int32_t heatH, float textThreshold, + float linkThreshold, float lowTextThreshold, float restoreRatio, + bool charLevel) { + // Deinterleave the [text, affinity] channels of the half-res heatmap. + ::cv::Mat interleaved(heatH, heatW, CV_32FC2, data); + std::array<::cv::Mat, 2> channels; + ::cv::split(interleaved, channels); + std::vector boxes = componentBoxes(channels[0], channels[1], textThreshold, linkThreshold, + lowTextThreshold, charLevel); + for (auto &b : boxes) { + b.x0 *= restoreRatio; + b.y0 *= restoreRatio; + b.x1 *= restoreRatio; + b.y1 *= restoreRatio; + } + return boxes; +} + +// ------------------------------ DBNet branch ------------------------------- +// DBNet prob map [H,W] -> oriented quads. The map must be post-sigmoid +// probabilities — any activation is baked into the model's export. +std::vector extractDbnet(const ::cv::Mat &prob, float binThreshold, float boxThreshold, + float unclipRatio, int32_t minBoxSide, int32_t maxCandidates) { + const int32_t w = prob.cols; + const int32_t h = prob.rows; + + ::cv::Mat bitmap; + ::cv::threshold(prob, bitmap, static_cast(binThreshold), 255, ::cv::THRESH_BINARY); + bitmap.convertTo(bitmap, CV_8UC1); + + std::vector> contours; + ::cv::findContours(bitmap, contours, ::cv::RETR_LIST, ::cv::CHAIN_APPROX_SIMPLE); + + std::vector quads; + const int32_t maxN = static_cast( + std::min(contours.size(), static_cast(maxCandidates))); + for (int32_t i = 0; i < maxN; ++i) { + const auto &contour = contours[static_cast(i)]; + if (contour.size() < 4) { + continue; + } + ::cv::RotatedRect rr = ::cv::minAreaRect(contour); + if (std::min(rr.size.width, rr.size.height) < static_cast(minBoxSide)) { + continue; + } + // Score inside the contour's bounding rect only — a full-frame mask per + // candidate would make scoring O(candidates · H · W). + const ::cv::Rect bounds = ::cv::boundingRect(contour); + ::cv::Mat mask = ::cv::Mat::zeros(bounds.size(), CV_8UC1); + ::cv::drawContours(mask, contours, i, ::cv::Scalar(255), ::cv::FILLED, ::cv::LINE_8, + ::cv::noArray(), 0, -bounds.tl()); + const float score = static_cast(::cv::mean(prob(bounds), mask)[0]); + if (score < boxThreshold) { + continue; + } + const double area = static_cast(rr.size.width) * static_cast(rr.size.height); + const double perim = + 2.0 * (static_cast(rr.size.width) + static_cast(rr.size.height)); + const double distance = perim > 0.0 ? area * static_cast(unclipRatio) / perim : 0.0; + const auto grow = static_cast(2.0 * distance); + ::cv::RotatedRect expanded(rr.center, + ::cv::Size2f(rr.size.width + grow, rr.size.height + grow), + rr.angle); + if (std::min(expanded.size.width, expanded.size.height) < + static_cast(minBoxSide + 2)) { + continue; + } + std::array<::cv::Point2f, 4> c; + expanded.points(c.data()); + Quad q; + auto minX = static_cast(w); + auto minY = static_cast(h); + float maxX = 0; + float maxY = 0; + for (int32_t k = 0; k < 4; ++k) { + const float px = std::clamp(c[static_cast(k)].x, 0.0f, static_cast(w)); + const float py = std::clamp(c[static_cast(k)].y, 0.0f, static_cast(h)); + q[static_cast(k)] = {px, py}; + minX = std::min(minX, px); + minY = std::min(minY, py); + maxX = std::max(maxX, px); + maxY = std::max(maxY, py); + } + if (maxX - minX < 1.0f || maxY - minY < 1.0f) { + continue; + } + quads.push_back(q); + } + // Output order is unspecified — the TypeScript pipeline derives reading + // order geometrically for every result set. + return quads; +} + +// Flatten quads to a JS double array, 8 per box (x0,y0..x3,y3). +jsi::Array quadsToArray(jsi::Runtime &rt, const std::vector &quads) { + jsi::Array out(rt, quads.size() * 8); + size_t idx = 0; + for (const auto &q : quads) { + for (std::size_t k = 0; k < 4; ++k) { + out.setValueAtIndex(rt, idx++, jsi::Value(static_cast(q[k].x))); + out.setValueAtIndex(rt, idx++, jsi::Value(static_cast(q[k].y))); + } + } + return out; +} + +// Flatten component boxes to a JS double array, 5 per box (x0,y0,x1,y1,angle). +jsi::Array boxesToArray(jsi::Runtime &rt, const std::vector &boxes) { + jsi::Array out(rt, boxes.size() * 5); + size_t idx = 0; + for (const auto &b : boxes) { + out.setValueAtIndex(rt, idx++, jsi::Value(static_cast(b.x0))); + out.setValueAtIndex(rt, idx++, jsi::Value(static_cast(b.y0))); + out.setValueAtIndex(rt, idx++, jsi::Value(static_cast(b.x1))); + out.setValueAtIndex(rt, idx++, jsi::Value(static_cast(b.y1))); + out.setValueAtIndex(rt, idx++, jsi::Value(static_cast(b.angle))); + } + return out; +} + +} // namespace + +void install_extractCraftTextBoxes(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "extractCraftTextBoxes"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value &, const jsi::Value *args, + size_t count) -> jsi::Value { + if (count != 2) { + throw jsi::JSError(rt, "Usage: extractCraftTextBoxes(src, options)"); + } + const char *ctx = "extractCraftTextBoxes"; + auto src = tensor::fromJs(rt, "extractCraftTextBoxes: src", args[0], DType::float32, std::nullopt); + const auto opts = conversions::asType(rt, "extractCraftTextBoxes: options", args[1]); + auto srcLock = tensor::tryLockShared(rt, "extractCraftTextBoxes: src", src); + auto *dataPtr = reinterpret_cast(src->data_.get()); + + // src is [1,Hd,Wd,2] or [Hd,Wd,2] interleaved (text, affinity), half-res. + const auto &s = src->shape_; + if (s.size() < 3 || s.back() != 2) { + throw jsi::JSError(rt, "extractCraftTextBoxes: src must be [..,Hd,Wd,2]"); + } + const int32_t heatW = s[s.size() - 2]; + const int32_t heatH = s[s.size() - 3]; + if (static_cast(heatW) * static_cast(heatH) * 2 != src->numel_) { + throw jsi::JSError(rt, "extractCraftTextBoxes: src Hd*Wd*2 does not match numel"); + } + const auto targetH = conversions::getRequiredProperty(rt, ctx, opts, "targetHeight"); + const float restoreRatio = static_cast(targetH) / static_cast(heatH); + const bool charLevel = conversions::getRequiredProperty(rt, ctx, opts, "charLevel"); + + std::vector boxes; + try { + boxes = extractCraft( + dataPtr, heatW, heatH, + static_cast(conversions::getRequiredProperty(rt, ctx, opts, "textThreshold")), + static_cast(conversions::getRequiredProperty(rt, ctx, opts, "linkThreshold")), + static_cast(conversions::getRequiredProperty(rt, ctx, opts, "lowTextThreshold")), + restoreRatio, charLevel); + } catch (const std::exception &e) { + throw jsi::JSError(rt, std::string("extractCraftTextBoxes: OpenCV error: ") + e.what()); + } + return boxesToArray(rt, boxes); + }; + module.setProperty(rt, name, + jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), + 2, fnBody)); +} + +void install_extractDbnetTextBoxes(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "extractDbnetTextBoxes"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value &, const jsi::Value *args, + size_t count) -> jsi::Value { + if (count != 2) { + throw jsi::JSError(rt, "Usage: extractDbnetTextBoxes(src, options)"); + } + const char *ctx = "extractDbnetTextBoxes"; + auto src = tensor::fromJs(rt, "extractDbnetTextBoxes: src", args[0], DType::float32, std::nullopt); + const auto opts = conversions::asType(rt, "extractDbnetTextBoxes: options", args[1]); + auto srcLock = tensor::tryLockShared(rt, "extractDbnetTextBoxes: src", src); + auto *dataPtr = reinterpret_cast(src->data_.get()); + + // src is [1,1,H,W] or [H,W] probability map (full-res). + const auto &s = src->shape_; + if (s.size() < 2) { + throw jsi::JSError(rt, "extractDbnetTextBoxes: src must be [..,H,W]"); + } + const int32_t w = s[s.size() - 1]; + const int32_t h = s[s.size() - 2]; + if (static_cast(w) * static_cast(h) != src->numel_) { + throw jsi::JSError(rt, "extractDbnetTextBoxes: src H*W does not match numel"); + } + + std::vector quads; + try { + ::cv::Mat prob(h, w, CV_32F, dataPtr); + quads = extractDbnet( + prob, static_cast(conversions::getRequiredProperty(rt, ctx, opts, "binThreshold")), + static_cast(conversions::getRequiredProperty(rt, ctx, opts, "boxThreshold")), + static_cast(conversions::getRequiredProperty(rt, ctx, opts, "unclipRatio")), + conversions::getRequiredProperty(rt, ctx, opts, "minBoxSide"), + conversions::getRequiredProperty(rt, ctx, opts, "maxCandidates")); + } catch (const std::exception &e) { + throw jsi::JSError(rt, std::string("extractDbnetTextBoxes: OpenCV error: ") + e.what()); + } + return quadsToArray(rt, quads); + }; + module.setProperty(rt, name, + jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), + 2, fnBody)); +} + +// --------------------------- ctcGreedyDecode ------------------------------- +// Per-timestep argmax + max value over [..,T,V] logits. `values` are the raw +// max activations; if a caller needs probabilities it softmaxes the tensor (via +// the math.softmax op) before decoding — this op takes no options. +void install_ctcGreedyDecode(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "ctcGreedyDecode"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value &, const jsi::Value *args, + size_t count) -> jsi::Value { + if (count != 1) { + throw jsi::JSError(rt, "Usage: ctcGreedyDecode(src)"); + } + auto src = tensor::fromJs(rt, "ctcGreedyDecode: src", args[0], DType::float32, std::nullopt); + auto srcLock = tensor::tryLockShared(rt, "ctcGreedyDecode: src", src); + + const auto &s = src->shape_; + if (s.size() < 2) { + throw jsi::JSError(rt, "ctcGreedyDecode: src must be at least 2-D [..,T,V]"); + } + const int32_t vocab = s.back(); + if (vocab < 1) { + throw jsi::JSError(rt, "ctcGreedyDecode: vocab dimension must be >= 1"); + } + if (src->numel_ % static_cast(vocab) != 0) { + throw jsi::JSError(rt, "ctcGreedyDecode: numel must be a multiple of the vocab dim"); + } + const int32_t timesteps = static_cast(src->numel_) / vocab; + const auto *data = reinterpret_cast(src->data_.get()); + + jsi::Array out(rt, static_cast(timesteps) * 2); + size_t oi = 0; + for (int32_t t = 0; t < timesteps; ++t) { + const float *row = data + static_cast(t) * static_cast(vocab); + const float *maxIt = std::max_element(row, row + vocab); + const auto maxIdx = static_cast(maxIt - row); + out.setValueAtIndex(rt, oi++, jsi::Value(static_cast(maxIdx))); + out.setValueAtIndex(rt, oi++, jsi::Value(static_cast(*maxIt))); + } + return out; + }; + module.setProperty(rt, name, + jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), + 1, fnBody)); +} + +} // namespace rnexecutorch::extensions::cv::ocr_ops diff --git a/packages/react-native-executorch/cpp/extensions/cv/ocr_ops.h b/packages/react-native-executorch/cpp/extensions/cv/ocr_ops.h new file mode 100644 index 0000000000..1e47193bb0 --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/cv/ocr_ops.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace rnexecutorch::extensions::cv::ocr_ops { +void install_extractCraftTextBoxes(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_extractDbnetTextBoxes(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_ctcGreedyDecode(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::cv::ocr_ops diff --git a/packages/react-native-executorch/cpp/extensions/cv/utils.h b/packages/react-native-executorch/cpp/extensions/cv/utils.h index 710124b1d9..edc1f0b7c1 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/utils.h +++ b/packages/react-native-executorch/cpp/extensions/cv/utils.h @@ -1,8 +1,10 @@ #pragma once #include "core/dtype.h" +#include #include #include +#include namespace rnexecutorch::extensions::cv { From f5f75d45da4fef322377d3b747a50e42358d95ae Mon Sep 17 00:00:00 2001 From: benITo47 Date: Mon, 20 Jul 2026 17:09:26 +0200 Subject: [PATCH 02/13] [RNE Rewrite] feat(ocr): pluggable detector extractors (CRAFT, DBNet) --- .../src/extensions/cv/tasks/ocr/detectors.ts | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 packages/react-native-executorch/src/extensions/cv/tasks/ocr/detectors.ts diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/ocr/detectors.ts b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/detectors.ts new file mode 100644 index 0000000000..eaedc31427 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/detectors.ts @@ -0,0 +1,158 @@ +// Built-in detector box-extraction strategies (DBNet, CRAFT): each turns one +// architecture's native detect output into quads. The pipeline is +// model-agnostic — it just calls OcrModelOptions.extractBoxes. + +import { rnexecutorchJsi } from '../../../../native/bridge'; +import type { Tensor } from '../../../../core/tensor'; +import { quadsFromFlat, type Quad } from '../../ops/quad'; +import { boxesFromFlat, groupBoxes, boxToQuad } from './geometry'; + +/** + * A detector's box-extraction strategy. Plug a new detector architecture into the + * OCR pipeline by supplying an object of this type (a built-in below, or your + * own). Both methods MUST be worklets. + * @category Types + */ +export type TextBoxExtractor = { + /** + * The `float32` output tensor shapes the `detect` method produces for a given + * detector input size, so the caller can pre-allocate them. One shape per + * `detect` output, in order. + * @param inputSize The detector input size the image was letterboxed to. + * @returns One shape per `detect` output tensor. + */ + readonly outputShapes: (inputSize: { + readonly width: number; + readonly height: number; + }) => number[][]; + /** + * Turns the model's `detect` output tensors into oriented {@link Quad}s in + * detector-input pixel space. + * @param outputs The model's `detect` output tensors, in order. + * @param inputSize The detector input size the image was letterboxed to. + * @param charLevel Emit one box per glyph instead of grouped lines; strategies + * without a char-level mode ignore it. + * @returns Oriented quads (TL, TR, BR, BL) in detector-input pixel space. + */ + readonly extract: ( + outputs: readonly Tensor[], + inputSize: { readonly width: number; readonly height: number }, + charLevel: boolean + ) => Quad[]; +}; + +/** + * CRAFT decode thresholds — tuning knobs for {@link makeCraftExtractBoxes}. All + * optional; the defaults suit the built-in CRAFT models. + * @category Types + */ +export type CraftExtractOptions = { + /** Region-score threshold for a pixel to count as text. Default 0.4. */ + readonly textThreshold?: number; + /** Affinity-score threshold for linking adjacent glyphs into a line. Default 0.4. */ + readonly linkThreshold?: number; + /** Minimum peak region score for a component to survive. Default 0.7. */ + readonly lowTextThreshold?: number; +}; + +/** + * Builds a CRAFT box-extraction strategy: native region+affinity heatmap decode + * (`outputs[0]` = `[1, Hd, Wd, 2]`), then line grouping + de-skew in TS — + * per-glyph boxes when `charLevel`. Detector input dims must be even (half-res + * heatmap). {@link craftExtractBoxes} is this with default thresholds. + * @param opts Threshold overrides; omit for the built-in defaults. + * @returns A {@link TextBoxExtractor} bound to those thresholds. + * @category Typescript API + */ +export function makeCraftExtractBoxes(opts: CraftExtractOptions = {}): TextBoxExtractor { + const textThreshold = opts.textThreshold ?? 0.4; + const linkThreshold = opts.linkThreshold ?? 0.4; + const lowTextThreshold = opts.lowTextThreshold ?? 0.7; + return { + outputShapes: ({ width, height }) => { + 'worklet'; + if (width % 2 !== 0 || height % 2 !== 0) { + throw new Error( + 'OCR: CRAFT detect input dimensions must be even (half-resolution heatmap).' + ); + } + return [[1, height / 2, width / 2, 2]]; + }, + extract: (outputs, inputSize, charLevel) => { + 'worklet'; + const flat = rnexecutorchJsi.cv.extractCraftTextBoxes(outputs[0]!, { + textThreshold, + linkThreshold, + lowTextThreshold, + targetHeight: inputSize.height, + charLevel, + }) as number[]; + // Char-level glyphs are read individually, so they skip line grouping. + const boxes = boxesFromFlat(flat); + return (charLevel ? boxes : groupBoxes(boxes)).map(boxToQuad); + }, + }; +} + +/** + * CRAFT box-extraction strategy with default thresholds. + * @category Typescript API + */ +export const craftExtractBoxes: TextBoxExtractor = makeCraftExtractBoxes(); + +/** + * DBNet decode thresholds — tuning knobs for {@link makeDbnetExtractBoxes}. All + * optional; the defaults suit the built-in DBNet models. + * @category Types + */ +export type DbnetExtractOptions = { + /** Binarization threshold on the probability map. Default 0.3. */ + readonly binThreshold?: number; + /** Minimum mean box score to keep a candidate. Default 0.6. */ + readonly boxThreshold?: number; + /** How far to expand (unclip) each shrunk box. Default 1.5. */ + readonly unclipRatio?: number; + /** Discard boxes with a side smaller than this (px). Default 3. */ + readonly minBoxSide?: number; + /** Cap on contour candidates scored per map. Default 1000. */ + readonly maxCandidates?: number; +}; + +/** + * Builds a DBNet box-extraction strategy: thresholds and unclips the full-res + * post-sigmoid probability map (`outputs[0]` = `[1, 1, H, W]`) into oriented + * quads. No char-level mode. {@link dbnetExtractBoxes} is this with defaults. + * @param opts Threshold overrides; omit for the built-in defaults. + * @returns A {@link TextBoxExtractor} bound to those thresholds. + * @category Typescript API + */ +export function makeDbnetExtractBoxes(opts: DbnetExtractOptions = {}): TextBoxExtractor { + const binThreshold = opts.binThreshold ?? 0.3; + const boxThreshold = opts.boxThreshold ?? 0.6; + const unclipRatio = opts.unclipRatio ?? 1.5; + const minBoxSide = opts.minBoxSide ?? 3; + const maxCandidates = opts.maxCandidates ?? 1000; + return { + outputShapes: ({ width, height }) => { + 'worklet'; + return [[1, 1, height, width]]; + }, + extract: (outputs) => { + 'worklet'; + const flat = rnexecutorchJsi.cv.extractDbnetTextBoxes(outputs[0]!, { + binThreshold, + boxThreshold, + unclipRatio, + minBoxSide, + maxCandidates, + }) as number[]; + return quadsFromFlat(flat); + }, + }; +} + +/** + * DBNet box-extraction strategy with default thresholds. + * @category Typescript API + */ +export const dbnetExtractBoxes: TextBoxExtractor = makeDbnetExtractBoxes(); From c806ec376f56f555ee179d0a891e088175218cfd Mon Sep 17 00:00:00 2001 From: benITo47 Date: Mon, 20 Jul 2026 17:09:29 +0200 Subject: [PATCH 03/13] =?UTF-8?q?[RNE=20Rewrite]=20feat(ocr):=20document?= =?UTF-8?q?=20models=20=E2=80=94=20orientation,=20dewarp,=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../extensions/cv/tasks/ocr/documentModels.ts | 468 ++++++++++++++++++ 1 file changed, 468 insertions(+) create mode 100644 packages/react-native-executorch/src/extensions/cv/tasks/ocr/documentModels.ts diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/ocr/documentModels.ts b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/documentModels.ts new file mode 100644 index 0000000000..50810d981a --- /dev/null +++ b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/documentModels.ts @@ -0,0 +1,468 @@ +import type { WorkletRuntime } from 'react-native-worklets'; + +import { tensor, type Tensor } from '../../../../core/tensor'; +import { loadModel, type Model } from '../../../../core/model'; +import { validateModelSchema, SymbolicTensor } from '../../../../core/modelSchema'; +import { wrapAsync } from '../../../../core/runtime'; + +import { IMAGENET_NORM } from '../../../../constants'; +import type { ImageBuffer, ImageFormat } from '../../image'; +import { FORMAT_CHANNELS, warpByGrid } from '../../ops/image'; +import { boundsOfPoints } from '../../ops/quad'; +import { createImagePreprocessor } from '../preprocessing'; +import type { OcrDetection } from './engine'; + +/** A detected page orientation: clockwise rotation and classifier confidence `[0,1]`. */ +export type Orientation = { + readonly rotationCW: 0 | 90 | 180 | 270; + readonly confidence: number; +}; + +/** + * Table-structure recognition parameters: `structureVocab` maps a token id to its + * HTML fragment, `eosTokenId` ends decoding, `maxSteps` caps it. + * @category Types + */ +export type TableConfig = { + readonly structureVocab: readonly string[]; + readonly eosTokenId: number; + readonly maxSteps: number; +}; + +/** + * Config for the fused document-helpers model (orientation + dewarp + table + * structure in one file). Every capability is opt-in; `orientation`/`dewarp` are + * defaults for the per-run flags of the same name. + * @category Types + */ +export type DocumentModelsConfig = { + readonly modelPath: string; + readonly orientation?: boolean; + readonly dewarp?: boolean; + readonly orientationMinConfidence?: number; + /** + * Decline the dewarp when it keeps less than this fraction of the source + * image variance (a bad grid can collapse a page without clear borders to near-blank). + * Default 0.5; lower = more permissive. + */ + readonly dewarpMinVarianceRatio?: number; + readonly table?: TableConfig; +}; + +type ImagePreprocessor = ReturnType; + +// Index of the max value in `arr[offset, offset+len)`. Shared by orientation + table. +function argmaxRange(arr: ArrayLike, offset: number, len: number): number { + 'worklet'; + let index = 0; + let best = arr[offset]!; + for (let i = 1; i < len; i++) { + if (arr[offset + i]! > best) { + best = arr[offset + i]!; + index = i; + } + } + return index; +} + +// ===== Orientation =========================================================== + +function detectPageOrientation( + model: Model, + preprocessor: ImagePreprocessor, + tOri: Tensor, + oriBuf: Float32Array, + page: Tensor, + format: ImageFormat +): Orientation { + 'worklet'; + const tInput = preprocessor.processTensor(page, format); + model.execute('orientation', [tInput], [tOri]); + tOri.getData(oriBuf); + const cls = argmaxRange(oriBuf, 0, oriBuf.length); + const best = oriBuf[cls]!; + let sumExp = 0; + for (let i = 0; i < oriBuf.length; i++) { + sumExp += Math.exp(oriBuf[i]! - best); + } + return { rotationCW: (cls * 90) as 0 | 90 | 180 | 270, confidence: 1 / sumExp }; +} + +// ===== Dewarp ================================================================ + +// Default for DocumentModelsConfig.dewarpMinVarianceRatio: a grid estimated on a +// page without clear borders can collapse the output to near-blank, so decline the warp +// when it keeps less than this fraction of the source variance. +const DEFAULT_DEWARP_MIN_VARIANCE_RATIO = 0.5; +const VARIANCE_SAMPLE_STRIDE = 31; + +// Variance of one channel, sampled sparsely — a cheap proxy for image content. +function pixelVariance(data: Uint8Array, channels: number): number { + 'worklet'; + let n = 0; + let sum = 0; + let sumSq = 0; + const step = channels * VARIANCE_SAMPLE_STRIDE; + for (let i = 0; i < data.length; i += step) { + sum += data[i]!; + sumSq += data[i]! * data[i]!; + n++; + } + if (n === 0) { + return 0; + } + return sumSq / n - (sum / n) * (sum / n); +} + +// Estimates the sampling field and applies it (cv::remap). Returns the input +// `page` unchanged when the warp is declined; caller owns whichever is returned. +function dewarpPage( + model: Model, + preprocessor: ImagePreprocessor, + tGrid: Tensor, + page: Tensor, + format: ImageFormat, + minVarianceRatio: number +): Tensor { + 'worklet'; + const tInput = preprocessor.processTensor(page, format); + model.execute('dewarp', [tInput], [tGrid]); + const h = page.shape[0]!; + const w = page.shape[1]!; + const ch = FORMAT_CHANNELS[format]; + const tDst = tensor('uint8', [h, w, ch]); + try { + warpByGrid(page, tGrid, tDst); + const out = new Uint8Array(w * h * ch); + const src = new Uint8Array(w * h * ch); + tDst.getData(out); + page.getData(src); + if (pixelVariance(out, ch) < minVarianceRatio * pixelVariance(src, ch)) { + tDst.dispose(); + return page; + } + return tDst; + } catch (e) { + tDst.dispose(); // caller can't free it on the throw path + throw e; + } +} + +// ===== Table structure ======================================================= + +/** The table decoder's pre-allocated tensors and staging buffers. */ +type TableDecodeState = { + readonly tFeatures: Tensor; + readonly tHidden: Tensor; + readonly tOnehot: Tensor; + readonly tProbs: Tensor; + readonly tNewHidden: Tensor; + readonly zeroHidden: Float32Array; + readonly zeroVocab: Float32Array; + readonly onehotBuf: Float32Array; + readonly probsBuf: Float32Array; +}; + +// Decodes the table's `/` HTML skeleton token-by-token; cell text is +// filled in later from the region's OCR lines (fillTableCells). +function recognizeTableStructure( + model: Model, + preprocessor: ImagePreprocessor, + state: TableDecodeState, + tableConfig: TableConfig, + input: ImageBuffer +): string { + 'worklet'; + const { structureVocab, eosTokenId, maxSteps } = tableConfig; + const { tFeatures, tHidden, tOnehot, tProbs, tNewHidden } = state; + const tInput = preprocessor.process(input); + model.execute('table_encode', [tInput], [tFeatures]); + tHidden.setData(state.zeroHidden); + tOnehot.setData(state.zeroVocab); + let html = ''; + for (let step = 0; step < maxSteps; step++) { + model.execute('table_decode_step', [tFeatures, tHidden, tOnehot], [tProbs, tNewHidden]); + tProbs.getData(state.probsBuf); + const tok = argmaxRange(state.probsBuf, 0, state.probsBuf.length); + if (tok === eosTokenId) { + break; + } + // Drop the reserved start/end token range from the assembled skeleton. + if (tok > 0 && tok < eosTokenId && tok < structureVocab.length) { + html += structureVocab[tok]!; + } + tNewHidden.copyTo(tHidden); + state.onehotBuf.fill(0); + state.onehotBuf[tok] = 1; + tOnehot.setData(state.onehotBuf); + } + return html; +} + +// Clusters 1-D values into `k` groups at the k-1 widest gaps, each reduced to its +// mean — matches how table cells distribute (tight within a row/col, gaps between). +function clusterByGaps(values: readonly number[], k: number): number[] { + 'worklet'; + const sorted = [...values].sort((a, b) => a - b); + if (sorted.length <= k) { + return sorted; + } + const gaps = sorted.slice(1).map((value, i) => ({ at: i + 1, size: value - sorted[i]! })); + gaps.sort((a, b) => b.size - a.size); + const cuts = gaps + .slice(0, k - 1) + .map((gap) => gap.at) + .sort((a, b) => a - b); + const centers: number[] = []; + let prev = 0; + for (const cut of [...cuts, sorted.length]) { + const group = sorted.slice(prev, cut); + centers.push(group.reduce((sum, value) => sum + value, 0) / group.length); + prev = cut; + } + return centers; +} + +// Fills a table skeleton's cells with a region's OCR lines, assigning each line's +// box center to its nearest row/column cluster (document-order fallback if no grid). +export function fillTableCells(html: string, lines: readonly OcrDetection[]): string { + 'worklet'; + const rowCount = (html.match(//g) ?? []).length; + let colCount = 0; + const rowRegex = /([\s\S]*?)<\/tr>/g; + let row: RegExpExecArray | null; + while ((row = rowRegex.exec(html)) !== null) { + colCount = Math.max(colCount, (row[1]!.match(/ l.text); + } else { + const centersX: number[] = []; + const centersY: number[] = []; + for (const line of lines) { + const box = boundsOfPoints(line.quad, 'xyxy'); + centersX.push((box.xmin + box.xmax) / 2); + centersY.push((box.ymin + box.ymax) / 2); + } + const rowCenters = clusterByGaps(centersY, rowCount); + const colCenters = clusterByGaps(centersX, colCount); + const grid: string[][] = Array.from({ length: rowCenters.length }, () => + new Array(colCenters.length).fill('') + ); + for (let i = 0; i < lines.length; i++) { + const r = rowCenters.reduce( + (best, center, j) => + Math.abs(centersY[i]! - center) < Math.abs(centersY[i]! - rowCenters[best]!) ? j : best, + 0 + ); + const c = colCenters.reduce( + (best, center, j) => + Math.abs(centersX[i]! - center) < Math.abs(centersX[i]! - colCenters[best]!) ? j : best, + 0 + ); + grid[r]![c] = `${grid[r]![c]!} ${lines[i]!.text}`.trim(); + } + cellTexts = grid.flat(); + } + + // Fill the decoded skeleton's empty cells in order, preserving each cell's tag + // and span attributes rather than rebuilding the table from scratch. + let cell = 0; + return html.replace(/]*)><\/td>/g, (_match, attrs) => { + const text = cell < cellTexts.length ? cellTexts[cell]! : ''; + cell++; + return `${text}`; + }); +} + +// ===== Contract + factory ==================================================== + +// The tensor shapes the constructor pre-allocates from, grouped per model. +type DocumentModelsShapes = { + readonly orientation: { readonly input: number[]; readonly output: number[] }; + readonly dewarp: { readonly input: number[]; readonly grid: number[] }; + readonly table?: { + readonly input: number[]; + readonly features: number[]; + readonly hidden: number[]; + readonly probs: number[]; + }; +}; + +// Validates orientation + dewarp always, and the table methods only when a table +// vocab size is given. Returns the shapes to pre-allocate from. Throws on mismatch. +function resolveDocumentModelsShapes(model: Model, tableVocabSize?: number): DocumentModelsShapes { + const oriMeta = validateModelSchema( + model, + 'orientation', + [SymbolicTensor('float32', [1, 3, 'H', 'W'])], + [SymbolicTensor('float32', [1, 'K'])] + ); + const dewMeta = validateModelSchema( + model, + 'dewarp', + [SymbolicTensor('float32', [1, 3, 'H', 'W'])], + [SymbolicTensor('float32', [1, 2, 'gH', 'gW'])] + ); + const oriClasses = oriMeta.outputTensorMeta[0]!.shape[1]!; + if (oriClasses !== 4) { + throw new Error( + `DocumentModels: orientation head must output 4 classes ([0°, 90°, 180°, 270°]), ` + + `but the model declares ${oriClasses}.` + ); + } + const base = { + orientation: { + input: oriMeta.inputTensorMeta[0]!.shape, + output: oriMeta.outputTensorMeta[0]!.shape, + }, + dewarp: { input: dewMeta.inputTensorMeta[0]!.shape, grid: dewMeta.outputTensorMeta[0]!.shape }, + }; + if (tableVocabSize === undefined) { + return base; + } + + const encMeta = validateModelSchema( + model, + 'table_encode', + [SymbolicTensor('float32', [1, 3, 'H', 'W'])], + [SymbolicTensor('float32', [1, 'C', 'F'])] + ); + const decMeta = validateModelSchema( + model, + 'table_decode_step', + [ + SymbolicTensor('float32', [1, 'C', 'F']), + SymbolicTensor('float32', [1, 'H']), + SymbolicTensor('float32', [1, 'V']), + ], + [SymbolicTensor('float32', [1, 'V']), SymbolicTensor('float32', [1, 'H'])] + ); + const probs = decMeta.outputTensorMeta[0]!.shape; + // The decoder feeds each token back as a one-hot input, so its one-hot input dim + // must equal its output vocab (matchShape maps `V` per tensor, not across them). + const onehotVocab = decMeta.inputTensorMeta[2]!.shape[1]!; + if (onehotVocab !== probs[1]!) { + throw new Error( + `DocumentModels: table one-hot input vocab (${onehotVocab}) must match output vocab (${probs[1]}).` + ); + } + if (probs[1]! !== tableVocabSize) { + throw new Error( + `DocumentModels: structure vocab (${tableVocabSize}) must match the model token dim (${probs[1]}).` + ); + } + return { + ...base, + table: { + input: encMeta.inputTensorMeta[0]!.shape, + features: encMeta.outputTensorMeta[0]!.shape, + hidden: decMeta.outputTensorMeta[1]!.shape, + probs, + }, + }; +} + +// Creates the document-helpers runner from one model file, loaded once. Each +// capability is wired only when enabled in `config`. Internal to the OCR task. +export async function createDocumentModels( + config: DocumentModelsConfig, + runtime?: WorkletRuntime +): Promise<{ + dispose: () => void; + detectOrientationWorklet: (page: Tensor, format: ImageFormat) => Orientation; + dewarpWorklet: (page: Tensor, format: ImageFormat) => Tensor; + recognizeTableWorklet: (input: ImageBuffer) => string; +}> { + const { modelPath, table } = config; + const dewarpMinVarianceRatio = config.dewarpMinVarianceRatio ?? DEFAULT_DEWARP_MIN_VARIANCE_RATIO; + const model = await wrapAsync(loadModel, runtime)(modelPath); + + // Track each artifact as it's built so the catch can dispose them all on a + // mid-sequence failure (no native leak on a bad config). + const created: { dispose: () => void }[] = []; + try { + const s = resolveDocumentModelsShapes(model, table?.structureVocab.length); + + // Orientation + dewarp are always available (their methods are always present). + const orientationPreprocessor = createImagePreprocessor( + { resizeMode: 'stretch', interpolation: 'linear', ...IMAGENET_NORM }, + s.orientation.input + ); + created.push(orientationPreprocessor); + const dewarpPreprocessor = createImagePreprocessor( + { resizeMode: 'stretch', interpolation: 'linear', alpha: 1 / 255, beta: 0 }, + s.dewarp.input + ); + created.push(dewarpPreprocessor); + const tOri = tensor('float32', s.orientation.output); + created.push(tOri); + const tGrid = tensor('float32', s.dewarp.grid); + created.push(tGrid); + const oriBuf = new Float32Array(s.orientation.output[1]!); + + let tablePreprocessor: ImagePreprocessor | null = null; + let decodeState: TableDecodeState | null = null; + if (table && s.table) { + tablePreprocessor = createImagePreprocessor( + { resizeMode: 'stretch', interpolation: 'linear', ...IMAGENET_NORM }, + s.table.input + ); + created.push(tablePreprocessor); + const tensors = [ + tensor('float32', s.table.features), + tensor('float32', s.table.hidden), + tensor('float32', s.table.probs), + tensor('float32', s.table.probs), + tensor('float32', s.table.hidden), + ] as const; + tensors.forEach((t) => created.push(t)); + const [tFeatures, tHidden, tOnehot, tProbs, tNewHidden] = tensors; + const hidLen = s.table.hidden[1]!; + const vocabLen = s.table.probs[1]!; + decodeState = { + tFeatures, + tHidden, + tOnehot, + tProbs, + tNewHidden, + zeroHidden: new Float32Array(hidLen), + zeroVocab: new Float32Array(vocabLen), + onehotBuf: new Float32Array(vocabLen), + probsBuf: new Float32Array(vocabLen), + }; + } + + const dispose = () => { + created.forEach((c) => c.dispose()); + model.dispose(); + }; + + // Thin executor bindings — logic lives in the top-level worklets above. + const detectOrientationWorklet = (page: Tensor, format: ImageFormat): Orientation => { + 'worklet'; + return detectPageOrientation(model, orientationPreprocessor, tOri, oriBuf, page, format); + }; + const dewarpWorklet = (page: Tensor, format: ImageFormat): Tensor => { + 'worklet'; + return dewarpPage(model, dewarpPreprocessor, tGrid, page, format, dewarpMinVarianceRatio); + }; + const recognizeTableWorklet = (input: ImageBuffer): string => { + 'worklet'; + if (!tablePreprocessor || !decodeState || !table) { + throw new Error('DocumentModels: table recognition was not configured.'); + } + return recognizeTableStructure(model, tablePreprocessor, decodeState, table, input); + }; + + return { dispose, detectOrientationWorklet, dewarpWorklet, recognizeTableWorklet }; + } catch (e) { + created.forEach((c) => c.dispose()); + model.dispose(); + throw e; + } +} From 30049c4b874328f1759a11fb8bc300a8e2117b57 Mon Sep 17 00:00:00 2001 From: benITo47 Date: Mon, 20 Jul 2026 17:09:43 +0200 Subject: [PATCH 04/13] [RNE Rewrite] feat(ocr): two-stage OCR pipeline + reading order + app wiring --- .cspell-wordlist.txt | 38 + apps/computer-vision/app/_layout.tsx | 14 + apps/computer-vision/app/document/index.tsx | 387 ++++++++++ apps/computer-vision/app/index.tsx | 6 + apps/computer-vision/app/ocr/index.tsx | 300 ++++++++ .../components/ImageViewport.tsx | 55 +- .../react-native-executorch/src/constants.ts | 96 +++ .../react-native-executorch/src/core/model.ts | 41 ++ .../src/extensions/cv/ops/boxes.ts | 29 +- .../src/extensions/cv/ops/image.ts | 113 ++- .../src/extensions/cv/ops/index.ts | 1 + .../src/extensions/cv/ops/points.ts | 20 +- .../src/extensions/cv/ops/quad.ts | 259 +++++++ .../src/extensions/cv/tasks/ocr/charsets.ts | 23 + .../src/extensions/cv/tasks/ocr/engine.ts | 666 ++++++++++++++++++ .../src/extensions/cv/tasks/ocr/geometry.ts | 441 ++++++++++++ .../src/extensions/cv/tasks/ocr/ocr.ts | 355 ++++++++++ .../src/extensions/cv/tasks/preprocessing.ts | 38 +- .../src/hooks/useOcr.ts | 66 ++ packages/react-native-executorch/src/index.ts | 5 + .../react-native-executorch/src/models.ts | 265 +++++++ 21 files changed, 3174 insertions(+), 44 deletions(-) create mode 100644 apps/computer-vision/app/document/index.tsx create mode 100644 apps/computer-vision/app/ocr/index.tsx create mode 100644 packages/react-native-executorch/src/extensions/cv/ops/quad.ts create mode 100644 packages/react-native-executorch/src/extensions/cv/tasks/ocr/charsets.ts create mode 100644 packages/react-native-executorch/src/extensions/cv/tasks/ocr/engine.ts create mode 100644 packages/react-native-executorch/src/extensions/cv/tasks/ocr/geometry.ts create mode 100644 packages/react-native-executorch/src/extensions/cv/tasks/ocr/ocr.ts create mode 100644 packages/react-native-executorch/src/hooks/useOcr.ts diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 55fa014084..05bb2ec138 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -298,3 +298,41 @@ deis Coeff Multistep multistep +dbnet +svtr +softmaxed +softmax +unclip +cand +parameterizes +pyimagesearch +letterbox +CRNN +CRAFT +PaddleOCR +EasyOCR +cornerMean +ctc +Vatti +softmaxing +ppocrv +PPOCRV +ctcGreedyDecode +dewarp +vctx +onehot +slanet +letterboxed +redetect +redetections +eos +doclayout +dynint +softmaxes +hconcat +autoregressive +dewarped +dewarps +nums +ocrv +unclips diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx index 9be87a6759..4c598660e3 100644 --- a/apps/computer-vision/app/_layout.tsx +++ b/apps/computer-vision/app/_layout.tsx @@ -76,6 +76,20 @@ export default function Layout() { title: 'Instance Segmentation', }} /> + + b.platforms.includes(Platform.OS)); +const BACKEND_OPTIONS: ModelOption[] = AVAILABLE.map((b, i) => ({ label: b.label, value: i })); + +type Cell = { text: string; colspan: number; rowspan: number }; + +// Parse the SLANet structure HTML (filled) into rows of cells for rendering. +function parseTable(html: string): Cell[][] { + const rows: Cell[][] = []; + const trRe = /([\s\S]*?)<\/tr>/g; + let tr: RegExpExecArray | null; + while ((tr = trRe.exec(html))) { + const cells: Cell[] = []; + const tdRe = /]*)>([\s\S]*?)<\/td>/g; + let td: RegExpExecArray | null; + while ((td = tdRe.exec(tr[1]!))) { + const attrs = td[1] ?? ''; + cells.push({ + text: td[2] ?? '', + colspan: Number(/colspan="(\d+)"/.exec(attrs)?.[1] ?? 1), + rowspan: Number(/rowspan="(\d+)"/.exec(attrs)?.[1] ?? 1), + }); + } + rows.push(cells); + } + return rows; +} + +function TableView({ html }: { html: string }) { + const rows = parseTable(html); + if (rows.length === 0) { + return {html}; + } + // Fixed-width cells inside a horizontal scroll — wide tables scroll instead of + // squishing every column into the screen width. + return ( + + + {rows.map((cells, r) => ( + + {cells.map((c, i) => ( + + {c.text} + + ))} + + ))} + + + ); +} + +function DocumentContent() { + const [backendIdx, setBackendIdx] = useState(0); + const [layoutOn, setLayoutOn] = useState(true); + const [supportingOn, setSupportingOn] = useState(true); + const [orientation, setOrientation] = useState(true); + // Off by default: dewarp (UVDoc) corrects photographed, physically-warped pages; + // on a flat screenshot it has nothing to fix and visibly distorts clean text. + const [dewarp, setDewarp] = useState(false); + const [imageUri, setImageUri] = useState(null); + const [isProcessing, setIsProcessing] = useState(false); + const [blocks, setBlocks] = useState[]>([]); + // The frame the result boxes are relative to (orientation/dewarp may move it + // away from the original), so the overlay lines up. + const [processed, setProcessed] = useState(null); + const [wallMs, setWallMs] = useState(null); + const [error, setError] = useState(null); + + const backend = AVAILABLE[backendIdx]!; + + const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err))); + + // Hosted configs — `useOcr` downloads + caches each enabled model. + // orientation/dewarp are NOT baked here: they're passed per-run to + // `runOcr` below, so toggling them takes effect without a reload. + const config = { + ...models.ocr.PADDLE.PPOCRV6_SMALL[backend.key], + ...(layoutOn ? { layout: models.layoutDetection.PP_DOCLAYOUT[backend.key] } : {}), + ...(supportingOn ? { documentModels: models.documentModels.PP_HELPERS[backend.key] } : {}), + }; + + const { isReady, downloadProgress, error: loadError, runOcr } = useOcr(config); + + const handlePick = async (useCamera: boolean) => { + setError(null); + try { + const uri = await getImage(useCamera); + if (uri) { + setImageUri(uri); + setBlocks([]); + setProcessed(null); + setWallMs(null); + } + } catch (e: any) { + setError(e.message || String(e)); + } + }; + + const run = async () => { + if (!skiaImage || !runOcr) return; + setIsProcessing(true); + setError(null); + try { + const pixels = skiaImage.readPixels(); + if (!(pixels instanceof Uint8Array)) throw new Error('Expected Uint8Array from readPixels'); + const start = Date.now(); + const out = await runOcr( + { + data: pixels, + width: skiaImage.width(), + height: skiaImage.height(), + format: 'rgba' as const, + layout: 'hwc' as const, + }, + { orientation, dewarp } + ); + setWallMs(Date.now() - start); + setBlocks(out.blocks); + // Show the frame the boxes are relative to (orientation/dewarp may have + // rotated/warped it), so the overlaid boxes align. + const frame = out.image; + const skData = Skia.Data.fromBytes(frame.data); + const frameImage = Skia.Image.MakeImage( + { + width: frame.width, + height: frame.height, + colorType: ColorType.RGBA_8888, + alphaType: AlphaType.Premul, + }, + skData, + frame.width * 4 + ); + if (!frameImage) throw new Error('Failed to build the processed frame image'); + setProcessed(frameImage); + } catch (e: any) { + setError(e.message || String(e)); + } finally { + setIsProcessing(false); + } + }; + + const activeError = loadError ? String(loadError) : error; + const boxes = useMemo( + () => + blocks.map((b) => [ + { x: b.bbox.xmin, y: b.bbox.ymin }, + { x: b.bbox.xmax, y: b.bbox.ymin }, + { x: b.bbox.xmax, y: b.bbox.ymax }, + { x: b.bbox.xmin, y: b.bbox.ymax }, + ]), + [blocks] + ); + + return ( + + + Full document pipeline: layout → OCR grouped into reading-ordered blocks, with orientation, + table-structure recognition and (optional) dewarp. PaddleOCR is always on; dewarp is off by + default — it only helps photographed, warped pages. Orientation/dewarp are per-run, so + toggling them takes effect on the next run without reloading the models. + + + { + setBackendIdx(v); + setBlocks([]); + setProcessed(null); + setWallMs(null); + }} + /> + + + + + + + + + handlePick(false)} + /> + + +