feat: add native code for object detection

ios/RnExecutorch/ObjectDetection.h

@@ -1,5 +1,5 @@
 #import <RnExecutorchSpec/RnExecutorchSpec.h>
 
-@interface SSDLiteLarge : NSObject <SSDLite>
+@interface ObjectDetection : NSObject <NativeObjectDetectionSpec>
 
 @end

ios/RnExecutorch/ObjectDetection.mm

@@ -1,7 +1,61 @@
-//
-//  SSDLiteLarge.mm
-//  RnExecutorch
-//
-//  Created by Jakub Chmura on 09/12/2024.
-//
+#import "ObjectDetection.h"
+#import <ExecutorchLib/ETModel.h>
+#import <React/RCTBridgeModule.h>
+#import "models/object_detection/SSDLiteLargeModel.hpp"
 
+@implementation ObjectDetection {
+  SSDLiteLargeModel* model;
+}
+
+RCT_EXPORT_MODULE()
+
+- (void)loadModule:(NSString *)modelSource
+           resolve:(RCTPromiseResolveBlock)resolve
+           reject:(RCTPromiseRejectBlock)reject {
+  model = [[SSDLiteLargeModel alloc] init];
+  [model loadModel: [NSURL URLWithString:modelSource] completion:^(BOOL success, NSNumber *errorCode){
+    if(success){
+      resolve(errorCode);
+      return;
+    }
+
+    NSError *error = [NSError
+                      errorWithDomain:@"StyleTransferErrorDomain"
+                      code:[errorCode intValue]
+                      userInfo:@{
+      NSLocalizedDescriptionKey : [NSString
+                                   stringWithFormat:@"%ld", (long)[errorCode longValue]]
+    }];
+
+    reject(@"init_module_error", error.localizedDescription, error);
+    return;
+  }];
+}
+
+
+- (void)forward:(NSString *)input
+        resolve:(RCTPromiseResolveBlock)resolve
+         reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    NSURL *url = [NSURL URLWithString:input];
+    NSData *data = [NSData dataWithContentsOfURL:url];
+    if (!data) {
+      reject(@"img_loading_error", @"Unable to load image data", nil);
+      return;
+    }
+    cv::Mat decodedImage = cv::imdecode(cv::Mat(1, [data length], CV_8UC1, (void*)data.bytes), cv::IMREAD_COLOR);
+    NSArray *result = [model runModel:decodedImage];
+    resolve(result);
+  } @catch (NSException *exception) {
+    NSLog(@"An exception occurred: %@, %@", exception.name, exception.reason);
+    reject(@"result_error", [NSString stringWithFormat:@"%@", exception.reason],
+           nil);
+  }
+}
+
+- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
+(const facebook::react::ObjCTurboModule::InitParams &)params {
+  return std::make_shared<facebook::react::NativeObjectDetectionSpecJSI>(params);
+}
+
+@end

ios/RnExecutorch/models/object_detection/ObjectDetectionUtils.hpp

@@ -0,0 +1,21 @@
+#ifndef ObjectDetectionUtils_hpp
+#define ObjectDetectionUtils_hpp
+
+#include <stdio.h>
+#include <vector>
+#import <Foundation/Foundation.h>
+
+struct Detection{
+  float x1;
+  float y1;
+  float x2;
+  float y2;
+  float label;
+  float score;
+};
+
+NSDictionary* detectionToNSDictionary(const Detection& detection);
+float iou(const Detection& a, const Detection& b);
+std::vector<Detection> nms(std::vector<Detection> detections, float iouThreshold);
+
+#endif /* ObjectDetectionUtils_hpp */

ios/RnExecutorch/models/object_detection/ObjectDetectionUtils.mm

@@ -0,0 +1,69 @@
+#include <vector>
+#include "ObjectDetectionUtils.hpp"
+
+NSDictionary* detectionToNSDictionary(const Detection& detection) {
+    return @{
+        @"x1": @(detection.x1),
+        @"y1": @(detection.y1),
+        @"x2": @(detection.x2),
+        @"y2": @(detection.y2),
+        @"label": @(detection.label),
+        @"score": @(detection.score)
+    };
+}
+
+float iou(const Detection& a, const Detection& b) {
+  float x1 = std::max(a.x1, b.x1);
+  float y1 = std::max(a.y1, b.y1);
+  float x2 = std::min(a.x2, b.x2);
+  float y2 = std::min(a.y2, b.y2);
+
+  float intersectionArea = std::max(0.0f, x2 - x1) * std::max(0.0f, y2 - y1);
+  float areaA = (a.x2 - a.x1) * (a.y2 - a.y1);
+  float areaB = (b.x2 - b.x1) * (b.y2 - b.y1);
+  float unionArea = areaA + areaB - intersectionArea;
+
+  return intersectionArea / unionArea;
+};
+
+std::vector<Detection> nms(std::vector<Detection> detections, float iouThreshold) {
+    if (detections.empty()) {
+        return {};
+    }
+
+    // Sort by label, then by score
+    std::sort(detections.begin(), detections.end(), [](const Detection& a, const Detection& b) {
+        if (a.label == b.label) {
+            return a.score > b.score;
+        }
+        return a.label < b.label;
+    });
+
+
+    std::vector<Detection> result;
+    // Apply NMS for each label
+    for (size_t i = 0; i < detections.size();) {
+        float currentLabel = detections[i].label;
+
+        std::vector<Detection> labelDetections;
+        while (i < detections.size() && detections[i].label == currentLabel) {
+            labelDetections.push_back(detections[i]);
+            ++i;
+        }
+
+        std::vector<Detection> filteredLabelDetections;
+        while (!labelDetections.empty()) {
+            Detection current = labelDetections.front();
+            filteredLabelDetections.push_back(current);
+            labelDetections.erase(
+                std::remove_if(labelDetections.begin(), labelDetections.end(),
+                               [&](const Detection& other) {
+                                   return iou(current, other) > iouThreshold;
+                               }),
+                labelDetections.end());
+        }
+        result.insert(result.end(), filteredLabelDetections.begin(), filteredLabelDetections.end());
+    }
+    return result;
+}
+

ios/RnExecutorch/models/object_detection/SSDLiteLargeModel.hpp

@@ -1,11 +1,11 @@
-#ifndef SSDLITELARGE_HPP
-#define SSDLITELARGE_HPP
-
-#import <UIKit/UIKit.h>
 #import "BaseModel.h"
+#import <UIKit/UIKit.h>
+#include <opencv2/opencv.hpp>
 
 @interface SSDLiteLargeModel : BaseModel
 
-- (void) runModel:(UIImage *)input;
+- (NSArray *)runModel:(cv::Mat)input;
+- (NSArray *)preprocess:(cv::Mat)input;
+- (NSArray *)postprocess:(NSArray *)input;
 
 @end

ios/RnExecutorch/models/object_detection/SSDLiteLargeModel.mm

@@ -1,67 +1,62 @@
-#import "SSDLiteLarge.h"
-#import "ImageProcessor.h"
+#include "SSDLiteLargeModel.hpp"
+#include "ImageProcessor.h"
+#include "ObjectDetectionUtils.hpp"
+#include <vector>
 
-@implementation SSDLiteLarge
+inline float constexpr iouThreshold = 0.55;
+inline float constexpr detectionThreshold = 0.7;
+inline int constexpr inputWidth = 320;
+inline int constexpr inputHeight = 320;
 
-- (float *) NSArrayToFloatArray:(NSArray<NSNumber *> *)array outLength:(size_t )outLength {
-  if (!array || array.count == 0) {
-    NSLog(@"Invalid NSArray input.");
-    outLength = 0;
-    return nullptr;
-  }
-
-  size_t length = array.count;
-  float* floatArray = new float[length];
-
-  for (size_t i = 0; i < length; ++i) {
-    floatArray[i] = [array[i] floatValue];
-  }
-
-  outLength = length;
-  return floatArray;
+@implementation SSDLiteLargeModel
+
+- (NSArray *)preprocess:(cv::Mat)input {
+  cv::resize(input, input, cv::Size(inputWidth, inputHeight));
+  NSArray *modelInput = [ImageProcessor matToNSArray:input];
+  return modelInput;
 }
 
-- (NSArray *) floatArrayToNSArray:(float*) floatArray length:(size_t)length {
-  if (floatArray == nullptr || length == 0) {
-    NSLog(@"Invalid input array or length.");
-    return nil;
+- (NSArray *)postprocess:(NSArray *)input {
+  NSArray *bboxes = [input objectAtIndex:0];
+  NSArray *scores = [input objectAtIndex:1];
+  NSArray *labels = [input objectAtIndex:2];
+
+  std::vector<Detection> detections;
+
+  for (NSUInteger idx = 0; idx < scores.count; idx++) {
+    float score = [scores[idx] floatValue];
+    float label = [labels[idx] floatValue];
+    if (score < detectionThreshold) {
+      continue;
+    }
+    float x1 = [bboxes[idx * 4] floatValue];
+    float y1 = [bboxes[idx * 4 + 1] floatValue];
+    float x2 = [bboxes[idx * 4 + 2] floatValue];
+    float y2 = [bboxes[idx * 4 + 3] floatValue];
+
+    Detection det = {x1, y1, x2, y2, label, score};
+    detections.push_back(det);
   }
+  std::vector<Detection> nms_output = nms(detections, iouThreshold);
 
-  NSMutableArray *array = [NSMutableArray arrayWithCapacity:length];
-
-  for (size_t i = 0; i < length; ++i) {
-    NSNumber *number = [NSNumber numberWithFloat:floatArray[i]];
-    [array addObject:number];
+  NSMutableArray *output = [NSMutableArray array];
+  for (Detection& detection: nms_output) {
+    [output addObject: detectionToNSDictionary(detection)];
   }
-
-  return [array copy];
-}
-
-- (UIImage *)preprocess:(UIImage *)input {
-  CGSize targetSize = CGSizeMake(320, 320);
-  return [ImageProcessor resizeImage:input toSize:targetSize];
-}
 
-- (UIImage *)postprocess:(UIImage *)input {
-  // Assume any necessary format conversions or adjustments
-  return input;
+  return output;
 }
 
-- (void) runModel:(UIImage *)input {
-  CGSize inputSize = CGSize({320, 320});
-  UIImage *preprocessedImage = [self preprocess:input];
-
-  float *preprocessedImageData = [ImageProcessor imageToFloatArray:preprocessedImage size:&inputSize];
-  NSArray *modelInput = [self floatArrayToNSArray:preprocessedImageData length:1228800];
-
-  NSError* forwardError = nil;
-  NSArray *result = [self forward:modelInput shape:@[@1, @3, @320, @320] inputType:@3 error:&forwardError];
-
-  float* outputData = [self NSArrayToFloatArray:result outLength:1228800];
-
-  free(preprocessedImageData);
-  free(outputData);
-//  return [self postprocess:outputImage];
+- (NSArray *)runModel:(cv::Mat)input {
+  NSLog(@"Calling runMoodel");
+  NSArray *modelInput = [self preprocess:input];
+  NSError *forwardError = nil;
+  NSArray *forwardResult = [self forward:modelInput
+                            shape:@[ @1, @3, @320, @320 ]
+                        inputType:@3
+                            error:&forwardError];
+  NSArray *output = [self postprocess:forwardResult];
+  return output;
 }
 
 @end