feat: ocr(ios)

examples/computer-vision/App.tsx

@@ -8,11 +8,13 @@ import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context';
 import { View, StyleSheet } from 'react-native';
 import { ClassificationScreen } from './screens/ClassificationScreen';
 import { ObjectDetectionScreen } from './screens/ObjectDetectionScreen';
+import { OCRScreen } from './screens/OCRScreen';
 
 enum ModelType {
   STYLE_TRANSFER,
   OBJECT_DETECTION,
   CLASSIFICATION,
+  OCR,
 }
 
 export default function App() {
@@ -46,6 +48,8 @@ export default function App() {
         return (
           <ClassificationScreen imageUri={imageUri} setImageUri={setImageUri} />
         );
+      case ModelType.OCR:
+        return <OCRScreen imageUri={imageUri} setImageUri={setImageUri} />;
       default:
         return (
           <StyleTransferScreen imageUri={imageUri} setImageUri={setImageUri} />
@@ -64,6 +68,7 @@ export default function App() {
                 'Style Transfer',
                 'Object Detection',
                 'Classification',
+                'OCR',
               ]}
               onValueChange={(_, selectedIndex) => {
                 handleModeChange(selectedIndex);

examples/computer-vision/components/ImageWithOCRBboxes.tsx

@@ -0,0 +1,103 @@
+// Import necessary components
+import React from 'react';
+import { Image, StyleSheet, View } from 'react-native';
+import Svg, { Polygon } from 'react-native-svg';
+import { OCRDetection } from 'react-native-executorch';
+
+interface Props {
+  imageUri: string;
+  detections: OCRDetection[];
+  imageWidth: number;
+  imageHeight: number;
+}
+
+export default function ImageWithOCRBboxes({
+  imageUri,
+  detections,
+  imageWidth,
+  imageHeight,
+}: Props) {
+  const [layout, setLayout] = React.useState({ width: 0, height: 0 });
+
+  const calculateAdjustedDimensions = () => {
+    const imageRatio = imageWidth / imageHeight;
+    const layoutRatio = layout.width / layout.height;
+    let sx, sy;
+    if (imageRatio > layoutRatio) {
+      sx = layout.width / imageWidth;
+      sy = layout.width / imageRatio / imageHeight;
+    } else {
+      sy = layout.height / imageHeight;
+      sx = (layout.height * imageRatio) / imageWidth;
+    }
+    return {
+      scaleX: sx,
+      scaleY: sy,
+      offsetX: (layout.width - imageWidth * sx) / 2,
+      offsetY: (layout.height - imageHeight * sy) / 2,
+    };
+  };
+
+  return (
+    <View
+      style={styles.container}
+      onLayout={(event) => {
+        const { width, height } = event.nativeEvent.layout;
+        setLayout({ width, height });
+      }}
+    >
+      <Image
+        style={styles.image}
+        resizeMode="contain"
+        source={
+          imageUri
+            ? { uri: imageUri }
+            : require('../assets/icons/executorch_logo.png')
+        }
+      />
+      <Svg style={styles.svgContainer}>
+        {detections.map((detection, index) => {
+          const { scaleX, scaleY, offsetX, offsetY } =
+            calculateAdjustedDimensions();
+          const points = detection.bbox.map((point) => ({
+            x: point.x * scaleX + offsetX,
+            y: point.y * scaleY + offsetY,
+          }));
+
+          const pointsString = points
+            .map((point) => `${point.x},${point.y}`)
+            .join(' ');
+
+          return (
+            <Polygon
+              key={index}
+              points={pointsString}
+              fill="none"
+              stroke="red"
+              strokeWidth="2"
+            />
+          );
+        })}
+      </Svg>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    position: 'relative',
+  },
+  image: {
+    flex: 1,
+    width: '100%',
+    height: '100%',
+  },
+  svgContainer: {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    right: 0,
+    bottom: 0,
+  },
+});

examples/computer-vision/screens/OCRScreen.tsx

@@ -0,0 +1,112 @@
+import Spinner from 'react-native-loading-spinner-overlay';
+import { BottomBar } from '../components/BottomBar';
+import { getImage } from '../utils';
+import { useOCR } from 'react-native-executorch';
+import { View, StyleSheet, Image, Text } from 'react-native';
+import { useState } from 'react';
+import ImageWithBboxes2 from '../components/ImageWithOCRBboxes';
+
+export const OCRScreen = ({
+  imageUri,
+  setImageUri,
+}: {
+  imageUri: string;
+  setImageUri: (imageUri: string) => void;
+}) => {
+  const [results, setResults] = useState<any[]>([]);
+  const [imageDimensions, setImageDimensions] = useState<{
+    width: number;
+    height: number;
+  }>();
+  const [detectedText, setDetectedText] = useState<string>('');
+  const model = useOCR({
+    detectorSource:
+      'https://huggingface.co/nklockiewicz/ocr/resolve/main/xnnpack_craft_800.pte',
+    recognizerSources: {
+      recognizerLarge:
+        'https://huggingface.co/nklockiewicz/ocr/resolve/main/xnnpack_crnn_512.pte',
+      recognizerMedium:
+        'https://huggingface.co/nklockiewicz/ocr/resolve/main/xnnpack_crnn_256.pte',
+      recognizerSmall:
+        'https://huggingface.co/nklockiewicz/ocr/resolve/main/xnnpack_crnn_128.pte',
+    },
+    language: 'en',
+  });
+
+  const handleCameraPress = async (isCamera: boolean) => {
+    const image = await getImage(isCamera);
+    const width = image?.width;
+    const height = image?.height;
+    setImageDimensions({ width: width as number, height: height as number });
+    const uri = image?.uri;
+    if (typeof uri === 'string') {
+      setImageUri(uri as string);
+      setResults([]);
+      setDetectedText('');
+    }
+  };
+
+  const runForward = async () => {
+    try {
+      const output = await model.forward(imageUri);
+      setResults(output);
+      console.log(output);
+      let txt = '';
+      output.forEach((detection: any) => {
+        txt += detection.text + ' ';
+      });
+      setDetectedText(txt);
+    } catch (e) {
+      console.error(e);
+    }
+  };
+
+  if (!model.isReady) {
+    return (
+      <Spinner visible={!model.isReady} textContent={`Loading the model...`} />
+    );
+  }
+
+  return (
+    <>
+      <View style={styles.imageContainer}>
+        <View style={styles.image}>
+          {imageUri && imageDimensions?.width && imageDimensions?.height ? (
+            <ImageWithBboxes2
+              detections={results}
+              imageWidth={imageDimensions?.width}
+              imageHeight={imageDimensions?.height}
+              imageUri={
+                imageUri || require('../assets/icons/executorch_logo.png')
+              }
+            />
+          ) : (
+            <Image
+              style={{ width: '100%', height: '100%' }}
+              resizeMode="contain"
+              source={require('../assets/icons/executorch_logo.png')}
+            />
+          )}
+        </View>
+        <Text>{detectedText}</Text>
+      </View>
+      <BottomBar
+        handleCameraPress={handleCameraPress}
+        runForward={runForward}
+      />
+    </>
+  );
+};
+
+const styles = StyleSheet.create({
+  image: {
+    flex: 2,
+    borderRadius: 8,
+    width: '100%',
+  },
+  imageContainer: {
+    flex: 6,
+    width: '100%',
+    padding: 16,
+  },
+});

ios/RnExecutorch.xcodeproj/project.pbxproj

@@ -37,12 +37,20 @@
 				LLM.h,
 			);
 		};
+		552754CC2D394AC9006B38A2 /* Exceptions for "RnExecutorch" folder in "Compile Sources" phase from "RnExecutorch" target */ = {
+			isa = PBXFileSystemSynchronizedGroupBuildPhaseMembershipExceptionSet;
+			buildPhase = 550986852CEF541900FECBB8 /* Sources */;
+			membershipExceptions = (
+				models/ocr/utils/DetectorUtils.h,
+			);
+		};
 /* End PBXFileSystemSynchronizedGroupBuildPhaseMembershipExceptionSet section */
 
 /* Begin PBXFileSystemSynchronizedRootGroup section */
 		5509868B2CEF541900FECBB8 /* RnExecutorch */ = {
 			isa = PBXFileSystemSynchronizedRootGroup;
 			exceptions = (
+				552754CC2D394AC9006B38A2 /* Exceptions for "RnExecutorch" folder in "Compile Sources" phase from "RnExecutorch" target */,
 				550986902CEF541900FECBB8 /* Exceptions for "RnExecutorch" folder in "Copy Files" phase from "RnExecutorch" target */,
 			);
 			path = RnExecutorch;
@@ -123,6 +131,7 @@
 				TargetAttributes = {
 					550986882CEF541900FECBB8 = {
 						CreatedOnToolsVersion = 16.1;
+						LastSwiftMigration = 1610;
 					};
 				};
 			};
@@ -275,6 +284,7 @@
 		550986942CEF541900FECBB8 /* Debug */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
+				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_STYLE = Automatic;
 				OTHER_LDFLAGS = "-ObjC";
 				PRODUCT_NAME = "$(TARGET_NAME)";
@@ -283,13 +293,16 @@
 				SUPPORTS_MACCATALYST = NO;
 				SUPPORTS_MAC_DESIGNED_FOR_IPHONE_IPAD = NO;
 				SUPPORTS_XR_DESIGNED_FOR_IPHONE_IPAD = NO;
+				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
+				SWIFT_VERSION = 6.0;
 				TARGETED_DEVICE_FAMILY = "1,2";
 			};
 			name = Debug;
 		};
 		550986952CEF541900FECBB8 /* Release */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
+				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_STYLE = Automatic;
 				OTHER_LDFLAGS = "-ObjC";
 				PRODUCT_NAME = "$(TARGET_NAME)";
@@ -298,6 +311,7 @@
 				SUPPORTS_MACCATALYST = NO;
 				SUPPORTS_MAC_DESIGNED_FOR_IPHONE_IPAD = NO;
 				SUPPORTS_XR_DESIGNED_FOR_IPHONE_IPAD = NO;
+				SWIFT_VERSION = 6.0;
 				TARGETED_DEVICE_FAMILY = "1,2";
 			};
 			name = Release;

ios/RnExecutorch/OCR.h

@@ -0,0 +1,7 @@
+#import <RnExecutorchSpec/RnExecutorchSpec.h>
+
+constexpr CGFloat recognizerRatio = 1.6;
+
+@interface OCR : NSObject <NativeOCRSpec>
+
+@end