Skip to content

Commit

Permalink
feat: update TS object detection api to match the native code
Browse files Browse the repository at this point in the history
  • Loading branch information
chmjkb committed Dec 11, 2024
1 parent a0b804e commit 7594519
Show file tree
Hide file tree
Showing 2 changed files with 91 additions and 28 deletions.
23 changes: 7 additions & 16 deletions src/models/object_detection/ObjectDetection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,11 @@ import { Image } from 'react-native';
import { ETError, getError } from '../../Error';
import { ObjectDetection } from '../../native/RnExecutorchModules';
import {

Check failure on line 5 in src/models/object_detection/ObjectDetection.ts

View workflow job for this annotation

GitHub Actions / lint

Replace `⏎····ObjectDetectionResult,⏎` with `·ObjectDetectionResult·`
ObjectDetectionModel,
ObjectDetectionOutputType,
ObjectDetectionResult,
} from './types';

interface Props {
model: keyof typeof ObjectDetectionModel;
path: string | number;
modelSource: string | number;
}

interface ObjectDetectionModule {
Expand All @@ -19,29 +16,25 @@ interface ObjectDetectionModule {
isModelGenerating: boolean;
forward: (

Check failure on line 17 in src/models/object_detection/ObjectDetection.ts

View workflow job for this annotation

GitHub Actions / lint

Replace `⏎····input:·string,⏎··` with `input:·string`
input: string,
outputType: keyof typeof ObjectDetectionOutputType,
topk: number // TODO: find an alternative way
) => Promise<ObjectDetectionResult>;
}

export const useObjectDetection = ({
model,
path,
modelSource,
}: Props): ObjectDetectionModule => {
const [error, setError] = useState<null | string>(null);
const [isModelLoading, setIsModelLoading] = useState(true);
const [isModelGenerating, setIsModelGenerating] = useState(false);

useEffect(() => {
// TODO: handle the case where kind is wrong
const loadModel = async () => {
if (typeof path === 'number') {
path = Image.resolveAssetSource(path).uri;
if (typeof modelSource === 'number') {
modelSource = Image.resolveAssetSource(modelSource).uri;

Check failure on line 32 in src/models/object_detection/ObjectDetection.ts

View workflow job for this annotation

GitHub Actions / lint

Assignments to the 'modelSource' variable from inside React Hook useEffect will be lost after each render. To preserve the value over time, store it in a useRef Hook and keep the mutable value in the '.current' property. Otherwise, you can move this variable directly inside useEffect
}

try {
setIsModelLoading(true);
await ObjectDetection.loadModule(path, model);
await ObjectDetection.loadModule(modelSource);
} catch (e) {
setError(getError(e));
} finally {
Expand All @@ -50,20 +43,18 @@ export const useObjectDetection = ({
};

loadModel();
}, [path]);
}, [modelSource]);

const forward = async (

Check failure on line 48 in src/models/object_detection/ObjectDetection.ts

View workflow job for this annotation

GitHub Actions / lint

Replace `⏎····input:·string,⏎··` with `input:·string`
input: string,
outptutType: keyof typeof ObjectDetectionOutputType,
topk: number
) => {
if (isModelLoading) {
throw new Error(getError(ETError.ModuleNotLoaded));
}

try {
setIsModelGenerating(true);
const output = await ObjectDetection.forward(input, outptutType, topk);
const output = await ObjectDetection.forward(input);
setIsModelGenerating(false);
return output;
} catch (e) {
Expand Down
96 changes: 84 additions & 12 deletions src/models/object_detection/types.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,3 @@
export enum ObjectDetectionModel {
SSDLITE_LARGE = 'SSDLITE_LARGE',
}

export enum ObjectDetectionOutputType {
IMAGE = 1,
DETECTIONS = 2,
ALL = 3,
}

export interface Bbox {
x1: number;
y1: number;
Expand All @@ -17,11 +7,93 @@ export interface Bbox {

export interface Detection {
bbox: Bbox;
label: string; // TODO
label: keyof typeof CocoLabel;
score: number;
}

export interface ObjectDetectionResult {
outputImageUri?: String[];
detections: Detection[];
}

enum CocoLabel {
PERSON = 1,
BICYCLE = 2,
CAR = 3,
MOTORCYCLE = 4,
AIRPLANE = 5,
BUS = 6,
TRAIN = 7,
TRUCK = 8,
BOAT = 9,
TRAFFIC_LIGHT = 10,
FIRE_HYDRANT = 11,
STOP_SIGN = 12,
PARKING_METER = 13,
BENCH = 14,
BIRD = 15,
CAT = 16,
DOG = 17,
HORSE = 18,
SHEEP = 19,
COW = 20,
ELEPHANT = 21,
BEAR = 22,
ZEBRA = 23,
GIRAFFE = 24,
BACKPACK = 25,
UMBRELLA = 26,
HANDBAG = 27,
TIE = 28,
SUITCASE = 29,
FRISBEE = 30,
SKIS = 31,
SNOWBOARD = 32,
SPORTS_BALL = 33,
KITE = 34,
BASEBALL_BAT = 35,
BASEBALL_GLOVE = 36,
SKATEBOARD = 37,
SURFBOARD = 38,
TENNIS_RACKET = 39,
BOTTLE = 40,
WINE_GLASS = 41,
CUP = 42,
FORK = 43,
KNIFE = 44,
SPOON = 45,
BOWL = 46,
BANANA = 47,
APPLE = 48,
SANDWICH = 49,
ORANGE = 50,
BROCCOLI = 51,
CARROT = 52,
HOT_DOG = 53,
PIZZA = 54,
DONUT = 55,
CAKE = 56,
CHAIR = 57,
COUCH = 58,
POTTED_PLANT = 59,
BED = 60,
DINING_TABLE = 61,
TOILET = 62,
TV = 63,
LAPTOP = 64,
MOUSE = 65,
REMOTE = 66,
KEYBOARD = 67,
CELL_PHONE = 68,
MICROWAVE = 69,
OVEN = 70,
TOASTER = 71,
SINK = 72,
REFRIGERATOR = 73,
BOOK = 74,
CLOCK = 75,
VASE = 76,
SCISSORS = 77,
TEDDY_BEAR = 78,
HAIR_DRIER = 79,
TOOTHBRUSH = 80,
}

0 comments on commit 7594519

Please sign in to comment.