From 08bc86b1eea0a90b40f16e95ca36e974e20c165d Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 28 Nov 2023 15:54:30 +0800 Subject: [PATCH 01/28] hacks for model small --- flutter/cpp/datasets/imagenet.cc | 7 ++-- .../flutter/generated_plugin_registrant.cc | 6 ++++ .../windows/flutter/generated_plugins.cmake | 2 ++ .../settings/tflite_settings_pixel6.pbtxt | 33 +++++++++---------- .../tflite_settings_android.pbtxt | 4 +-- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/flutter/cpp/datasets/imagenet.cc b/flutter/cpp/datasets/imagenet.cc index 90a729ab0..91f3d5c78 100644 --- a/flutter/cpp/datasets/imagenet.cc +++ b/flutter/cpp/datasets/imagenet.cc @@ -62,10 +62,9 @@ Imagenet::Imagenet(Backend *backend, const std::string &image_dir, // Prepares the preprocessing stage. tflite::evaluation::ImagePreprocessingConfigBuilder builder( "image_preprocessing", DataType2TfType(input_format_.at(0).type)); - builder.AddResizingStep(image_width / kCroppingFraction, - image_height / kCroppingFraction, true); - builder.AddCroppingStep(image_width, image_height, false); - builder.AddDefaultNormalizationStep(); + builder.AddCroppingStep(kCroppingFraction, true); + builder.AddResizingStep(image_width, image_height, false); + builder.AddPerChannelNormalizationStep(123.675, 116.28, 103.53, 1/57.0); preprocessing_stage_.reset( new tflite::evaluation::ImagePreprocessingStage(builder.build())); if (preprocessing_stage_->Init() != kTfLiteOk) { diff --git a/flutter/windows/flutter/generated_plugin_registrant.cc b/flutter/windows/flutter/generated_plugin_registrant.cc index d5013ba11..42fec2ff9 100644 --- a/flutter/windows/flutter/generated_plugin_registrant.cc +++ b/flutter/windows/flutter/generated_plugin_registrant.cc @@ -6,11 +6,17 @@ #include "generated_plugin_registrant.h" +#include +#include #include #include #include void RegisterPlugins(flutter::PluginRegistry* registry) { + DesktopWebviewAuthPluginRegisterWithRegistrar( + registry->GetRegistrarForPlugin("DesktopWebviewAuthPlugin")); + FirebaseCorePluginCApiRegisterWithRegistrar( + registry->GetRegistrarForPlugin("FirebaseCorePluginCApi")); PermissionHandlerWindowsPluginRegisterWithRegistrar( registry->GetRegistrarForPlugin("PermissionHandlerWindowsPlugin")); SharePlusWindowsPluginCApiRegisterWithRegistrar( diff --git a/flutter/windows/flutter/generated_plugins.cmake b/flutter/windows/flutter/generated_plugins.cmake index a0d138869..f24e192f4 100644 --- a/flutter/windows/flutter/generated_plugins.cmake +++ b/flutter/windows/flutter/generated_plugins.cmake @@ -3,6 +3,8 @@ # list(APPEND FLUTTER_PLUGIN_LIST + desktop_webview_auth + firebase_core permission_handler_windows share_plus url_launcher_windows diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 1022ac06f..449fdea21 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -30,6 +30,15 @@ benchmark_setting { delegate_selected: "NNAPI" } +benchmark_setting { + benchmark_id: "image_classification_v2" + accelerator: "tpu" + accelerator_desc: "Google Edge TPU" + framework: "TFLite NNAPI" + model_path: "local:///small.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" +} + benchmark_setting { benchmark_id: "image_classification_offline" framework: "TFLite" @@ -74,24 +83,12 @@ benchmark_setting { benchmark_setting { benchmark_id: "image_classification_offline_v2" - framework: "TFLite" - delegate_choice: { - delegate_name: "NNAPI" - accelerator_name: "tpu" - accelerator_desc: "Google EdgeTPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" - batch_size: 16 - } - delegate_choice: { - delegate_name: "GPU" - accelerator_name: "gpu" - accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" - batch_size: 2 - } - delegate_selected: "NNAPI" + accelerator: "tpu" + accelerator_desc: "Google Edge TPU" + framework: "TFLite NNAPI" + batch_size: 16 + model_path: "local:///small.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" } benchmark_setting { diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index 3bbe1ae17..fa036f0f9 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -38,7 +38,7 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_path: "local:///small.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" } delegate_choice: { @@ -80,7 +80,7 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_path: "local:///small.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" batch_size: 2 } From 8a9e510376783e8a789f5630f1856259aafeecf3 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 28 Nov 2023 16:19:22 +0800 Subject: [PATCH 02/28] small -> medium --- flutter/cpp/flutter/dart_run_benchmark.cc | 2 +- .../cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt | 4 ++-- .../backend_settings/tflite_settings_android.pbtxt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flutter/cpp/flutter/dart_run_benchmark.cc b/flutter/cpp/flutter/dart_run_benchmark.cc index 5be456540..27838005e 100644 --- a/flutter/cpp/flutter/dart_run_benchmark.cc +++ b/flutter/cpp/flutter/dart_run_benchmark.cc @@ -73,7 +73,7 @@ struct dart_ffi_run_benchmark_out* dart_ffi_run_benchmark( case ::mlperf::mobile::DatasetConfig::IMAGENET: dataset = std::make_unique<::mlperf::mobile::Imagenet>( backend.get(), in->dataset_data_path, in->dataset_groundtruth_path, - in->dataset_offset, 224, 224 /* width, height */); + in->dataset_offset, 256, 256 /* width, height */); break; case ::mlperf::mobile::DatasetConfig::COCO: dataset = std::make_unique<::mlperf::mobile::Coco>( diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 449fdea21..b98e339a4 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -35,7 +35,7 @@ benchmark_setting { accelerator: "tpu" accelerator_desc: "Google Edge TPU" framework: "TFLite NNAPI" - model_path: "local:///small.tflite" + model_path: "local:///medium.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" } @@ -87,7 +87,7 @@ benchmark_setting { accelerator_desc: "Google Edge TPU" framework: "TFLite NNAPI" batch_size: 16 - model_path: "local:///small.tflite" + model_path: "local:///medium.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" } diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index fa036f0f9..edff25915 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -38,7 +38,7 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "local:///small.tflite" + model_path: "local:///medium.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" } delegate_choice: { @@ -80,7 +80,7 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "local:///small.tflite" + model_path: "local:///medium.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" batch_size: 2 } From cf3fa946f16a6be3c7fdcbdf8becc20e3c84ac1f Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 28 Nov 2023 16:55:40 +0800 Subject: [PATCH 03/28] medium -> large --- flutter/cpp/flutter/dart_run_benchmark.cc | 2 +- .../cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt | 4 ++-- .../backend_settings/tflite_settings_android.pbtxt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flutter/cpp/flutter/dart_run_benchmark.cc b/flutter/cpp/flutter/dart_run_benchmark.cc index 27838005e..763e6ea24 100644 --- a/flutter/cpp/flutter/dart_run_benchmark.cc +++ b/flutter/cpp/flutter/dart_run_benchmark.cc @@ -73,7 +73,7 @@ struct dart_ffi_run_benchmark_out* dart_ffi_run_benchmark( case ::mlperf::mobile::DatasetConfig::IMAGENET: dataset = std::make_unique<::mlperf::mobile::Imagenet>( backend.get(), in->dataset_data_path, in->dataset_groundtruth_path, - in->dataset_offset, 256, 256 /* width, height */); + in->dataset_offset, 384, 384 /* width, height */); break; case ::mlperf::mobile::DatasetConfig::COCO: dataset = std::make_unique<::mlperf::mobile::Coco>( diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index b98e339a4..d01021886 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -35,7 +35,7 @@ benchmark_setting { accelerator: "tpu" accelerator_desc: "Google Edge TPU" framework: "TFLite NNAPI" - model_path: "local:///medium.tflite" + model_path: "local:///large.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" } @@ -87,7 +87,7 @@ benchmark_setting { accelerator_desc: "Google Edge TPU" framework: "TFLite NNAPI" batch_size: 16 - model_path: "local:///medium.tflite" + model_path: "local:///large.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" } diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index edff25915..ce5e463d0 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -38,7 +38,7 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "local:///medium.tflite" + model_path: "local:///large.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" } delegate_choice: { @@ -80,7 +80,7 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "local:///medium.tflite" + model_path: "local:///large.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" batch_size: 2 } From 4bea66518d4a32e5a1e8a65e3bc5892581199acb Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 28 Nov 2023 18:09:50 +0800 Subject: [PATCH 04/28] for quantized models --- flutter/cpp/datasets/imagenet.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flutter/cpp/datasets/imagenet.cc b/flutter/cpp/datasets/imagenet.cc index 91f3d5c78..3fd6bbf7c 100644 --- a/flutter/cpp/datasets/imagenet.cc +++ b/flutter/cpp/datasets/imagenet.cc @@ -64,7 +64,12 @@ Imagenet::Imagenet(Backend *backend, const std::string &image_dir, "image_preprocessing", DataType2TfType(input_format_.at(0).type)); builder.AddCroppingStep(kCroppingFraction, true); builder.AddResizingStep(image_width, image_height, false); - builder.AddPerChannelNormalizationStep(123.675, 116.28, 103.53, 1/57.0); + switch (input_format_.at(0).type) { + case DataType::Float32: + builder.AddPerChannelNormalizationStep(123.675, 116.28, 103.53, 1 / 57.0); + default: + builder.AddDefaultNormalizationStep(); + } preprocessing_stage_.reset( new tflite::evaluation::ImagePreprocessingStage(builder.build())); if (preprocessing_stage_->Init() != kTfLiteOk) { From 22834227573c0b4df6e449a155d4d163cf0692d9 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 2 Jan 2024 21:29:30 +0800 Subject: [PATCH 05/28] make float model work again fixed a stupid bug --- flutter/cpp/datasets/imagenet.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/flutter/cpp/datasets/imagenet.cc b/flutter/cpp/datasets/imagenet.cc index 3fd6bbf7c..b8ce92611 100644 --- a/flutter/cpp/datasets/imagenet.cc +++ b/flutter/cpp/datasets/imagenet.cc @@ -67,6 +67,7 @@ Imagenet::Imagenet(Backend *backend, const std::string &image_dir, switch (input_format_.at(0).type) { case DataType::Float32: builder.AddPerChannelNormalizationStep(123.675, 116.28, 103.53, 1 / 57.0); + break; default: builder.AddDefaultNormalizationStep(); } From 7317b9b76a7ee1c9e896abd01eabae96c4d01c4c Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Wed, 3 Jan 2024 09:56:47 +0800 Subject: [PATCH 06/28] fix pixel and neuron settings --- .../settings/tflite_settings_pixel6.pbtxt | 48 ++++++++++++++----- .../tflite_settings_mtk.pbtxt | 48 +++++++++++++++++++ 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index d01021886..e0a1ff2d4 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -32,11 +32,23 @@ benchmark_setting { benchmark_setting { benchmark_id: "image_classification_v2" - accelerator: "tpu" - accelerator_desc: "Google Edge TPU" - framework: "TFLite NNAPI" - model_path: "local:///large.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + framework: "TFLite" + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "tpu" + accelerator_desc: "Google EdgeTPU" + model_path: "local:///large.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_float.tflite" + model_checksum: "0da3fa6adc8bbaaeb0b5647e2d46c8a4" + } + delegate_selected: "NNAPI" + } benchmark_setting { @@ -44,7 +56,7 @@ benchmark_setting { framework: "TFLite" delegate_choice: { delegate_name: "NNAPI" - accelerator_name: "tpu" + accelerator: "tpu" accelerator_desc: "Google EdgeTPU" model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_uint8.tflite" model_checksum: "008dfcb1c1962fedbeef1b998d4c84f2" @@ -83,12 +95,24 @@ benchmark_setting { benchmark_setting { benchmark_id: "image_classification_offline_v2" - accelerator: "tpu" - accelerator_desc: "Google Edge TPU" - framework: "TFLite NNAPI" - batch_size: 16 - model_path: "local:///large.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + framework: "TFLite" + delegate_choice: { + delegate_name: "NNAPI" + accelerator: "tpu" + accelerator_desc: "Google EdgeTPU" + model_path: "local:///large.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + batch_size: 64 + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/main/v1_1/tflite/mobilenet_edgetpu_224_1.0_float.tflite" + model_checksum: "66bb4eba50987221608f8487ed405794" + batch_size: 2 + } + delegate_selected: "NNAPI" } benchmark_setting { diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt index 118e8609a..2bcab61e5 100644 --- a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt @@ -35,6 +35,26 @@ benchmark_setting { delegate_selected: "Neuron" } +benchmark_setting { + benchmark_id: "image_classification_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Neuron" + accelerator_name: "neuron-mdla" + accelerator_desc: "MediaTek NN accelerator via the Neuron Delegate" + model_path: "local:///large.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + } + delegate_selected: "Neuron" +} + benchmark_setting { benchmark_id: "image_classification_offline" framework: "TFLite" @@ -63,6 +83,34 @@ benchmark_setting { delegate_selected: "Neuron" } +benchmark_setting { + benchmark_id: "image_classification_offline" + framework: "TFLite" + delegate_choice: { + delegate_name: "Neuron" + accelerator_name: "neuron-mdla" + accelerator_desc: "MediaTek NN accelerator via the Neuron Delegate" + model_path: "local:///large.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + batch_size: 256 + } + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "npu" + accelerator_desc: "NPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_uint8.tflite" + model_checksum: "008dfcb1c1962fedbeef1b998d4c84f2" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_float.tflite" + model_checksum: "0da3fa6adc8bbaaeb0b5647e2d46c8a4" + } + delegate_selected: "Neuron" +} + benchmark_setting { benchmark_id: "object_detection" framework: "TFLite" From bb7dda66a58505af1eba9955500e9bf8c766bc20 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Thu, 11 Jan 2024 09:54:27 +0800 Subject: [PATCH 07/28] change preprocessing and label offset --- flutter/assets/tasks.pbtxt | 2 +- flutter/cpp/datasets/imagenet.cc | 14 +++++--------- .../backend_settings/tflite_settings_mtk.pbtxt | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index 01029bfd1..88a6f7522 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -246,7 +246,7 @@ task { model { id: "MobileNetEdgeTPUv2-L" name: "MobileNetEdgeTPUv2-L" - offset: 1 + offset: 0 image_width: 224 image_height: 224 num_classes: 1000 diff --git a/flutter/cpp/datasets/imagenet.cc b/flutter/cpp/datasets/imagenet.cc index b8ce92611..7d35b309a 100644 --- a/flutter/cpp/datasets/imagenet.cc +++ b/flutter/cpp/datasets/imagenet.cc @@ -62,15 +62,11 @@ Imagenet::Imagenet(Backend *backend, const std::string &image_dir, // Prepares the preprocessing stage. tflite::evaluation::ImagePreprocessingConfigBuilder builder( "image_preprocessing", DataType2TfType(input_format_.at(0).type)); - builder.AddCroppingStep(kCroppingFraction, true); - builder.AddResizingStep(image_width, image_height, false); - switch (input_format_.at(0).type) { - case DataType::Float32: - builder.AddPerChannelNormalizationStep(123.675, 116.28, 103.53, 1 / 57.0); - break; - default: - builder.AddDefaultNormalizationStep(); - } + builder.AddResizingStep(image_width / kCroppingFraction, + image_height / kCroppingFraction, true); + builder.AddCroppingStep(image_width, image_height, false); + builder.AddDefaultNormalizationStep(); + preprocessing_stage_.reset( new tflite::evaluation::ImagePreprocessingStage(builder.build())); if (preprocessing_stage_->Init() != kTfLiteOk) { diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt index 2bcab61e5..2a83edef2 100644 --- a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt @@ -84,7 +84,7 @@ benchmark_setting { } benchmark_setting { - benchmark_id: "image_classification_offline" + benchmark_id: "image_classification_offline_v2" framework: "TFLite" delegate_choice: { delegate_name: "Neuron" From c2593337b3cca1bedd1c2533890b5adba21314a5 Mon Sep 17 00:00:00 2001 From: Mohit Mundhra Date: Mon, 8 Jan 2024 16:28:14 +0530 Subject: [PATCH 08/28] Adding changes for calibration dataset preprocessing --- datasets/util/common/jpg_to_raw.py | 13 ++++++++++--- datasets/util/imagenet/Resize224.py | 27 +++++++++++++++------------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/datasets/util/common/jpg_to_raw.py b/datasets/util/common/jpg_to_raw.py index 6e6c40350..b55085f44 100644 --- a/datasets/util/common/jpg_to_raw.py +++ b/datasets/util/common/jpg_to_raw.py @@ -35,9 +35,16 @@ original_im = Image.open(BytesIO(jpeg_str)) converted_image = original_im.convert('RGB') npimage = np.asarray(converted_image).astype(np.float32) - npimage = npimage * 0.00784313771874 - npimage = npimage - 1.0 - img_ndarray = np.array(npimage) + + red = npimage[:,:,0] + blue = npimage[:,:,1] + green = npimage[:,:,2] + + red=(red-123.675)*0.0175438596 + blue=(blue-116.28)*0.0175438596 + green=(green-103.53)*0.0175438596 + + npimage = np.dstack([red,blue,green]) tmp = dst.split(".") tmp[-1] = "raw" f_name = ".".join(tmp) diff --git a/datasets/util/imagenet/Resize224.py b/datasets/util/imagenet/Resize224.py index 05dd3ef81..3f009389b 100644 --- a/datasets/util/imagenet/Resize224.py +++ b/datasets/util/imagenet/Resize224.py @@ -16,17 +16,20 @@ from PIL import Image import os -for f in os.listdir(os.sys.argv[1]): - img = Image.open(os.sys.argv[1]+"/"+f) - img256 = img.resize((256, 256), Image.ANTIALIAS) - # Seems like the required algo is to resize to 256x256 - # Then center crop a 224x244 image - left = (256 - 224)/2 - top = (256 - 224)/2 - right = (256 + 224)/2 - bottom = (256 + 224)/2 +def center_crop(img): + frac=0.875 + left = img.size[0]*((1-frac)/2) + upper = img.size[1]*((1-frac)/2) + right = img.size[0]-((1-frac)/2)*img.size[0] + bottom = img.size[1]-((1-frac)/2)*img.size[1] + img = img.crop((left, upper, right, bottom)) + return img - # Crop 224x224 image from the center - img224 = img256.crop((left, top, right, bottom)) - img224.save(os.sys.argv[2]+"/"+f) +for f in os.listdir(os.sys.argv[1]): + print(os.sys.argv[1]+"/"+f) + img = Image.open(os.sys.argv[1]+"/"+f) + # center crop + img = center_crop(img) + img = img.resize((384, 384), resample= Image.Resampling.BILINEAR) + img.save(os.sys.argv[2]+"/"+f) \ No newline at end of file From c15f4b025495487b1f693850595efc39c45a5c9a Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Thu, 11 Jan 2024 14:44:06 +0800 Subject: [PATCH 09/28] hacks --- flutter/cpp/datasets/imagenet.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flutter/cpp/datasets/imagenet.cc b/flutter/cpp/datasets/imagenet.cc index 7d35b309a..06303141c 100644 --- a/flutter/cpp/datasets/imagenet.cc +++ b/flutter/cpp/datasets/imagenet.cc @@ -62,9 +62,16 @@ Imagenet::Imagenet(Backend *backend, const std::string &image_dir, // Prepares the preprocessing stage. tflite::evaluation::ImagePreprocessingConfigBuilder builder( "image_preprocessing", DataType2TfType(input_format_.at(0).type)); +#if 0 builder.AddResizingStep(image_width / kCroppingFraction, image_height / kCroppingFraction, true); builder.AddCroppingStep(image_width, image_height, false); +#endif +#if 1 + builder.AddCroppingStep(kCroppingFraction, true); + builder.AddResizingStep(image_width, image_height, false); +#endif + builder.AddDefaultNormalizationStep(); preprocessing_stage_.reset( From b13ab6205c337586967462b6f8dfb7d6062ef971 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Thu, 1 Feb 2024 13:50:42 +0800 Subject: [PATCH 10/28] clean up --- flutter/assets/tasks.pbtxt | 10 +++++----- flutter/cpp/datasets/imagenet.cc | 8 -------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index 88a6f7522..7795e39fd 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -247,8 +247,8 @@ task { id: "MobileNetEdgeTPUv2-L" name: "MobileNetEdgeTPUv2-L" offset: 0 - image_width: 224 - image_height: 224 + image_width: 384 + image_height: 384 num_classes: 1000 } } @@ -283,9 +283,9 @@ task { model { id: "MobileNetEdgeTPUv2-L" name: "MobileNetEdgeTPUv2-L" - offset: 1 - image_width: 224 - image_height: 224 + offset: 0 + image_width: 384 + image_height: 384 num_classes: 1000 } } diff --git a/flutter/cpp/datasets/imagenet.cc b/flutter/cpp/datasets/imagenet.cc index 06303141c..90a729ab0 100644 --- a/flutter/cpp/datasets/imagenet.cc +++ b/flutter/cpp/datasets/imagenet.cc @@ -62,18 +62,10 @@ Imagenet::Imagenet(Backend *backend, const std::string &image_dir, // Prepares the preprocessing stage. tflite::evaluation::ImagePreprocessingConfigBuilder builder( "image_preprocessing", DataType2TfType(input_format_.at(0).type)); -#if 0 builder.AddResizingStep(image_width / kCroppingFraction, image_height / kCroppingFraction, true); builder.AddCroppingStep(image_width, image_height, false); -#endif -#if 1 - builder.AddCroppingStep(kCroppingFraction, true); - builder.AddResizingStep(image_width, image_height, false); -#endif - builder.AddDefaultNormalizationStep(); - preprocessing_stage_.reset( new tflite::evaluation::ImagePreprocessingStage(builder.build())); if (preprocessing_stage_->Init() != kTfLiteOk) { From 5d46760432ffa6a16e23b865906203c59c9d0013 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Fri, 2 Feb 2024 17:09:30 +0800 Subject: [PATCH 11/28] fix pixel backend settings --- .../backend_tflite/settings/tflite_settings_pixel6.pbtxt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index e0a1ff2d4..ff2734425 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -56,7 +56,7 @@ benchmark_setting { framework: "TFLite" delegate_choice: { delegate_name: "NNAPI" - accelerator: "tpu" + accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_uint8.tflite" model_checksum: "008dfcb1c1962fedbeef1b998d4c84f2" @@ -98,7 +98,7 @@ benchmark_setting { framework: "TFLite" delegate_choice: { delegate_name: "NNAPI" - accelerator: "tpu" + accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" model_path: "local:///large.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" @@ -106,7 +106,7 @@ benchmark_setting { } delegate_choice: { delegate_name: "GPU" - accelerator_name: "gpu" + accelerator_name: "tpu" accelerator_desc: "GPU" model_path: "https://github.com/mlcommons/mobile_models/raw/main/v1_1/tflite/mobilenet_edgetpu_224_1.0_float.tflite" model_checksum: "66bb4eba50987221608f8487ed405794" From ccc59bf9a1183065226f88025bc2465f922026e8 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Fri, 2 Feb 2024 17:24:00 +0800 Subject: [PATCH 12/28] fix pixel settings & icons for classificaton v2 --- flutter/lib/ui/icons.dart | 5 ++++ .../settings/tflite_settings_pixel6.pbtxt | 23 +------------------ 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/flutter/lib/ui/icons.dart b/flutter/lib/ui/icons.dart index edc1dcc80..b3a1b4145 100644 --- a/flutter/lib/ui/icons.dart +++ b/flutter/lib/ui/icons.dart @@ -64,6 +64,8 @@ class BenchmarkIcons { BenchmarkId.naturalLanguageProcessing: AppIcons.languageProcessing, BenchmarkId.superResolution: AppIcons.superResolution, BenchmarkId.imageClassificationOffline: AppIcons.imageClassificationOffline, + BenchmarkId.imageClassificationV2: AppIcons.imageClassification, + BenchmarkId.imageClassificationOfflineV2: AppIcons.imageClassificationOffline, }; static final lightSet = { @@ -74,6 +76,9 @@ class BenchmarkIcons { BenchmarkId.superResolution: AppIcons.superResolutionWhite, BenchmarkId.imageClassificationOffline: AppIcons.imageClassificationOfflineWhite, + BenchmarkId.imageClassificationV2: AppIcons.imageClassificationWhite, + BenchmarkId.imageClassificationOfflineV2: + AppIcons.imageClassificationOfflineWhite, }; static Widget getDarkIcon(String benchmarkId) => diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index ff2734425..796543da4 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -73,26 +73,6 @@ benchmark_setting { delegate_selected: "NNAPI" } -benchmark_setting { - benchmark_id: "image_classification_v2" - framework: "TFLite" - delegate_choice: { - delegate_name: "NNAPI" - accelerator_name: "tpu" - accelerator_desc: "Google EdgeTPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" - } - delegate_choice: { - delegate_name: "GPU" - accelerator_name: "gpu" - accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" - } - delegate_selected: "NNAPI" -} - benchmark_setting { benchmark_id: "image_classification_offline_v2" framework: "TFLite" @@ -108,7 +88,7 @@ benchmark_setting { delegate_name: "GPU" accelerator_name: "tpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/main/v1_1/tflite/mobilenet_edgetpu_224_1.0_float.tflite" + model_path: "local:///large.tflite" model_checksum: "66bb4eba50987221608f8487ed405794" batch_size: 2 } @@ -133,7 +113,6 @@ benchmark_setting { model_checksum: "566ceb72a4c7c8926fe4ac8eededb5bf" } delegate_selected: "NNAPI" - } benchmark_setting { From cffb43df523edb336a7c37c2726517fcceed2566 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Fri, 2 Feb 2024 17:36:23 +0800 Subject: [PATCH 13/28] fix lint issue --- flutter/lib/ui/icons.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flutter/lib/ui/icons.dart b/flutter/lib/ui/icons.dart index b3a1b4145..9c3a662cd 100644 --- a/flutter/lib/ui/icons.dart +++ b/flutter/lib/ui/icons.dart @@ -65,7 +65,8 @@ class BenchmarkIcons { BenchmarkId.superResolution: AppIcons.superResolution, BenchmarkId.imageClassificationOffline: AppIcons.imageClassificationOffline, BenchmarkId.imageClassificationV2: AppIcons.imageClassification, - BenchmarkId.imageClassificationOfflineV2: AppIcons.imageClassificationOffline, + BenchmarkId.imageClassificationOfflineV2: + AppIcons.imageClassificationOffline, }; static final lightSet = { From 55c5cd0f8dbec658111275db406739dd349f8173 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 6 Feb 2024 11:02:40 +0800 Subject: [PATCH 14/28] Revert "Adding changes for calibration dataset preprocessing" This reverts commit c2593337b3cca1bedd1c2533890b5adba21314a5. New model doesn't need these --- datasets/util/common/jpg_to_raw.py | 13 +++---------- datasets/util/imagenet/Resize224.py | 27 ++++++++++++--------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/datasets/util/common/jpg_to_raw.py b/datasets/util/common/jpg_to_raw.py index b55085f44..6e6c40350 100644 --- a/datasets/util/common/jpg_to_raw.py +++ b/datasets/util/common/jpg_to_raw.py @@ -35,16 +35,9 @@ original_im = Image.open(BytesIO(jpeg_str)) converted_image = original_im.convert('RGB') npimage = np.asarray(converted_image).astype(np.float32) - - red = npimage[:,:,0] - blue = npimage[:,:,1] - green = npimage[:,:,2] - - red=(red-123.675)*0.0175438596 - blue=(blue-116.28)*0.0175438596 - green=(green-103.53)*0.0175438596 - - npimage = np.dstack([red,blue,green]) + npimage = npimage * 0.00784313771874 + npimage = npimage - 1.0 + img_ndarray = np.array(npimage) tmp = dst.split(".") tmp[-1] = "raw" f_name = ".".join(tmp) diff --git a/datasets/util/imagenet/Resize224.py b/datasets/util/imagenet/Resize224.py index 3f009389b..05dd3ef81 100644 --- a/datasets/util/imagenet/Resize224.py +++ b/datasets/util/imagenet/Resize224.py @@ -16,20 +16,17 @@ from PIL import Image import os - -def center_crop(img): - frac=0.875 - left = img.size[0]*((1-frac)/2) - upper = img.size[1]*((1-frac)/2) - right = img.size[0]-((1-frac)/2)*img.size[0] - bottom = img.size[1]-((1-frac)/2)*img.size[1] - img = img.crop((left, upper, right, bottom)) - return img - for f in os.listdir(os.sys.argv[1]): - print(os.sys.argv[1]+"/"+f) img = Image.open(os.sys.argv[1]+"/"+f) - # center crop - img = center_crop(img) - img = img.resize((384, 384), resample= Image.Resampling.BILINEAR) - img.save(os.sys.argv[2]+"/"+f) \ No newline at end of file + img256 = img.resize((256, 256), Image.ANTIALIAS) + + # Seems like the required algo is to resize to 256x256 + # Then center crop a 224x244 image + left = (256 - 224)/2 + top = (256 - 224)/2 + right = (256 + 224)/2 + bottom = (256 + 224)/2 + + # Crop 224x224 image from the center + img224 = img256.crop((left, top, right, bottom)) + img224.save(os.sys.argv[2]+"/"+f) From 74b4922cae17f96da4a9feadd5b819d8a8e88060 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 6 Feb 2024 11:03:51 +0800 Subject: [PATCH 15/28] revert imagenet width and height we'll use #853 to handle different width and height --- flutter/cpp/flutter/dart_run_benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter/cpp/flutter/dart_run_benchmark.cc b/flutter/cpp/flutter/dart_run_benchmark.cc index 763e6ea24..5be456540 100644 --- a/flutter/cpp/flutter/dart_run_benchmark.cc +++ b/flutter/cpp/flutter/dart_run_benchmark.cc @@ -73,7 +73,7 @@ struct dart_ffi_run_benchmark_out* dart_ffi_run_benchmark( case ::mlperf::mobile::DatasetConfig::IMAGENET: dataset = std::make_unique<::mlperf::mobile::Imagenet>( backend.get(), in->dataset_data_path, in->dataset_groundtruth_path, - in->dataset_offset, 384, 384 /* width, height */); + in->dataset_offset, 224, 224 /* width, height */); break; case ::mlperf::mobile::DatasetConfig::COCO: dataset = std::make_unique<::mlperf::mobile::Coco>( From 0f3d5b95305d5a5db637dc2966db5a706bac9f5e Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 6 Feb 2024 13:25:23 +0800 Subject: [PATCH 16/28] fix offline for Pixel backend --- .../cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 796543da4..787c25d6d 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -82,7 +82,7 @@ benchmark_setting { accelerator_desc: "Google EdgeTPU" model_path: "local:///large.tflite" model_checksum: "60f651765fa0d52b468f2e1c15c47f91" - batch_size: 64 + batch_size: 32 } delegate_choice: { delegate_name: "GPU" From 8d2a0e0482ea6dcbf957d0fcb2bedcde1bdaeb7f Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 6 Feb 2024 14:03:04 +0700 Subject: [PATCH 17/28] Undo update for Windows build --- flutter/windows/flutter/generated_plugin_registrant.cc | 6 ------ flutter/windows/flutter/generated_plugins.cmake | 2 -- 2 files changed, 8 deletions(-) diff --git a/flutter/windows/flutter/generated_plugin_registrant.cc b/flutter/windows/flutter/generated_plugin_registrant.cc index 42fec2ff9..d5013ba11 100644 --- a/flutter/windows/flutter/generated_plugin_registrant.cc +++ b/flutter/windows/flutter/generated_plugin_registrant.cc @@ -6,17 +6,11 @@ #include "generated_plugin_registrant.h" -#include -#include #include #include #include void RegisterPlugins(flutter::PluginRegistry* registry) { - DesktopWebviewAuthPluginRegisterWithRegistrar( - registry->GetRegistrarForPlugin("DesktopWebviewAuthPlugin")); - FirebaseCorePluginCApiRegisterWithRegistrar( - registry->GetRegistrarForPlugin("FirebaseCorePluginCApi")); PermissionHandlerWindowsPluginRegisterWithRegistrar( registry->GetRegistrarForPlugin("PermissionHandlerWindowsPlugin")); SharePlusWindowsPluginCApiRegisterWithRegistrar( diff --git a/flutter/windows/flutter/generated_plugins.cmake b/flutter/windows/flutter/generated_plugins.cmake index f24e192f4..a0d138869 100644 --- a/flutter/windows/flutter/generated_plugins.cmake +++ b/flutter/windows/flutter/generated_plugins.cmake @@ -3,8 +3,6 @@ # list(APPEND FLUTTER_PLUGIN_LIST - desktop_webview_auth - firebase_core permission_handler_windows share_plus url_launcher_windows From 92d092ef47ec72834d008bb2b9dc933452e36d77 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Thu, 8 Feb 2024 11:47:46 +0800 Subject: [PATCH 18/28] update MobilenetV4 settings for TFLite, Neuron, and Pixel --- .../settings/tflite_settings_pixel6.pbtxt | 19 +++++++------ .../tflite_settings_android.pbtxt | 16 +++++------ .../tflite_settings_mtk.pbtxt | 27 ++++++++++++------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 787c25d6d..3db4ab8ea 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -37,18 +37,17 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - model_path: "local:///large.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_float.tflite" - model_checksum: "0da3fa6adc8bbaaeb0b5647e2d46c8a4" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "NNAPI" - } benchmark_setting { @@ -80,16 +79,16 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - model_path: "local:///large.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" batch_size: 32 + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" - accelerator_name: "tpu" + accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "local:///large.tflite" - model_checksum: "66bb4eba50987221608f8487ed405794" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 2 } delegate_selected: "NNAPI" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index ce5e463d0..7f0b3b8a6 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -38,15 +38,15 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "local:///large.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "NNAPI" } @@ -80,16 +80,16 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "local:///large.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" batch_size: 2 } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 2 } delegate_selected: "NNAPI" diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt index 2a83edef2..eddd47301 100644 --- a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt @@ -42,15 +42,22 @@ benchmark_setting { delegate_name: "Neuron" accelerator_name: "neuron-mdla" accelerator_desc: "MediaTek NN accelerator via the Neuron Delegate" - model_path: "local:///large.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + } + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "npu" + accelerator_desc: "NPU" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "Neuron" } @@ -90,23 +97,23 @@ benchmark_setting { delegate_name: "Neuron" accelerator_name: "neuron-mdla" accelerator_desc: "MediaTek NN accelerator via the Neuron Delegate" - model_path: "local:///large.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" batch_size: 256 } delegate_choice: { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_uint8.tflite" - model_checksum: "008dfcb1c1962fedbeef1b998d4c84f2" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_float.tflite" - model_checksum: "0da3fa6adc8bbaaeb0b5647e2d46c8a4" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "Neuron" } From d9e044ab815f4140b64b297d19ebebb77cb561fd Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Thu, 8 Feb 2024 11:54:08 +0800 Subject: [PATCH 19/28] update windows setting for Mobilenet V4 --- .../backend_settings/tflite_settings_windows.pbtxt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt index 2eae34450..71b8821a9 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt @@ -45,8 +45,11 @@ benchmark_setting { delegate_name: "CPU" accelerator_name: "cpu" accelerator_desc: "CPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + } + delegate_selected: "NNAPI" + } delegate_selected: "CPU" } @@ -59,8 +62,8 @@ benchmark_setting { accelerator_name: "cpu" accelerator_desc: "CPU" batch_size: 2 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_selected: "CPU" } From fe4d2d14f314c7f9c814f969085d833ffaac03e3 Mon Sep 17 00:00:00 2001 From: Anh Date: Thu, 8 Feb 2024 16:10:21 +0700 Subject: [PATCH 20/28] Use MobileNetV4 model in Apple devices --- .../cpp/backend_coreml/coreml_settings.pbtxt | 24 ++++++++-------- mobile_back_apple/dev-utils/Makefile | 24 ++++++++++------ mobile_back_apple/models/icv2.py | 28 +++++++++---------- .../tflite_settings_apple_iphone11.pbtxt | 8 +++--- .../tflite_settings_apple_iphone12.pbtxt | 8 +++--- .../tflite_settings_apple_iphoneX.pbtxt | 8 +++--- .../tflite_settings_apple_main.pbtxt | 8 +++--- 7 files changed, 57 insertions(+), 51 deletions(-) diff --git a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt index 7e8389600..758d675cc 100644 --- a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt +++ b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt @@ -65,22 +65,22 @@ benchmark_setting { delegate_name: "CPU & GPU & ANE" accelerator_name: "cpu&gpu&ane" accelerator_desc: "All compute units" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & GPU" accelerator_name: "cpu&gpu" accelerator_desc: "CPU and GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_selected: "CPU & GPU & ANE" } @@ -93,24 +93,24 @@ benchmark_setting { accelerator_name: "cpu&gpu&ane" accelerator_desc: "All compute units" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & GPU" accelerator_name: "cpu&gpu" accelerator_desc: "CPU and GPU" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_selected: "CPU & GPU & ANE" } diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 1015847c5..863d4909b 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -60,11 +60,13 @@ tflite-run-ic-v2: bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification_v2 \ --mode=PerformanceOnly \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-fp32.tflite" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 tflite-run-ic-offline-v2: cd ${REPO_ROOT_DIR} && \ @@ -73,11 +75,13 @@ tflite-run-ic-offline-v2: --scenario=Offline \ --batch_size=32 \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-fp32.tflite" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 tflite-run-od: cd ${REPO_ROOT_DIR} && \ @@ -155,11 +159,13 @@ coreml-run-ic-v2: bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification \ --mode=PerformanceOnly \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 coreml-run-ic-offline-v2: cd ${REPO_ROOT_DIR} && \ @@ -168,11 +174,13 @@ coreml-run-ic-offline-v2: --scenario=Offline \ --batch_size=32 \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 coreml-run-od: cd ${REPO_ROOT_DIR} && \ diff --git a/mobile_back_apple/models/icv2.py b/mobile_back_apple/models/icv2.py index 61ff905e9..4af313776 100644 --- a/mobile_back_apple/models/icv2.py +++ b/mobile_back_apple/models/icv2.py @@ -16,36 +16,34 @@ # limitations under the License. import coremltools as ct -import tensorflow as tf -import tensorflow_hub as hub def main(): - print("Retrieve model from TFHub...") - keras_layer = hub.KerasLayer('https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1') - keras_model = tf.keras.Sequential([keras_layer]) - keras_model.build([None, 224, 224, 3]) - input_tensor = tf.ones((4, 224, 224, 3)) - output_tensor = keras_model(input_tensor) - print("output_tensor.shape:", output_tensor.shape) - + """ + Download the MobileNetV4-Conv-Large-fp32 from https://github.com/mlcommons/mobile_open/releases + Tested with tensorflow==2.15.0 and coremltools==7.1 + """ + saved_model_dir = '../dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-saved-model' + export_fpath = '../dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel' + print("Converting model...") model = ct.convert( - keras_model, + saved_model_dir, + source="tensorflow", convert_to="neuralnetwork", - inputs=[ct.TensorType(shape=(1, 224, 224, 3))], + inputs=[ct.TensorType(shape=(1, 384, 384, 3))], ) - model.short_description = "MobilenetEdgeTPUv2 from https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1" + model.short_description = "MobileNetV4-Conv-Large-fp32 from https://github.com/mlcommons/mobile_open" spec = model.get_spec() for n in (1, 1001): spec.description.output[0].type.multiArrayType.shape.append(n) - ct.utils.rename_feature(spec, "keras_layer_input", "images") + ct.utils.rename_feature(spec, "inputs", "images") ct.utils.rename_feature(spec, "Identity", "Softmax") print(spec.description) - export_fpath = '../dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel' ct.models.MLModel(spec).save(export_fpath) + print("Done! Core ML model exported to:", export_fpath) if __name__ == "__main__": diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt index 8669e7fb4..4b60e43c8 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 64 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 64 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt index 44b482330..2b77ca063 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 8 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 8 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt index f90255d22..6285f4eeb 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 32 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 32 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt index 42cd0e2d1..355aa3d42 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt @@ -38,15 +38,15 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "Core ML" } From d15f5a8ff25db30880cfdd5b7d120b8416d019ff Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 13 Feb 2024 11:38:54 +0700 Subject: [PATCH 21/28] Fix tflite_settings_windows.pbtxt --- .../backend_settings/tflite_settings_windows.pbtxt | 3 --- 1 file changed, 3 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt index 71b8821a9..b9772f27a 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt @@ -47,9 +47,6 @@ benchmark_setting { accelerator_desc: "CPU" model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" model_checksum: "590a7a88640a18d28b16b6f571cdfc93" - } - delegate_selected: "NNAPI" - } delegate_selected: "CPU" } From dc42a63710e07260a0d80bcb8a3f905fb9027a0c Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 13 Feb 2024 11:43:31 +0700 Subject: [PATCH 22/28] Update expected_accuracy --- flutter/integration_test/expected_accuracy.dart | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index 880f07fe4..f63a2779b 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -28,14 +28,14 @@ const Map _imageClassification = { }; const Map _imageClassificationV2 = { - 'cpu': Interval(min: 0.69, max: 0.71), - 'npu': Interval(min: 0.69, max: 0.71), - 'tpu': Interval(min: 0.69, max: 0.71), - 'ane': Interval(min: 0.69, max: 0.71), - 'cpu&gpu&ane': Interval(min: 0.69, max: 0.71), - 'snpe_dsp': Interval(min: 0.69, max: 0.71), - 'psnpe_dsp': Interval(min: 0.69, max: 0.71), - 'neuron-mdla': Interval(min: 0.69, max: 0.71), + 'cpu': Interval(min: 0.88, max: 0.91), + 'npu': Interval(min: 0.88, max: 0.91), + 'tpu': Interval(min: 0.88, max: 0.91), + 'ane': Interval(min: 0.88, max: 0.91), + 'cpu&gpu&ane': Interval(min: 0.88, max: 0.91), + 'snpe_dsp': Interval(min: 0.88, max: 0.91), + 'psnpe_dsp': Interval(min: 0.88, max: 0.91), + 'neuron-mdla': Interval(min: 0.79, max: 0.81), }; const Map _objectDetection = { From f980d58da4895a00929e23416e76f1c42d23d678 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 13 Feb 2024 14:17:18 +0700 Subject: [PATCH 23/28] Update expected_accuracy and expected_throughput for Apple devices --- flutter/integration_test/expected_accuracy.dart | 4 ++-- flutter/integration_test/expected_throughput.dart | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index f63a2779b..dc50aee8b 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -31,8 +31,8 @@ const Map _imageClassificationV2 = { 'cpu': Interval(min: 0.88, max: 0.91), 'npu': Interval(min: 0.88, max: 0.91), 'tpu': Interval(min: 0.88, max: 0.91), - 'ane': Interval(min: 0.88, max: 0.91), - 'cpu&gpu&ane': Interval(min: 0.88, max: 0.91), + 'ane': Interval(min: 0.79, max: 0.91), + 'cpu&gpu&ane': Interval(min: 0.79, max: 0.91), 'snpe_dsp': Interval(min: 0.88, max: 0.91), 'psnpe_dsp': Interval(min: 0.88, max: 0.91), 'neuron-mdla': Interval(min: 0.79, max: 0.81), diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 29c215a53..1d06a7580 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -59,11 +59,11 @@ const Map> _imageClassificationV2 = { _kCloudBuildX28: Interval(min: 2, max: 12), _kRyzen5600: Interval(min: 1, max: 37), _kPixel5: Interval(min: 60, max: 75), - _kIphoneOnGitHubAction: Interval(min: 1, max: 7), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), _kIphoneOnMacbookM1: Interval(min: 10, max: 27), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 1, max: 7), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), }, _kPixelBackend: { _kPixel6: Interval(min: 500, max: 600), @@ -199,11 +199,11 @@ const Map> _imageClassificationOfflineV2 = { _kCloudBuildX28: Interval(min: 3, max: 14), _kRyzen5600: Interval(min: 20, max: 60), _kPixel5: Interval(min: 60, max: 180), - _kIphoneOnGitHubAction: Interval(min: 1, max: 15), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), _kIphoneOnMacbookM1: Interval(min: 10, max: 45), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 1, max: 15), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), }, _kPixelBackend: { _kPixel6: Interval(min: 700, max: 1700), From 441cbf7d0f2b82f4e9fe032f2451bb1d982c7ac2 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 13 Feb 2024 15:37:44 +0700 Subject: [PATCH 24/28] Use fp32 model for Windows --- .../backend_settings/tflite_settings_windows.pbtxt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt index b9772f27a..6bfad7fd5 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt @@ -45,8 +45,8 @@ benchmark_setting { delegate_name: "CPU" accelerator_name: "cpu" accelerator_desc: "CPU" - model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" - model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "CPU" } @@ -59,8 +59,8 @@ benchmark_setting { accelerator_name: "cpu" accelerator_desc: "CPU" batch_size: 2 - model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" - model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "CPU" } From 73dc65848ddd8ac1fe680cb3baf722da65b5d29c Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 13 Feb 2024 18:38:23 +0700 Subject: [PATCH 25/28] Update expected_throughput for Windows --- flutter/integration_test/expected_throughput.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 1d06a7580..752718544 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -55,8 +55,8 @@ const Map> _imageClassification = { // TODO (anhappdev): update the expected value for _imageClassificationV2 after gathering some statistics const Map> _imageClassificationV2 = { _kTFLiteBackend: { - _kCloudBuildX23: Interval(min: 2, max: 11), - _kCloudBuildX28: Interval(min: 2, max: 12), + _kCloudBuildX23: Interval(min: 1, max: 9), + _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 1, max: 37), _kPixel5: Interval(min: 60, max: 75), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), @@ -89,7 +89,7 @@ const Map> _objectDetection = { _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4), }, _kPixelBackend: { - _kPixel6: Interval(min: 250, max: 450), + _kPixel6: Interval(min: 200, max: 450), }, _kQtiBackend: { _kS22Ultra: Interval(min: 800, max: 1400), @@ -195,8 +195,8 @@ const Map> _imageClassificationOffline = { // TODO (anhappdev): update the expected value for _imageClassificationOfflineV2 after gathering some statistics const Map> _imageClassificationOfflineV2 = { _kTFLiteBackend: { - _kCloudBuildX23: Interval(min: 4, max: 14), - _kCloudBuildX28: Interval(min: 3, max: 14), + _kCloudBuildX23: Interval(min: 1, max: 9), + _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 20, max: 60), _kPixel5: Interval(min: 60, max: 180), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), From 11e6c4c3568288b43a8d97b767d6676170821fab Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 13 Feb 2024 18:49:43 +0700 Subject: [PATCH 26/28] Update expected_throughput for Android --- flutter/integration_test/expected_throughput.dart | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 752718544..85c7a3c59 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -58,7 +58,7 @@ const Map> _imageClassificationV2 = { _kCloudBuildX23: Interval(min: 1, max: 9), _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 1, max: 37), - _kPixel5: Interval(min: 60, max: 75), + _kPixel5: Interval(min: 20, max: 75), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), _kIphoneOnMacbookM1: Interval(min: 10, max: 27), }, @@ -66,13 +66,13 @@ const Map> _imageClassificationV2 = { _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), }, _kPixelBackend: { - _kPixel6: Interval(min: 500, max: 600), + _kPixel6: Interval(min: 100, max: 600), }, _kQtiBackend: { _kS22Ultra: Interval(min: 1900, max: 2200), }, _kMediatekBackend: { - _kDN2103: Interval(min: 30, max: 50), + _kDN2103: Interval(min: 5, max: 50), }, }; @@ -198,7 +198,7 @@ const Map> _imageClassificationOfflineV2 = { _kCloudBuildX23: Interval(min: 1, max: 9), _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 20, max: 60), - _kPixel5: Interval(min: 60, max: 180), + _kPixel5: Interval(min: 20, max: 180), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), _kIphoneOnMacbookM1: Interval(min: 10, max: 45), }, @@ -206,13 +206,13 @@ const Map> _imageClassificationOfflineV2 = { _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), }, _kPixelBackend: { - _kPixel6: Interval(min: 700, max: 1700), + _kPixel6: Interval(min: 100, max: 1700), }, _kQtiBackend: { _kS22Ultra: Interval(min: 2600, max: 3000), }, _kMediatekBackend: { - _kDN2103: Interval(min: 75, max: 110), + _kDN2103: Interval(min: 5, max: 90), }, }; From 5c2717d6c99786d140f4865630419c30b98f1602 Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Tue, 13 Feb 2024 10:55:06 -0600 Subject: [PATCH 27/28] Update S3 provider in android-build-test.yml (#855) * Update S3 provider in android-build-test.yml * Update env var name --------- Co-authored-by: Anh --- .github/workflows/android-build-test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 0ac420f01..8ced2cd29 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -86,10 +86,10 @@ jobs: python3 -m pip install awscli - name: Download Samsung libraries env: - AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }} run: | - aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \ + aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \ cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/samsung_libs_v3.1_20231116.zip /tmp/ && \ unzip /tmp/samsung_libs_v3.1_20231116.zip -d /tmp/samsung_libs_v3.1_20231116 && \ rm /tmp/samsung_libs_v3.1_20231116.zip && \ @@ -97,12 +97,12 @@ jobs: mv /tmp/samsung_libs_v3.1_20231116/* mobile_back_samsung/samsung/lib/internal/ - name: Download QTI libraries env: - AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }} run: | - aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \ + aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \ cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/qaisw-2.13.4.230831213719_60417-linux.zip /tmp/ && \ - unzip -P ${AWS_SECRET_ACCESS_KEY} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \ + unzip -P ${{ secrets.READONLY_AWS_KEY }} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \ rm /tmp/qaisw-2.13.4.230831213719_60417-linux.zip && \ mv /tmp/qaisw-2.13.4.230831213719_60417-linux/qaisw-2.13.4.230831213719_60417 mobile_back_qti/ - name: Cache bazel From ce5e74a73460631a1215a3367485f4cd40cdc392 Mon Sep 17 00:00:00 2001 From: Anh Date: Wed, 21 Feb 2024 11:52:10 +0700 Subject: [PATCH 28/28] Reduce batch_size for image_classification_offline_v2 in Pixel 6 --- .../cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 3db4ab8ea..6f07fee5f 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -79,7 +79,7 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - batch_size: 32 + batch_size: 16 model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" model_checksum: "590a7a88640a18d28b16b6f571cdfc93" }