From c78b889b149e47cdb3fc00e40e9c67490d961149 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 15 Oct 2024 13:10:48 +0700 Subject: [PATCH 01/27] feat: pass task-specific config to backend (#922) * Add TaskConfig.CustomConfig and pass them to backend * Add CustomConfig for main.cc * Use seed and num_steps from CustomConfig for TFLite backend * Replace std::cout with LOG(INFO) * Format files --- flutter/assets/tasks.pbtxt | 8 +++ flutter/cpp/binary/main.cc | 10 ++-- flutter/cpp/proto/mlperf_task.proto | 14 ++++- flutter/cpp/proto/test.cc | 4 +- flutter/cpp/utils.cc | 51 ++++++++++++++++--- flutter/cpp/utils.h | 5 +- flutter/lib/benchmark/benchmark.dart | 5 ++ mobile_back_apple/dev-utils/Makefile | 3 +- .../stable_diffusion_invoker.cc | 8 +-- .../stable_diffusion_pipeline.cc | 9 ++++ 10 files changed, 97 insertions(+), 20 deletions(-) diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index d9c05491a..b7e31935b 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -247,4 +247,12 @@ task { id: "stable_diffusion" name: "StableDiffusion" } + custom_config { + id: "stable_diffusion_seed" + value: "633994880" + } + custom_config { + id: "stable_diffusion_num_steps" + value: "20" + } } diff --git a/flutter/cpp/binary/main.cc b/flutter/cpp/binary/main.cc index f2f3e51c9..89f5758ad 100644 --- a/flutter/cpp/binary/main.cc +++ b/flutter/cpp/binary/main.cc @@ -132,7 +132,7 @@ int Main(int argc, char *argv[]) { command_line += " " + backend_name + " " + benchmark_id; // Command Line Flags for mlperf. - std::string mode, scenario = "SingleStream", output_dir; + std::string mode, scenario = "SingleStream", output_dir, custom_config; int min_query_count = 100, min_duration_ms = 100, max_duration_ms = 10 * 60 * 1000, single_stream_expected_latency_ns = 1000000; @@ -157,8 +157,9 @@ int Main(int argc, char *argv[]) { "A hint used by the loadgen to pre-generate " "enough samples to meet the minimum test duration."), Flag::CreateFlag("output_dir", &output_dir, - "The output directory of mlperf.", Flag::kRequired)}); - + "The output directory of mlperf.", Flag::kRequired), + Flag::CreateFlag("custom_config", &custom_config, + "Custom config in form key1:val1,key2:val2.")}); // Command Line Flags for backend. std::unique_ptr backend; std::unique_ptr dataset; @@ -207,9 +208,8 @@ int Main(int argc, char *argv[]) { } } } - SettingList setting_list = - createSettingList(backend_setting, benchmark_id); + CreateSettingList(backend_setting, custom_config, benchmark_id); ExternalBackend *external_backend = new ExternalBackend( model_file_path, lib_path, setting_list, native_lib_path); diff --git a/flutter/cpp/proto/mlperf_task.proto b/flutter/cpp/proto/mlperf_task.proto index 53e545e15..5c1012490 100644 --- a/flutter/cpp/proto/mlperf_task.proto +++ b/flutter/cpp/proto/mlperf_task.proto @@ -31,7 +31,7 @@ message MLPerfConfig { // Config of the mlperf tasks. // A task is basically a combination of models and a dataset. // -// Next ID: 11 +// Next ID: 12 message TaskConfig { // Must be unique in one task file. Ex: image_classification // used to match backend settings @@ -52,6 +52,7 @@ message TaskConfig { required string scenario = 7; required DatasetConfig datasets = 8; required ModelConfig model = 9; + repeated CustomConfig custom_config = 11; } // Datasets for a task @@ -107,3 +108,14 @@ message ModelConfig { // Number of detection classes if applicable optional int32 num_classes = 6; } + +// CustomConfig are task specific configuration. +// The TaskConfig.CustomConfig will be converted to +// BenchmarkSetting.CustomSetting and passed to the backend. +// To avoid name collision, the id should be prefixed with TaskConfig.id. +message CustomConfig { + // Id of this config. + required string id = 1; + // Value of this config. + required string value = 2; +} diff --git a/flutter/cpp/proto/test.cc b/flutter/cpp/proto/test.cc index 07d4fd3fd..6a66969eb 100644 --- a/flutter/cpp/proto/test.cc +++ b/flutter/cpp/proto/test.cc @@ -139,9 +139,11 @@ int test_proto() { std::list benchmarks; benchmarks.push_back("image_classification"); benchmarks.push_back("image_classification_offline"); + std::string custom_config = "key1:val1,key2:val2"; for (auto benchmark_id : benchmarks) { // Convert to SettingList - SettingList setting_list = createSettingList(backend_setting, benchmark_id); + SettingList setting_list = + CreateSettingList(backend_setting, custom_config, benchmark_id); std::cout << "SettingList for " << benchmark_id << ":\n"; dumpSettingList(setting_list); diff --git a/flutter/cpp/utils.cc b/flutter/cpp/utils.cc index 965426f4d..7cea64c91 100644 --- a/flutter/cpp/utils.cc +++ b/flutter/cpp/utils.cc @@ -125,22 +125,61 @@ mlperf_backend_configuration_t CppToCSettings(const SettingList &settings) { return c_settings; } -SettingList createSettingList(const BackendSetting &backend_setting, - std::string benchmark_id) { +// Split the string by a given delimiter +std::vector _splitString(const std::string &str, char delimiter) { + std::vector tokens; + std::stringstream ss(str); + std::string token; + while (std::getline(ss, token, delimiter)) { + tokens.push_back(token); + } + return tokens; +} + +// Parse the key:value string list +std::unordered_map _parseKeyValueList( + const std::string &input) { + std::unordered_map keyValueMap; + std::vector pairs = _splitString(input, ','); // Split by comma + + for (const std::string &pair : pairs) { + std::vector keyValue = + _splitString(pair, ':'); // Split by colon + if (keyValue.size() == 2) { + keyValueMap[keyValue[0]] = keyValue[1]; + } else { + LOG(ERROR) << "Invalid key:value pair: " << pair; + } + } + return keyValueMap; +} + +// Create the setting list for backend +SettingList CreateSettingList(const BackendSetting &backend_setting, + const std::string &custom_config, + const std::string &benchmark_id) { SettingList setting_list; int setting_index = 0; - - for (auto setting : backend_setting.common_setting()) { + for (const auto &setting : backend_setting.common_setting()) { setting_list.add_setting(); (*setting_list.mutable_setting(setting_index)) = setting; setting_index++; } // Copy the benchmark specific settings - setting_index = 0; - for (auto bm_setting : backend_setting.benchmark_setting()) { + for (const auto &bm_setting : backend_setting.benchmark_setting()) { if (bm_setting.benchmark_id() == benchmark_id) { setting_list.mutable_benchmark_setting()->CopyFrom(bm_setting); + + auto parsed = _parseKeyValueList(custom_config); + for (const auto &kv : parsed) { + CustomSetting custom_setting = CustomSetting(); + custom_setting.set_id(kv.first); + custom_setting.set_value(kv.second); + setting_list.mutable_benchmark_setting()->mutable_custom_setting()->Add( + std::move(custom_setting)); + } + break; } } LOG(INFO) << "setting_list:" << std::endl << setting_list.DebugString(); diff --git a/flutter/cpp/utils.h b/flutter/cpp/utils.h index 08d7e86c8..17d590024 100644 --- a/flutter/cpp/utils.h +++ b/flutter/cpp/utils.h @@ -65,8 +65,9 @@ void DeleteBackendConfiguration(mlperf_backend_configuration_t *configs); mlperf_backend_configuration_t CppToCSettings(const SettingList &settings); -SettingList createSettingList(const BackendSetting &backend_setting, - std::string benchmark_id); +SettingList CreateSettingList(const BackendSetting &backend_setting, + const std::string &custom_config, + const std::string &benchmark_id); } // namespace mobile } // namespace mlperf diff --git a/flutter/lib/benchmark/benchmark.dart b/flutter/lib/benchmark/benchmark.dart index 1ac2d00bd..511f45788 100644 --- a/flutter/lib/benchmark/benchmark.dart +++ b/flutter/lib/benchmark/benchmark.dart @@ -92,6 +92,11 @@ class Benchmark { setting: commonSettings, benchmarkSetting: benchmarkSettings, ); + // Convert TaskConfig.CustomConfig to BenchmarkSetting.CustomSetting + final customConfigs = taskConfig.customConfig + .map((e) => pb.CustomSetting(id: e.id, value: e.value)) + .toList(); + benchmarkSettings.customSetting.addAll(customConfigs); final uris = selectedDelegate.modelFile.map((e) => e.modelPath).toList(); final modelDirName = selectedDelegate.delegateName.replaceAll(' ', '_'); final backendModelPath = diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 7ca5c27ca..32d196fb1 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -131,8 +131,9 @@ tflite-run-sd: bazel-bin/flutter/cpp/binary/main EXTERNAL stable_diffusion \ --mode=PerformanceOnly \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/sd-models" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/dynamic-sd-models" \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ + --custom_config="stable_diffusion_num_steps:20,stable_diffusion_seed:633994880" \ --input_tfrecord="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/coco_gen_full.tfrecord" \ --input_clip_model="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/clip_model_512x512.tflite" \ --min_query_count=5 diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc index 55ea8be07..94ac36ed2 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc @@ -24,15 +24,15 @@ StableDiffusionInvoker::StableDiffusionInvoker(SDBackendData* backend_data) : backend_data_(backend_data) {} std::vector StableDiffusionInvoker::invoke() { - std::cout << "Prompt encoding started" << std::endl; + LOG(INFO) << "Prompt encoding started"; auto encoded_text = encode_prompt(backend_data_->input_prompt_tokens); auto unconditional_encoded_text = encode_prompt(backend_data_->unconditional_tokens); - std::cout << "Diffusion process started" << std::endl; + LOG(INFO) << "Diffusion process started"; auto latent = diffusion_process(encoded_text, unconditional_encoded_text, backend_data_->num_steps, backend_data_->seed); - std::cout << "Image decoding started" << std::endl; + LOG(INFO) << "Image decoding started"; return decode_image(latent); } @@ -108,7 +108,7 @@ std::vector StableDiffusionInvoker::diffusion_process( auto alphas_prev = std::get<1>(alphas_tuple); for (int i = timesteps.size() - 1; i >= 0; --i) { - std::cout << "Step " << timesteps.size() - 1 - i << "\n"; + LOG(INFO) << "Step " << timesteps.size() - 1 - i; auto latent_prev = latent; auto t_emb = get_timestep_embedding(timesteps[i]); diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc index 52d20b570..1003aaad3 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc @@ -64,6 +64,15 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( SDBackendData* backend_data = new SDBackendData(); backendExists = true; + for (int i = 0; i < configs->count; ++i) { + if (strcmp(configs->keys[i], "stable_diffusion_seed") == 0) { + backend_data->seed = atoi(configs->values[i]); + } + if (strcmp(configs->keys[i], "stable_diffusion_num_steps") == 0) { + backend_data->num_steps = atoi(configs->values[i]); + } + } + // Load models from the provided directory path std::string text_encoder_path = std::string(model_path) + "/sd_text_encoder_dynamic.tflite"; From fad36c149fb710aadd4c5df9f55f3166090d9c46 Mon Sep 17 00:00:00 2001 From: Anh Date: Wed, 23 Oct 2024 16:09:06 +0700 Subject: [PATCH 02/27] feat: add ConvertOutputs() API (#927) * Add ConvertOutputs() API * Add ConvertOutputs() for mobile_back_tflite * Set minimum macos version * Set minimum macos version to 13.1 * Update _kIphoneOnGitHubAction --- .github/workflows/ios-build-test.yml | 2 +- flutter/cpp/backend.h | 4 ++++ flutter/cpp/backends/external.cc | 4 +++- flutter/cpp/backends/external.h | 12 ++++++++++++ flutter/cpp/c/backend_c.h | 2 ++ flutter/cpp/datasets/coco_gen.cc | 6 ++++++ flutter/cpp/flutter/BUILD | 1 + flutter/integration_test/expected_throughput.dart | 2 +- mobile_back_apple/cpp/backend_coreml/BUILD | 1 + mobile_back_apple/dev-utils/Makefile | 5 +++-- mobile_back_tflite/cpp/backend_dummy/ios/BUILD | 2 ++ mobile_back_tflite/cpp/backend_tflite/ios/BUILD | 1 + mobile_back_tflite/cpp/backend_tflite/pipeline.h | 7 ++++++- .../cpp/backend_tflite/single_model_pipeline.cc | 4 ++++ .../cpp/backend_tflite/single_model_pipeline.h | 3 +++ .../cpp/backend_tflite/stable_diffusion_pipeline.cc | 4 ++++ .../cpp/backend_tflite/stable_diffusion_pipeline.h | 3 +++ mobile_back_tflite/cpp/backend_tflite/tflite_c.cc | 6 ++++++ 18 files changed, 63 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ios-build-test.yml b/.github/workflows/ios-build-test.yml index 2d2b03c6e..4dbc67bef 100644 --- a/.github/workflows/ios-build-test.yml +++ b/.github/workflows/ios-build-test.yml @@ -10,7 +10,7 @@ jobs: build: name: Build and test iOS app # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md - runs-on: macos-12 + runs-on: macos-13 timeout-minutes: 120 env: PERF_TEST: true diff --git a/flutter/cpp/backend.h b/flutter/cpp/backend.h index 5aa20dded..deb8fdc89 100644 --- a/flutter/cpp/backend.h +++ b/flutter/cpp/backend.h @@ -74,6 +74,10 @@ class Backend { virtual void ConvertInputs(int bytes, int image_width, int image_height, uint8_t* data) = 0; + // Allow backend to do output layout change + virtual void ConvertOutputs(int bytes, int image_width, int image_height, + uint8_t* data) = 0; + private: BackendSetting settings_; }; diff --git a/flutter/cpp/backends/external.cc b/flutter/cpp/backends/external.cc index e6e7aed28..3675e4943 100644 --- a/flutter/cpp/backends/external.cc +++ b/flutter/cpp/backends/external.cc @@ -187,7 +187,9 @@ BackendFunctions::BackendFunctions(const std::string& lib_path) { // Backends may need to change the format of the inputs (e.g. channel order) convert_inputs = reinterpret_cast( CheckSymbol("mlperf_backend_convert_inputs")); - + // Backends may need to change the format of the outputs (e.g. channel order) + convert_outputs = reinterpret_cast( + CheckSymbol("mlperf_backend_convert_outputs")); // If both functions are defined, then update if (get_buffer && release_buffer) { LOG(INFO) << "Using backend allocator"; diff --git a/flutter/cpp/backends/external.h b/flutter/cpp/backends/external.h index 7a2ef9b8d..12e17357e 100644 --- a/flutter/cpp/backends/external.h +++ b/flutter/cpp/backends/external.h @@ -67,6 +67,8 @@ struct BackendFunctions { mlperf_backend_ptr_t, uint32_t, int32_t, void**)>::type; using ConvertInputsPtr = std::add_pointer::type; + using ConvertOutputsPtr = std::add_pointer::type; // Required functions. BackendMatchesPtr match{nullptr}; @@ -91,6 +93,7 @@ struct BackendFunctions { AllocatorMgr::GetBufferFn get_buffer{nullptr}; AllocatorMgr::ReleaseBufferFn release_buffer{nullptr}; ConvertInputsPtr convert_inputs{nullptr}; + ConvertOutputsPtr convert_outputs{nullptr}; bool isLoaded() { return isloaded; } @@ -210,6 +213,15 @@ class ExternalBackend : public Backend { } } + // Optional function to do output data re-formatting + void ConvertOutputs(int bytes, int width, int height, + uint8_t* data) override { + if (backend_functions_.convert_outputs) { + backend_functions_.convert_outputs(backend_ptr_, bytes, width, height, + data); + } + } + private: std::string backend_name_; std::string vendor_; diff --git a/flutter/cpp/c/backend_c.h b/flutter/cpp/c/backend_c.h index dd863501b..47d3c9bb5 100644 --- a/flutter/cpp/c/backend_c.h +++ b/flutter/cpp/c/backend_c.h @@ -82,6 +82,8 @@ mlperf_status_t mlperf_backend_get_output(mlperf_backend_ptr_t backend_ptr, // Optional functions void mlperf_backend_convert_inputs(mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, uint8_t* data); +void mlperf_backend_convert_outputs(mlperf_backend_ptr_t backend_ptr, int bytes, + int width, int height, uint8_t* data); #ifdef __cplusplus } diff --git a/flutter/cpp/datasets/coco_gen.cc b/flutter/cpp/datasets/coco_gen.cc index 1b7a75ef8..6fbd161b3 100644 --- a/flutter/cpp/datasets/coco_gen.cc +++ b/flutter/cpp/datasets/coco_gen.cc @@ -81,6 +81,8 @@ void CocoGen::UnloadSamplesFromRam( } } +#define OUTPUT_WIDTH 512 +#define OUTPUT_HEIGHT 512 #define OUTPUT_SIZE 512 * 512 * 3 std::vector CocoGen::ProcessOutput(const int sample_idx, const std::vector& outputs) { @@ -100,6 +102,10 @@ std::vector CocoGen::ProcessOutput(const int sample_idx, } } + auto total_byte = output_format_[0].size * GetByte(output_format_[0]); + backend_->ConvertOutputs(total_byte, OUTPUT_WIDTH, OUTPUT_HEIGHT, + output_pixels.data()); + std::string raw_output_filename = raw_output_dir_ + "/output_" + std::to_string(sample_idx) + ".rgb8"; dump_output_pixels(output_pixels, raw_output_filename); diff --git a/flutter/cpp/flutter/BUILD b/flutter/cpp/flutter/BUILD index 0ac2f1b7e..eb5ddb103 100644 --- a/flutter/cpp/flutter/BUILD +++ b/flutter/cpp/flutter/BUILD @@ -55,6 +55,7 @@ apple_xcframework( }, minimum_os_versions = { "ios": "13.1", + "macos": "13.1", }, deps = [ "//flutter/cpp/flutter:bridge", diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 342254e59..d86744803 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -26,7 +26,7 @@ const _kS22Ultra = 'SM-S908U1'; // Galaxy S22 Ultra const _kDN2103 = 'DN2103'; // OnePlus DN2103 // iOS -const _kIphoneOnGitHubAction = 'iPhone15,3'; +const _kIphoneOnGitHubAction = 'iPhone16,2'; const _kIphoneOnMacbookM1 = 'iPhone14,7'; const Map> _imageClassificationV2 = { diff --git a/mobile_back_apple/cpp/backend_coreml/BUILD b/mobile_back_apple/cpp/backend_coreml/BUILD index c574b8a9c..4e8acf07d 100644 --- a/mobile_back_apple/cpp/backend_coreml/BUILD +++ b/mobile_back_apple/cpp/backend_coreml/BUILD @@ -39,6 +39,7 @@ apple_xcframework( }, minimum_os_versions = { "ios": "13.1", + "macos": "13.1", }, deps = [ "//mobile_back_apple/cpp/backend_coreml:coreml_c", diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 32d196fb1..2ef789f10 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -24,7 +24,8 @@ app: tflite-build: cd ${REPO_ROOT_DIR} && \ - bazel build -c opt --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //flutter/cpp/binary:main //mobile_back_tflite:tflitebackend + bazel build -c opt --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=12.1 \ + //flutter/cpp/binary:main //mobile_back_tflite:tflitebackend tflite-run-ic: cd ${REPO_ROOT_DIR} && \ @@ -140,7 +141,7 @@ tflite-run-sd: coreml-build: cd ${REPO_ROOT_DIR} && \ - bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 \ + bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=12.1 \ //flutter/cpp/binary:main //mobile_back_apple:coremlbackend coreml-run-ic: diff --git a/mobile_back_tflite/cpp/backend_dummy/ios/BUILD b/mobile_back_tflite/cpp/backend_dummy/ios/BUILD index c7adadbca..a3229e059 100644 --- a/mobile_back_tflite/cpp/backend_dummy/ios/BUILD +++ b/mobile_back_tflite/cpp/backend_dummy/ios/BUILD @@ -29,6 +29,7 @@ apple_xcframework( }, minimum_os_versions = { "ios": "13.1", + "macos": "13.1", }, deps = [ "//mobile_back_tflite/cpp/backend_dummy:dummy_backend", @@ -49,6 +50,7 @@ apple_xcframework( }, minimum_os_versions = { "ios": "13.1", + "macos": "13.1", }, deps = [ "//mobile_back_tflite/cpp/backend_dummy:dummy_backend", diff --git a/mobile_back_tflite/cpp/backend_tflite/ios/BUILD b/mobile_back_tflite/cpp/backend_tflite/ios/BUILD index f0a764a6d..74fa88aea 100644 --- a/mobile_back_tflite/cpp/backend_tflite/ios/BUILD +++ b/mobile_back_tflite/cpp/backend_tflite/ios/BUILD @@ -15,6 +15,7 @@ apple_xcframework( }, minimum_os_versions = { "ios": "13.1", + "macos": "13.1", }, deps = [ "//mobile_back_tflite/cpp/backend_tflite:tflite_c", diff --git a/mobile_back_tflite/cpp/backend_tflite/pipeline.h b/mobile_back_tflite/cpp/backend_tflite/pipeline.h index 41a9822f2..4ab1b4f1c 100644 --- a/mobile_back_tflite/cpp/backend_tflite/pipeline.h +++ b/mobile_back_tflite/cpp/backend_tflite/pipeline.h @@ -72,11 +72,16 @@ class Pipeline { virtual mlperf_status_t backend_get_output(mlperf_backend_ptr_t backend_ptr, uint32_t batchIndex, int32_t i, void **data) = 0; - + // Optional function to convert the inputs virtual void backend_convert_inputs(mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, uint8_t *data) = 0; + // Optional function to convert the outputs + virtual void backend_convert_outputs(mlperf_backend_ptr_t backend_ptr, + int bytes, int width, int height, + uint8_t *data) = 0; + virtual void *backend_get_buffer(size_t n) = 0; virtual void backend_release_buffer(void *p) = 0; diff --git a/mobile_back_tflite/cpp/backend_tflite/single_model_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/single_model_pipeline.cc index 4dc30507b..ce1eb7a1d 100644 --- a/mobile_back_tflite/cpp/backend_tflite/single_model_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/single_model_pipeline.cc @@ -635,6 +635,10 @@ void SingleModelPipeline::backend_convert_inputs( #endif } +void SingleModelPipeline::backend_convert_outputs( + mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, + uint8_t *data) {} + void *SingleModelPipeline::backend_get_buffer(size_t n) { #ifdef MTK_TFLITE_NEURON_BACKEND if (neuron_backend != nullptr) { diff --git a/mobile_back_tflite/cpp/backend_tflite/single_model_pipeline.h b/mobile_back_tflite/cpp/backend_tflite/single_model_pipeline.h index 30c639596..70d447588 100644 --- a/mobile_back_tflite/cpp/backend_tflite/single_model_pipeline.h +++ b/mobile_back_tflite/cpp/backend_tflite/single_model_pipeline.h @@ -64,6 +64,9 @@ class SingleModelPipeline : public Pipeline { void backend_convert_inputs(mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, uint8_t *data) override; + void backend_convert_outputs(mlperf_backend_ptr_t backend_ptr, int bytes, + int width, int height, uint8_t *data) override; + void *backend_get_buffer(size_t n) override; void backend_release_buffer(void *p) override; diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc index 1003aaad3..358f031c9 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc @@ -277,6 +277,10 @@ void StableDiffusionPipeline::backend_convert_inputs( mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, uint8_t* data) {} +void StableDiffusionPipeline::backend_convert_outputs( + mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, + uint8_t* data) {} + void* StableDiffusionPipeline::backend_get_buffer(size_t n) { return ::operator new(n); } diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h index c16a7d5c8..7104cb24e 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h @@ -91,6 +91,9 @@ class StableDiffusionPipeline : public Pipeline { void backend_convert_inputs(mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, uint8_t *data) override; + void backend_convert_outputs(mlperf_backend_ptr_t backend_ptr, int bytes, + int width, int height, uint8_t *data) override; + void *backend_get_buffer(size_t n) override; void backend_release_buffer(void *p) override; diff --git a/mobile_back_tflite/cpp/backend_tflite/tflite_c.cc b/mobile_back_tflite/cpp/backend_tflite/tflite_c.cc index dced8bf1d..62a6a18bc 100644 --- a/mobile_back_tflite/cpp/backend_tflite/tflite_c.cc +++ b/mobile_back_tflite/cpp/backend_tflite/tflite_c.cc @@ -229,6 +229,12 @@ void mlperf_backend_convert_inputs(mlperf_backend_ptr_t backend_ptr, int bytes, data); } +void mlperf_backend_convert_outputs(mlperf_backend_ptr_t backend_ptr, int bytes, + int width, int height, uint8_t *data) { + return pipeline->backend_convert_outputs(backend_ptr, bytes, width, height, + data); +} + void *mlperf_backend_get_buffer(size_t n) { return pipeline->backend_get_buffer(n); } From 48cb33c542af9a5ccdad1937a777da814823e435 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Mon, 28 Oct 2024 22:14:05 +0200 Subject: [PATCH 03/27] feat: timestamp-embedding-parser (WIP) --- mobile_back_tflite/cpp/backend_tflite/BUILD | 2 + .../cpp/backend_tflite/embedding_utils.cc | 93 +++++++++++++++++++ .../cpp/backend_tflite/embedding_utils.h | 56 +++++++++++ .../stable_diffusion_invoker.cc | 18 +++- .../stable_diffusion_pipeline.cc | 12 +++ .../stable_diffusion_pipeline.h | 2 +- 6 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc create mode 100644 mobile_back_tflite/cpp/backend_tflite/embedding_utils.h diff --git a/mobile_back_tflite/cpp/backend_tflite/BUILD b/mobile_back_tflite/cpp/backend_tflite/BUILD index f4d1a69da..562c33daa 100644 --- a/mobile_back_tflite/cpp/backend_tflite/BUILD +++ b/mobile_back_tflite/cpp/backend_tflite/BUILD @@ -39,6 +39,7 @@ pbtxt2header( cc_library( name = "tflite_c", srcs = [ + "embedding_utils.cc", "sd_utils.cc", "single_model_pipeline.cc", "stable_diffusion_invoker.cc", @@ -46,6 +47,7 @@ cc_library( "tflite_c.cc", ], hdrs = [ + "embedding_utils.h", "pipeline.h", "sd_utils.h", "single_model_pipeline.h", diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc new file mode 100644 index 000000000..c3865cad8 --- /dev/null +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc @@ -0,0 +1,93 @@ +#include "embedding_utils.h" + +#include +#include + + +bool TsEmbeddingParser::parse(std::ifstream& fin, size_t file_size) { + int32_t steps = 0; + size_t total_read = 0; + + if (!fin.read(reinterpret_cast(&steps), sizeof(int32_t))) { + return false; + } + + total_read += sizeof(int32_t); + ts_seq_map_[steps].resize(steps); + + if (!fin.read(reinterpret_cast(ts_seq_map_[steps].data()), + sizeof(int32_t) * steps)) { + return false; + } + + total_read += sizeof(int32_t) * steps; + ts_embedding_seq_map_[steps].resize(steps); + + for (int32_t idx = 0; idx < steps; idx++) { + ts_embedding_seq_map_[steps][idx].resize(TS_EMBEDDING_ELEMENT_COUNT); + + if (!fin.read(reinterpret_cast(ts_embedding_seq_map_[steps][idx].data()), + sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT)) { + return false; + } + + total_read += sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT; + } + + return total_read == file_size; +} + +std::vector TsEmbeddingParser::get_timestamp_embedding(int32_t steps, + int32_t step_index) const { + auto steps_it = ts_embedding_seq_map_.find(steps); + if (steps_it == ts_embedding_seq_map_.end() || + step_index >= steps_it->second.size()) { + return std::vector(); + } + return steps_it->second[step_index]; +} + +std::vector TsEmbeddingParser::get_timesteps(int32_t steps) const { + auto steps_it = ts_seq_map_.find(steps); + if (steps_it == ts_seq_map_.end()) { + return std::vector(); + } + return steps_it->second; +} + +bool EmbeddingManager::load_timestamp_embeddings(const std::string& filename) { + ts_parser_ = std::make_unique(); + + std::ifstream fin(filename, std::ios::binary); + if (!fin) { + std::cerr << "Failed to open file: " << filename << std::endl; + return false; + } + auto file_size = std::filesystem::file_size(filename); + bool success = ts_parser_->parse(fin, file_size); + if (!success) { + std::cerr << "Failed to parse timestamp embeddings from " << filename << std::endl; + // LOG(ERROR) << "Failed to parse embedding file: " << filename; + ts_parser_.reset(); + } + return success; +} + +std::vector EmbeddingManager::get_timestamp_embedding(int32_t timestep, + int num_steps) const { + if (!ts_parser_) { + std::cerr << "No timestamp parser initialized" << std::endl; + // LOG(ERROR) << "No timestamp parser initialized"; + return std::vector(); + } + return ts_parser_->get_timestamp_embedding(num_steps, timestep); +} + +std::vector EmbeddingManager::get_timesteps(int num_steps) const { + if (!ts_parser_) { + std::cerr << "No timestamp parser initialized" << std::endl; + // LOG(ERROR) << "No timestamp parser initialized"; + return std::vector(); + } + return ts_parser_->get_timesteps(num_steps); +} \ No newline at end of file diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h new file mode 100644 index 000000000..3ad93fa4d --- /dev/null +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h @@ -0,0 +1,56 @@ +// embedding_utils.h +#ifndef EMBEDDING_UTILS_H_ +#define EMBEDDING_UTILS_H_ + +#include +#include +#include +#include +#include + +using float32_t = float; +using tensor_data_float32_t = std::vector; + +#define TS_EMBEDDING_ELEMENT_COUNT (1*1280) + +class FileParser { +public: + virtual bool parse(std::ifstream& fin, size_t file_size) = 0; + virtual ~FileParser() = default; +}; + +class TsEmbeddingParser : public FileParser { +public: + TsEmbeddingParser() = default; + ~TsEmbeddingParser() override = default; + + bool parse(std::ifstream& fin, size_t file_size) override; + + // Retrieves the timestamp embedding for a specific step + std::vector get_timestamp_embedding(int32_t steps, int32_t step_index) const; + + // Get all timesteps for a specific number of steps + std::vector get_timesteps(int32_t steps) const; + +private: + std::map> ts_seq_map_; + std::map> ts_embedding_seq_map_; +}; + +class EmbeddingManager { +public: + static EmbeddingManager& getInstance() { + static EmbeddingManager instance; + return instance; + } + + bool load_timestamp_embeddings(const std::string& filename); + std::vector get_timestamp_embedding(int32_t timestep, int num_steps) const; + std::vector get_timesteps(int num_steps) const; + +private: + EmbeddingManager() = default; + std::unique_ptr ts_parser_; +}; + +#endif // EMBEDDING_UTILS_H_ diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc index 94ac36ed2..3dbea6fe2 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc @@ -4,6 +4,7 @@ #include #include +#include "embedding_utils.h" #include "sd_utils.h" #include "stable_diffusion_pipeline.h" #include "tensorflow/lite/c/c_api.h" @@ -102,7 +103,15 @@ std::vector StableDiffusionInvoker::diffusion_process( auto noise = get_normal(64 * 64 * 4, seed); auto latent = noise; - auto timesteps = get_timesteps(1, 1000, 1000 / num_steps); + // Get pre-calculated timesteps and embeddings + auto& embedding_manager = EmbeddingManager::getInstance(); + auto timesteps = embedding_manager.get_timesteps(num_steps); + + if (timesteps.empty()) { + LOG(ERROR) << "Failed to get timesteps for " << num_steps << " steps"; + return std::vector(); + } + auto alphas_tuple = get_initial_alphas(timesteps); auto alphas = std::get<0>(alphas_tuple); auto alphas_prev = std::get<1>(alphas_tuple); @@ -111,7 +120,12 @@ std::vector StableDiffusionInvoker::diffusion_process( LOG(INFO) << "Step " << timesteps.size() - 1 - i; auto latent_prev = latent; - auto t_emb = get_timestep_embedding(timesteps[i]); + // Get pre-calculated timestamp embedding + auto t_emb = embedding_manager.get_timestamp_embedding(i, num_steps); + if (t_emb.empty()) { + LOG(ERROR) << "Failed to get timestamp embedding for step " << i; + return std::vector(); + } auto unconditional_latent = diffusion_step(latent, t_emb, unconditional_encoded_text); diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc index 358f031c9..5c9e25bef 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc @@ -6,6 +6,7 @@ #include #include +#include "embedding_utils.h" #include "flutter/cpp/c/backend_c.h" #include "stable_diffusion_invoker.h" #include "tensorflow/lite/c/c_api.h" @@ -104,6 +105,17 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( return nullptr; } + std::string ts_embedding_path = + std::string(model_path) + + "/timestep_steps_20_int32_embedding_1x1280_float32.bin.ts"; + if (!EmbeddingManager::getInstance().load_timestamp_embeddings( + ts_embedding_path)) { + LOG(ERROR) << "Failed to load timestamp embeddings from " + << ts_embedding_path; + backend_delete(backend_data); + return nullptr; + } + return backend_data; } diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h index 7104cb24e..9ad01b61b 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h @@ -37,7 +37,7 @@ struct SDBackendData { std::vector input_prompt_tokens; std::vector unconditional_tokens; - int num_steps{10}; + int num_steps{20}; int seed{0}; std::vector output; From 6d9bc91aa64731b3a7c389ff774df9839e12eb4d Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Mon, 28 Oct 2024 22:15:13 +0200 Subject: [PATCH 04/27] disabled bitcode to be able compile with new XCode --- .bazelrc | 1 - 1 file changed, 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 5788b7d3b..a2ee67b8c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -43,7 +43,6 @@ build:android_x86_64 --fat_apk_cpu=x86_64 # iOS configs build:ios --apple_platform_type=ios -build:ios --apple_bitcode=embedded --copt=-fembed-bitcode build:ios --copt=-Wno-c++11-narrowing build:ios --cxxopt=-fobjc-arc From fadee084916ee17f4409c7208b3d635c348d40bc Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Mon, 28 Oct 2024 22:33:37 +0200 Subject: [PATCH 05/27] chore: formatting --- .../cpp/backend_tflite/embedding_utils.cc | 137 +++++++++--------- .../cpp/backend_tflite/embedding_utils.h | 66 +++++---- 2 files changed, 103 insertions(+), 100 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc index c3865cad8..20767865e 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc @@ -3,91 +3,92 @@ #include #include - bool TsEmbeddingParser::parse(std::ifstream& fin, size_t file_size) { - int32_t steps = 0; - size_t total_read = 0; - - if (!fin.read(reinterpret_cast(&steps), sizeof(int32_t))) { - return false; - } + int32_t steps = 0; + size_t total_read = 0; - total_read += sizeof(int32_t); - ts_seq_map_[steps].resize(steps); - - if (!fin.read(reinterpret_cast(ts_seq_map_[steps].data()), - sizeof(int32_t) * steps)) { - return false; - } + if (!fin.read(reinterpret_cast(&steps), sizeof(int32_t))) { + return false; + } + + total_read += sizeof(int32_t); + ts_seq_map_[steps].resize(steps); + + if (!fin.read(reinterpret_cast(ts_seq_map_[steps].data()), + sizeof(int32_t) * steps)) { + return false; + } + + total_read += sizeof(int32_t) * steps; + ts_embedding_seq_map_[steps].resize(steps); - total_read += sizeof(int32_t) * steps; - ts_embedding_seq_map_[steps].resize(steps); - - for (int32_t idx = 0; idx < steps; idx++) { - ts_embedding_seq_map_[steps][idx].resize(TS_EMBEDDING_ELEMENT_COUNT); - - if (!fin.read(reinterpret_cast(ts_embedding_seq_map_[steps][idx].data()), - sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT)) { - return false; - } - - total_read += sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT; + for (int32_t idx = 0; idx < steps; idx++) { + ts_embedding_seq_map_[steps][idx].resize(TS_EMBEDDING_ELEMENT_COUNT); + + if (!fin.read( + reinterpret_cast(ts_embedding_seq_map_[steps][idx].data()), + sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT)) { + return false; } - return total_read == file_size; + total_read += sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT; + } + + return total_read == file_size; } -std::vector TsEmbeddingParser::get_timestamp_embedding(int32_t steps, - int32_t step_index) const { - auto steps_it = ts_embedding_seq_map_.find(steps); - if (steps_it == ts_embedding_seq_map_.end() || - step_index >= steps_it->second.size()) { - return std::vector(); - } - return steps_it->second[step_index]; +std::vector TsEmbeddingParser::get_timestamp_embedding( + int32_t steps, int32_t step_index) const { + auto steps_it = ts_embedding_seq_map_.find(steps); + if (steps_it == ts_embedding_seq_map_.end() || + step_index >= steps_it->second.size()) { + return std::vector(); + } + return steps_it->second[step_index]; } std::vector TsEmbeddingParser::get_timesteps(int32_t steps) const { - auto steps_it = ts_seq_map_.find(steps); - if (steps_it == ts_seq_map_.end()) { - return std::vector(); - } - return steps_it->second; + auto steps_it = ts_seq_map_.find(steps); + if (steps_it == ts_seq_map_.end()) { + return std::vector(); + } + return steps_it->second; } bool EmbeddingManager::load_timestamp_embeddings(const std::string& filename) { - ts_parser_ = std::make_unique(); + ts_parser_ = std::make_unique(); - std::ifstream fin(filename, std::ios::binary); - if (!fin) { - std::cerr << "Failed to open file: " << filename << std::endl; - return false; - } - auto file_size = std::filesystem::file_size(filename); - bool success = ts_parser_->parse(fin, file_size); - if (!success) { - std::cerr << "Failed to parse timestamp embeddings from " << filename << std::endl; - // LOG(ERROR) << "Failed to parse embedding file: " << filename; - ts_parser_.reset(); - } - return success; + std::ifstream fin(filename, std::ios::binary); + if (!fin) { + std::cerr << "Failed to open file: " << filename << std::endl; + return false; + } + auto file_size = std::filesystem::file_size(filename); + bool success = ts_parser_->parse(fin, file_size); + if (!success) { + std::cerr << "Failed to parse timestamp embeddings from " << filename + << std::endl; + // LOG(ERROR) << "Failed to parse embedding file: " << filename; + ts_parser_.reset(); + } + return success; } -std::vector EmbeddingManager::get_timestamp_embedding(int32_t timestep, - int num_steps) const { - if (!ts_parser_) { - std::cerr << "No timestamp parser initialized" << std::endl; - // LOG(ERROR) << "No timestamp parser initialized"; - return std::vector(); - } - return ts_parser_->get_timestamp_embedding(num_steps, timestep); +std::vector EmbeddingManager::get_timestamp_embedding( + int32_t timestep, int num_steps) const { + if (!ts_parser_) { + std::cerr << "No timestamp parser initialized" << std::endl; + // LOG(ERROR) << "No timestamp parser initialized"; + return std::vector(); + } + return ts_parser_->get_timestamp_embedding(num_steps, timestep); } std::vector EmbeddingManager::get_timesteps(int num_steps) const { - if (!ts_parser_) { - std::cerr << "No timestamp parser initialized" << std::endl; - // LOG(ERROR) << "No timestamp parser initialized"; - return std::vector(); - } - return ts_parser_->get_timesteps(num_steps); + if (!ts_parser_) { + std::cerr << "No timestamp parser initialized" << std::endl; + // LOG(ERROR) << "No timestamp parser initialized"; + return std::vector(); + } + return ts_parser_->get_timesteps(num_steps); } \ No newline at end of file diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h index 3ad93fa4d..5408ef773 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h @@ -11,46 +11,48 @@ using float32_t = float; using tensor_data_float32_t = std::vector; -#define TS_EMBEDDING_ELEMENT_COUNT (1*1280) +#define TS_EMBEDDING_ELEMENT_COUNT (1 * 1280) class FileParser { -public: - virtual bool parse(std::ifstream& fin, size_t file_size) = 0; - virtual ~FileParser() = default; + public: + virtual bool parse(std::ifstream& fin, size_t file_size) = 0; + virtual ~FileParser() = default; }; class TsEmbeddingParser : public FileParser { -public: - TsEmbeddingParser() = default; - ~TsEmbeddingParser() override = default; - - bool parse(std::ifstream& fin, size_t file_size) override; - - // Retrieves the timestamp embedding for a specific step - std::vector get_timestamp_embedding(int32_t steps, int32_t step_index) const; - - // Get all timesteps for a specific number of steps - std::vector get_timesteps(int32_t steps) const; - -private: - std::map> ts_seq_map_; - std::map> ts_embedding_seq_map_; + public: + TsEmbeddingParser() = default; + ~TsEmbeddingParser() override = default; + + bool parse(std::ifstream& fin, size_t file_size) override; + + // Retrieves the timestamp embedding for a specific step + std::vector get_timestamp_embedding(int32_t steps, + int32_t step_index) const; + + // Get all timesteps for a specific number of steps + std::vector get_timesteps(int32_t steps) const; + + private: + std::map> ts_seq_map_; + std::map> ts_embedding_seq_map_; }; class EmbeddingManager { -public: - static EmbeddingManager& getInstance() { - static EmbeddingManager instance; - return instance; - } - - bool load_timestamp_embeddings(const std::string& filename); - std::vector get_timestamp_embedding(int32_t timestep, int num_steps) const; - std::vector get_timesteps(int num_steps) const; - -private: - EmbeddingManager() = default; - std::unique_ptr ts_parser_; + public: + static EmbeddingManager& getInstance() { + static EmbeddingManager instance; + return instance; + } + + bool load_timestamp_embeddings(const std::string& filename); + std::vector get_timestamp_embedding(int32_t timestep, + int num_steps) const; + std::vector get_timesteps(int num_steps) const; + + private: + EmbeddingManager() = default; + std::unique_ptr ts_parser_; }; #endif // EMBEDDING_UTILS_H_ From 1bddf376bea6cec3390a702f2c1f7458d633a9cd Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 12 Nov 2024 12:05:46 +0700 Subject: [PATCH 06/27] refactor: use custom setting in Core ML backend to detect NCHW input. (#924) * Add GetConfigValue() * Add custom setting data-format for Core ML * Use GetConfigValue() to get stable_diffusion_seed and stable_diffusion_num_steps --- flutter/cpp/utils.cc | 40 +++++++++++++++++++ flutter/cpp/utils.h | 4 ++ .../cpp/backend_coreml/coreml_settings.pbtxt | 34 +++++++++++++--- mobile_back_apple/cpp/backend_coreml/main.cc | 5 ++- mobile_back_apple/dev-utils/Makefile | 4 +- mobile_back_tflite/cpp/backend_tflite/BUILD | 1 + .../cpp/backend_tflite/neuron/BUILD | 1 + .../stable_diffusion_pipeline.cc | 22 ++++++---- 8 files changed, 94 insertions(+), 17 deletions(-) diff --git a/flutter/cpp/utils.cc b/flutter/cpp/utils.cc index 7cea64c91..126e9edf5 100644 --- a/flutter/cpp/utils.cc +++ b/flutter/cpp/utils.cc @@ -186,5 +186,45 @@ SettingList CreateSettingList(const BackendSetting &backend_setting, return setting_list; } +template +T GetConfigValue(mlperf_backend_configuration_t *configs, const char *key, + T defaultValue); + +template <> +int GetConfigValue(mlperf_backend_configuration_t *configs, + const char *key, int defaultValue) { + for (int i = 0; i < configs->count; ++i) { + if (strcmp(configs->keys[i], key) == 0) { + const char *valueStr = configs->values[i]; + char *endptr = nullptr; + errno = 0; + long value = + strtol(valueStr, &endptr, 10); // Base 10 for decimal conversion + if (errno == ERANGE || value < INT_MIN || value > INT_MAX) { + LOG(ERROR) << "Value out of range for int: " << valueStr; + return defaultValue; + } + if (endptr == valueStr || *endptr != '\0') { + LOG(ERROR) << "Invalid value for int: " << valueStr; + return defaultValue; + } + return static_cast(value); + } + } + return defaultValue; +} + +template <> +std::string GetConfigValue(mlperf_backend_configuration_t *configs, + const char *key, + std::string defaultValue) { + for (int i = 0; i < configs->count; ++i) { + if (strcmp(configs->keys[i], key) == 0) { + return std::string(configs->values[i]); + } + } + return defaultValue; +} + } // namespace mobile } // namespace mlperf diff --git a/flutter/cpp/utils.h b/flutter/cpp/utils.h index 17d590024..c66022401 100644 --- a/flutter/cpp/utils.h +++ b/flutter/cpp/utils.h @@ -69,6 +69,10 @@ SettingList CreateSettingList(const BackendSetting &backend_setting, const std::string &custom_config, const std::string &benchmark_id); +template +T GetConfigValue(mlperf_backend_configuration_t *configs, const char *key, + T defaultValue); + } // namespace mobile } // namespace mlperf diff --git a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt index 496d9c439..a607f61c2 100644 --- a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt +++ b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt @@ -22,7 +22,7 @@ benchmark_setting { model_checksum: "39483b20b878d46144ab4cfe9a3e5600" } } - delegate_choice: { + delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" @@ -57,7 +57,7 @@ benchmark_setting { model_checksum: "39483b20b878d46144ab4cfe9a3e5600" } } - delegate_choice: { + delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" @@ -81,6 +81,10 @@ benchmark_setting { model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip" model_checksum: "164c504eb3e9af6c730c1765b8b81b32" } + custom_setting { + id: "data-format" + value: "NCHW" + } } delegate_choice: { delegate_name: "CPU & GPU" @@ -90,6 +94,10 @@ benchmark_setting { model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip" model_checksum: "164c504eb3e9af6c730c1765b8b81b32" } + custom_setting { + id: "data-format" + value: "NCHW" + } } delegate_choice: { delegate_name: "CPU & ANE" @@ -99,6 +107,10 @@ benchmark_setting { model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip" model_checksum: "164c504eb3e9af6c730c1765b8b81b32" } + custom_setting { + id: "data-format" + value: "NCHW" + } } delegate_selected: "CPU & GPU & ANE" } @@ -115,6 +127,10 @@ benchmark_setting { model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip" model_checksum: "164c504eb3e9af6c730c1765b8b81b32" } + custom_setting { + id: "data-format" + value: "NCHW" + } } delegate_choice: { delegate_name: "CPU & GPU" @@ -125,6 +141,10 @@ benchmark_setting { model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip" model_checksum: "164c504eb3e9af6c730c1765b8b81b32" } + custom_setting { + id: "data-format" + value: "NCHW" + } } delegate_choice: { delegate_name: "CPU & ANE" @@ -135,6 +155,10 @@ benchmark_setting { model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip" model_checksum: "164c504eb3e9af6c730c1765b8b81b32" } + custom_setting { + id: "data-format" + value: "NCHW" + } } delegate_selected: "CPU & GPU & ANE" } @@ -160,7 +184,7 @@ benchmark_setting { model_checksum: "ef849fbf2132e205158f05ca42db25f4" } } - delegate_choice: { + delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" @@ -217,7 +241,7 @@ benchmark_setting { model_checksum: "362d6b5bb1b8e10ae5b4e223f60d4d10" } } - delegate_choice: { + delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" @@ -250,7 +274,7 @@ benchmark_setting { model_checksum: "62489706f20b0c2ae561fb2204eefb61" } } - delegate_choice: { + delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" diff --git a/mobile_back_apple/cpp/backend_coreml/main.cc b/mobile_back_apple/cpp/backend_coreml/main.cc index a2246345f..af753d566 100644 --- a/mobile_back_apple/cpp/backend_coreml/main.cc +++ b/mobile_back_apple/cpp/backend_coreml/main.cc @@ -101,8 +101,9 @@ mlperf_backend_ptr_t mlperf_backend_create( CoreMLBackendData *backend_data = new CoreMLBackendData(); backendExists = true; - // quick hack for checking if model expects NCHW input. - if (strcasestr(model_path, "NCHW") != nullptr) { + std::string dataFormat = + mlperf::mobile::GetConfigValue(configs, "data-format", std::string("")); + if (dataFormat == "NCHW") { backend_data->expectNCHW = true; LOG(INFO) << "Will convert inputs from NHWC to NCHW!"; } diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 2ef789f10..a9904fe38 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -24,7 +24,7 @@ app: tflite-build: cd ${REPO_ROOT_DIR} && \ - bazel build -c opt --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=12.1 \ + bazel build -c opt --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=13.1 \ //flutter/cpp/binary:main //mobile_back_tflite:tflitebackend tflite-run-ic: @@ -141,7 +141,7 @@ tflite-run-sd: coreml-build: cd ${REPO_ROOT_DIR} && \ - bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=12.1 \ + bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=13.1 \ //flutter/cpp/binary:main //mobile_back_apple:coremlbackend coreml-run-ic: diff --git a/mobile_back_tflite/cpp/backend_tflite/BUILD b/mobile_back_tflite/cpp/backend_tflite/BUILD index f4d1a69da..80f46059a 100644 --- a/mobile_back_tflite/cpp/backend_tflite/BUILD +++ b/mobile_back_tflite/cpp/backend_tflite/BUILD @@ -68,6 +68,7 @@ cc_library( }), deps = [ ":tflite_settings", + "//flutter/cpp:utils", "//flutter/cpp/c:headers", "@org_tensorflow//tensorflow/core:tflite_portable_logging", "@org_tensorflow//tensorflow/lite/c:c_api", diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD b/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD index 118076fea..3bd677e6a 100644 --- a/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD +++ b/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD @@ -70,6 +70,7 @@ cc_library( local_defines = ["MTK_TFLITE_NEURON_BACKEND"], deps = [ ":tflite_settings", + "//flutter/cpp:utils", "//flutter/cpp/c:headers", "//mobile_back_tflite/cpp/backend_tflite:tflite_settings", "@org_tensorflow//tensorflow/core:tflite_portable_logging", diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc index 358f031c9..adec6b695 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc @@ -7,6 +7,7 @@ #include #include "flutter/cpp/c/backend_c.h" +#include "flutter/cpp/utils.h" #include "stable_diffusion_invoker.h" #include "tensorflow/lite/c/c_api.h" #include "tensorflow/lite/c/common.h" @@ -58,21 +59,26 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( // Verify only one instance of the backend exists at any time if (backendExists) { + LOG(ERROR) << "Backend already exists"; return nullptr; } SDBackendData* backend_data = new SDBackendData(); backendExists = true; - for (int i = 0; i < configs->count; ++i) { - if (strcmp(configs->keys[i], "stable_diffusion_seed") == 0) { - backend_data->seed = atoi(configs->values[i]); - } - if (strcmp(configs->keys[i], "stable_diffusion_num_steps") == 0) { - backend_data->num_steps = atoi(configs->values[i]); - } + // Read seed and num_steps value from SD task settings + backend_data->seed = + mlperf::mobile::GetConfigValue(configs, "stable_diffusion_seed", 0); + if (backend_data->seed == 0) { + LOG(ERROR) << "Cannot get stable_diffusion_seed"; + return nullptr; + } + backend_data->num_steps = + mlperf::mobile::GetConfigValue(configs, "stable_diffusion_num_steps", 0); + if (backend_data->num_steps == 0) { + LOG(ERROR) << "Cannot get stable_diffusion_num_steps"; + return nullptr; } - // Load models from the provided directory path std::string text_encoder_path = std::string(model_path) + "/sd_text_encoder_dynamic.tflite"; From 463d9746b4ce37a068210aa470fd7b07e5eca6ac Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 12 Nov 2024 13:32:48 +0700 Subject: [PATCH 07/27] fix: resolve crash due to permission denied on Android Play Store version (#930) * Set android:extractNativeLibs="true" * Set android.bundle.enableUncompressedNativeLibs=false --- flutter/android/app/src/main/AndroidManifest.xml | 3 ++- flutter/android/gradle.properties | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/flutter/android/app/src/main/AndroidManifest.xml b/flutter/android/app/src/main/AndroidManifest.xml index 79cfd7873..67a178c84 100644 --- a/flutter/android/app/src/main/AndroidManifest.xml +++ b/flutter/android/app/src/main/AndroidManifest.xml @@ -11,7 +11,8 @@ android:maxSdkVersion="29"/> + android:icon="@mipmap/ic_launcher" + android:extractNativeLibs="true"> Date: Tue, 19 Nov 2024 13:58:12 +0700 Subject: [PATCH 08/27] chore: increase Android minSdkVersion from 21 to 30 (#859) Increase minSdkVersion to 30 --- flutter/android/app/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter/android/app/build.gradle b/flutter/android/app/build.gradle index a0dd55f62..92d02ddd3 100644 --- a/flutter/android/app/build.gradle +++ b/flutter/android/app/build.gradle @@ -42,7 +42,7 @@ android { defaultConfig { applicationId "org.mlcommons.android.mlperfbench" - minSdkVersion 21 + minSdkVersion 30 targetSdkVersion 34 versionCode flutterVersionCode.toInteger() versionName flutterVersionName From 1008c4949def415402097b767741eebe1795d3c1 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Mon, 2 Dec 2024 22:40:08 +0200 Subject: [PATCH 09/27] feat: finalized SD pipeline to use embedding from the binary file. --- .../cpp/backend_tflite/embedding_utils.cc | 111 ++++++++++++++++-- .../cpp/backend_tflite/embedding_utils.h | 48 ++++---- .../cpp/backend_tflite/sd_utils.cc | 1 - .../stable_diffusion_invoker.cc | 16 ++- .../backend_tflite/stable_diffusion_invoker.h | 2 +- .../stable_diffusion_pipeline.cc | 10 +- .../stable_diffusion_pipeline.h | 1 + 7 files changed, 144 insertions(+), 45 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc index 20767865e..6a52a82fe 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc @@ -10,7 +10,6 @@ bool TsEmbeddingParser::parse(std::ifstream& fin, size_t file_size) { if (!fin.read(reinterpret_cast(&steps), sizeof(int32_t))) { return false; } - total_read += sizeof(int32_t); ts_seq_map_[steps].resize(steps); @@ -34,10 +33,101 @@ bool TsEmbeddingParser::parse(std::ifstream& fin, size_t file_size) { total_read += sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT; } - return total_read == file_size; + bool size_check = total_read == file_size; + // if (size_check) { + // std::string json_filename = "timestep_embeddings_debug.json"; + // if (!saveToJson(json_filename)) { + // std::cerr << "Warning: Failed to save embeddings to JSON" << std::endl; + // } + // } + + return size_check; +} + +std::string TsEmbeddingParser::vectorToJsonArray(const std::vector& vec, + bool full_output) { + std::ostringstream ss; + ss << "["; + size_t count = full_output ? vec.size() : std::min(size_t(10), vec.size()); + for (size_t i = 0; i < count; ++i) { + if (i > 0) ss << ", "; + ss << std::fixed << std::setprecision(6) << vec[i]; + } + if (!full_output && vec.size() > 10) ss << ", ..."; + ss << "]"; + return ss.str(); +} + +std::string TsEmbeddingParser::vectorToJsonArray( + const std::vector& vec) { + std::ostringstream ss; // Changed to ostringstream for consistency + ss << "["; + for (size_t i = 0; i < vec.size(); ++i) { + if (i > 0) ss << ", "; + ss << vec[i]; + } + ss << "]"; + return ss.str(); +} + +bool TsEmbeddingParser::saveToJson(const std::string& filename) const { + std::ofstream out(filename); + if (!out.is_open()) { + std::cerr << "Failed to open output file: " << filename << std::endl; + return false; + } + + out << "{\n \"timestep_sequences\": {\n"; + + // Save timestep sequences + size_t seq_count = 0; + for (const auto& [steps, sequence] : ts_seq_map_) { + if (seq_count++ > 0) out << ",\n"; + out << " \"" << steps << "\": " << vectorToJsonArray(sequence); + } + + out << "\n },\n \"embeddings\": {\n"; + + // Save embeddings + size_t steps_count = 0; + for (const auto& [steps, step_embeddings] : ts_embedding_seq_map_) { + if (steps_count++ > 0) out << ",\n"; + out << " \"" << steps << "\": [\n"; + + for (size_t i = 0; i < step_embeddings.size(); ++i) { + const auto& embedding = step_embeddings[i]; + + auto [min_it, max_it] = + std::minmax_element(embedding.begin(), embedding.end()); + float sum = std::accumulate(embedding.begin(), embedding.end(), 0.0f); + float avg = sum / embedding.size(); + + if (i > 0) out << ",\n"; + out << " {\n" + << " \"step\": " << i << ",\n" + << " \"embedding\": " << vectorToJsonArray(embedding, true) + << ",\n" // Full output + << " \"embedding_size\": " << embedding.size() << ",\n" + << " \"min\": " << *min_it << ",\n" + << " \"max\": " << *max_it << ",\n" + << " \"avg\": " << avg << "\n" + << " }"; + } + out << "\n ]"; + } + + out << "\n }\n}"; + + if (out.fail()) { + std::cerr << "Error while writing to file: " << filename << std::endl; + return false; + } + + std::cout << "Successfully saved embeddings to " << filename << std::endl; + return true; } -std::vector TsEmbeddingParser::get_timestamp_embedding( +std::vector TsEmbeddingParser::get_timestep_embedding( int32_t steps, int32_t step_index) const { auto steps_it = ts_embedding_seq_map_.find(steps); if (steps_it == ts_embedding_seq_map_.end() || @@ -55,7 +145,7 @@ std::vector TsEmbeddingParser::get_timesteps(int32_t steps) const { return steps_it->second; } -bool EmbeddingManager::load_timestamp_embeddings(const std::string& filename) { +bool EmbeddingManager::load_timestep_embeddings(const std::string& filename) { ts_parser_ = std::make_unique(); std::ifstream fin(filename, std::ios::binary); @@ -66,28 +156,25 @@ bool EmbeddingManager::load_timestamp_embeddings(const std::string& filename) { auto file_size = std::filesystem::file_size(filename); bool success = ts_parser_->parse(fin, file_size); if (!success) { - std::cerr << "Failed to parse timestamp embeddings from " << filename + std::cerr << "Failed to parse timestep embeddings from " << filename << std::endl; - // LOG(ERROR) << "Failed to parse embedding file: " << filename; ts_parser_.reset(); } return success; } -std::vector EmbeddingManager::get_timestamp_embedding( +std::vector EmbeddingManager::get_timestep_embedding( int32_t timestep, int num_steps) const { if (!ts_parser_) { - std::cerr << "No timestamp parser initialized" << std::endl; - // LOG(ERROR) << "No timestamp parser initialized"; + std::cerr << "No timestep parser initialized" << std::endl; return std::vector(); } - return ts_parser_->get_timestamp_embedding(num_steps, timestep); + return ts_parser_->get_timestep_embedding(num_steps, timestep); } std::vector EmbeddingManager::get_timesteps(int num_steps) const { if (!ts_parser_) { - std::cerr << "No timestamp parser initialized" << std::endl; - // LOG(ERROR) << "No timestamp parser initialized"; + std::cerr << "No timestep parser initialized" << std::endl; return std::vector(); } return ts_parser_->get_timesteps(num_steps); diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h index 5408ef773..7b915aa8a 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h @@ -1,41 +1,37 @@ -// embedding_utils.h #ifndef EMBEDDING_UTILS_H_ #define EMBEDDING_UTILS_H_ -#include +#include +#include #include +#include +#include #include +#include +#include +#include #include #include using float32_t = float; using tensor_data_float32_t = std::vector; -#define TS_EMBEDDING_ELEMENT_COUNT (1 * 1280) - -class FileParser { - public: - virtual bool parse(std::ifstream& fin, size_t file_size) = 0; - virtual ~FileParser() = default; -}; - -class TsEmbeddingParser : public FileParser { +class TsEmbeddingParser { public: - TsEmbeddingParser() = default; - ~TsEmbeddingParser() override = default; - - bool parse(std::ifstream& fin, size_t file_size) override; - - // Retrieves the timestamp embedding for a specific step - std::vector get_timestamp_embedding(int32_t steps, - int32_t step_index) const; - - // Get all timesteps for a specific number of steps + bool parse(std::ifstream& fin, size_t file_size); + std::vector get_timestep_embedding(int32_t steps, + int32_t step_index) const; std::vector get_timesteps(int32_t steps) const; + bool saveToJson(const std::string& filename) const; private: + static std::string vectorToJsonArray(const std::vector& vec, + bool full_output = false); + static std::string vectorToJsonArray(const std::vector& vec); + + static constexpr size_t TS_EMBEDDING_ELEMENT_COUNT = 1280; std::map> ts_seq_map_; - std::map> ts_embedding_seq_map_; + std::map>> ts_embedding_seq_map_; }; class EmbeddingManager { @@ -45,9 +41,9 @@ class EmbeddingManager { return instance; } - bool load_timestamp_embeddings(const std::string& filename); - std::vector get_timestamp_embedding(int32_t timestep, - int num_steps) const; + bool load_timestep_embeddings(const std::string& filename); + std::vector get_timestep_embedding(int32_t timestep, + int num_steps) const; std::vector get_timesteps(int num_steps) const; private: @@ -55,4 +51,4 @@ class EmbeddingManager { std::unique_ptr ts_parser_; }; -#endif // EMBEDDING_UTILS_H_ +#endif // EMBEDDING_UTILS_H_ \ No newline at end of file diff --git a/mobile_back_tflite/cpp/backend_tflite/sd_utils.cc b/mobile_back_tflite/cpp/backend_tflite/sd_utils.cc index 14aa858d9..c5901b66c 100644 --- a/mobile_back_tflite/cpp/backend_tflite/sd_utils.cc +++ b/mobile_back_tflite/cpp/backend_tflite/sd_utils.cc @@ -233,7 +233,6 @@ std::vector get_timestep_embedding(int timestep, int batch_size, int dim, embedding_cos.push_back(cosf(timestep * freq)); embedding_sin.push_back(sinf(timestep * freq)); } - std::vector embedding; for (int i = 0; i < batch_size; i++) { embedding.insert(embedding.end(), diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc index 3dbea6fe2..8c3405739 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc @@ -100,6 +100,7 @@ std::vector StableDiffusionInvoker::diffusion_process( const std::vector& unconditional_encoded_text, int num_steps, int seed) { float unconditional_guidance_scale = 7.5f; + auto noise = get_normal(64 * 64 * 4, seed); auto latent = noise; @@ -113,15 +114,25 @@ std::vector StableDiffusionInvoker::diffusion_process( } auto alphas_tuple = get_initial_alphas(timesteps); + auto alphas = std::get<0>(alphas_tuple); auto alphas_prev = std::get<1>(alphas_tuple); for (int i = timesteps.size() - 1; i >= 0; --i) { LOG(INFO) << "Step " << timesteps.size() - 1 - i; + std::cout << "\n=== Processing Step " << timesteps.size() - 1 - i + << " (timestamp: " << timesteps[i] << ") ===" << std::endl; + auto latent_prev = latent; - // Get pre-calculated timestamp embedding - auto t_emb = embedding_manager.get_timestamp_embedding(i, num_steps); + + auto t_emb = embedding_manager.get_timestep_embedding(i, num_steps); + + if (t_emb.empty()) { + LOG(ERROR) << "Failed to get timestamp embedding for step " << i; + return std::vector(); + } + if (t_emb.empty()) { LOG(ERROR) << "Failed to get timestamp embedding for step " << i; return std::vector(); @@ -146,6 +157,7 @@ std::vector StableDiffusionInvoker::diffusion_process( latent.assign(std::begin(l), std::end(l)); } + std::cout << "\nDiffusion process completed" << std::endl; return latent; } diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.h b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.h index ccbef1f9e..706589835 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.h +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.h @@ -5,7 +5,7 @@ #include #include -#include "stable_diffusion_pipeline.h" // Include the backend data structure +#include "stable_diffusion_pipeline.h" #include "tensorflow/lite/interpreter.h" #include "tensorflow/lite/model_builder.h" diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc index 5c9e25bef..b1ecae999 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc @@ -65,6 +65,8 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( SDBackendData* backend_data = new SDBackendData(); backendExists = true; + backend_data->model_dir = model_path; + for (int i = 0; i < configs->count; ++i) { if (strcmp(configs->keys[i], "stable_diffusion_seed") == 0) { backend_data->seed = atoi(configs->values[i]); @@ -77,8 +79,10 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( // Load models from the provided directory path std::string text_encoder_path = std::string(model_path) + "/sd_text_encoder_dynamic.tflite"; + std::string sd_model_path = - std::string(model_path) + "/sd_diffusion_model_dynamic.tflite"; + std::string(model_path) + + "/sd_diffusion_full_model_dynamic_int8_mlperfv41.tflite"; std::string decoder_path = std::string(model_path) + "/sd_decoder_dynamic.tflite"; @@ -108,9 +112,9 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( std::string ts_embedding_path = std::string(model_path) + "/timestep_steps_20_int32_embedding_1x1280_float32.bin.ts"; - if (!EmbeddingManager::getInstance().load_timestamp_embeddings( + if (!EmbeddingManager::getInstance().load_timestep_embeddings( ts_embedding_path)) { - LOG(ERROR) << "Failed to load timestamp embeddings from " + LOG(ERROR) << "Failed to load timestep embeddings from " << ts_embedding_path; backend_delete(backend_data); return nullptr; diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h index 9ad01b61b..09dd8b617 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h @@ -29,6 +29,7 @@ struct SDBackendData { TfLiteModel *text_encoder_model{nullptr}; TfLiteModel *sd_model{nullptr}; TfLiteModel *decoder_model{nullptr}; + std::string model_dir; TfLiteInterpreter *text_encoder_interpreter{nullptr}; TfLiteInterpreter *sd_interpreter{nullptr}; From 8527f368b05ef6911c8906b4ab5170aa76bc5ac5 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 3 Dec 2024 11:50:44 +0200 Subject: [PATCH 10/27] refactor: updated embedding_utils to parse pkl file --- .fvm/flutter_sdk | 1 + .fvm/fvm_config.json | 3 + .fvm/release | 1 + .fvm/version | 0 .fvm/versions/stable | 1 + .../cpp/backend_tflite/embedding_utils.cc | 182 +++--------------- .../cpp/backend_tflite/embedding_utils.h | 28 +-- 7 files changed, 43 insertions(+), 173 deletions(-) create mode 120000 .fvm/flutter_sdk create mode 100644 .fvm/fvm_config.json create mode 100644 .fvm/release create mode 100644 .fvm/version create mode 120000 .fvm/versions/stable diff --git a/.fvm/flutter_sdk b/.fvm/flutter_sdk new file mode 120000 index 000000000..380767fb9 --- /dev/null +++ b/.fvm/flutter_sdk @@ -0,0 +1 @@ +/Users/sergijrylskyj/fvm/versions/stable \ No newline at end of file diff --git a/.fvm/fvm_config.json b/.fvm/fvm_config.json new file mode 100644 index 000000000..1c922f80f --- /dev/null +++ b/.fvm/fvm_config.json @@ -0,0 +1,3 @@ +{ + "flutterSdkVersion": "stable" +} \ No newline at end of file diff --git a/.fvm/release b/.fvm/release new file mode 100644 index 000000000..870bbe4e5 --- /dev/null +++ b/.fvm/release @@ -0,0 +1 @@ +stable \ No newline at end of file diff --git a/.fvm/version b/.fvm/version new file mode 100644 index 000000000..e69de29bb diff --git a/.fvm/versions/stable b/.fvm/versions/stable new file mode 120000 index 000000000..380767fb9 --- /dev/null +++ b/.fvm/versions/stable @@ -0,0 +1 @@ +/Users/sergijrylskyj/fvm/versions/stable \ No newline at end of file diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc index 6a52a82fe..15f998880 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc @@ -1,181 +1,61 @@ #include "embedding_utils.h" - -#include #include -bool TsEmbeddingParser::parse(std::ifstream& fin, size_t file_size) { - int32_t steps = 0; - size_t total_read = 0; - - if (!fin.read(reinterpret_cast(&steps), sizeof(int32_t))) { - return false; - } - total_read += sizeof(int32_t); - ts_seq_map_[steps].resize(steps); - - if (!fin.read(reinterpret_cast(ts_seq_map_[steps].data()), - sizeof(int32_t) * steps)) { - return false; - } - - total_read += sizeof(int32_t) * steps; - ts_embedding_seq_map_[steps].resize(steps); - - for (int32_t idx = 0; idx < steps; idx++) { - ts_embedding_seq_map_[steps][idx].resize(TS_EMBEDDING_ELEMENT_COUNT); - - if (!fin.read( - reinterpret_cast(ts_embedding_seq_map_[steps][idx].data()), - sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT)) { - return false; - } - - total_read += sizeof(float32_t) * TS_EMBEDDING_ELEMENT_COUNT; - } - - bool size_check = total_read == file_size; - // if (size_check) { - // std::string json_filename = "timestep_embeddings_debug.json"; - // if (!saveToJson(json_filename)) { - // std::cerr << "Warning: Failed to save embeddings to JSON" << std::endl; - // } - // } - - return size_check; -} - -std::string TsEmbeddingParser::vectorToJsonArray(const std::vector& vec, - bool full_output) { - std::ostringstream ss; - ss << "["; - size_t count = full_output ? vec.size() : std::min(size_t(10), vec.size()); - for (size_t i = 0; i < count; ++i) { - if (i > 0) ss << ", "; - ss << std::fixed << std::setprecision(6) << vec[i]; - } - if (!full_output && vec.size() > 10) ss << ", ..."; - ss << "]"; - return ss.str(); -} - -std::string TsEmbeddingParser::vectorToJsonArray( - const std::vector& vec) { - std::ostringstream ss; // Changed to ostringstream for consistency - ss << "["; - for (size_t i = 0; i < vec.size(); ++i) { - if (i > 0) ss << ", "; - ss << vec[i]; - } - ss << "]"; - return ss.str(); -} - -bool TsEmbeddingParser::saveToJson(const std::string& filename) const { - std::ofstream out(filename); - if (!out.is_open()) { - std::cerr << "Failed to open output file: " << filename << std::endl; +bool TsEmbeddingParser::parse_pickle(const std::string& filename) { + std::ifstream file(filename, std::ios::binary); + if (!file) { + std::cerr << "Failed to open file: " << filename << std::endl; return false; } - out << "{\n \"timestep_sequences\": {\n"; - - // Save timestep sequences - size_t seq_count = 0; - for (const auto& [steps, sequence] : ts_seq_map_) { - if (seq_count++ > 0) out << ",\n"; - out << " \"" << steps << "\": " << vectorToJsonArray(sequence); - } - - out << "\n },\n \"embeddings\": {\n"; - - // Save embeddings - size_t steps_count = 0; - for (const auto& [steps, step_embeddings] : ts_embedding_seq_map_) { - if (steps_count++ > 0) out << ",\n"; - out << " \"" << steps << "\": [\n"; - - for (size_t i = 0; i < step_embeddings.size(); ++i) { - const auto& embedding = step_embeddings[i]; + // Read timesteps array + std::vector timesteps; + uint32_t num_timesteps; + file.read(reinterpret_cast(&num_timesteps), sizeof(uint32_t)); + timesteps.resize(num_timesteps); + file.read(reinterpret_cast(timesteps.data()), num_timesteps * sizeof(int32_t)); - auto [min_it, max_it] = - std::minmax_element(embedding.begin(), embedding.end()); - float sum = std::accumulate(embedding.begin(), embedding.end(), 0.0f); - float avg = sum / embedding.size(); - - if (i > 0) out << ",\n"; - out << " {\n" - << " \"step\": " << i << ",\n" - << " \"embedding\": " << vectorToJsonArray(embedding, true) - << ",\n" // Full output - << " \"embedding_size\": " << embedding.size() << ",\n" - << " \"min\": " << *min_it << ",\n" - << " \"max\": " << *max_it << ",\n" - << " \"avg\": " << avg << "\n" - << " }"; - } - out << "\n ]"; + // Read embeddings array + std::vector> embeddings(num_timesteps); + for (auto& emb : embeddings) { + emb.resize(EMBEDDING_DIM); + file.read(reinterpret_cast(emb.data()), EMBEDDING_DIM * sizeof(float)); } - out << "\n }\n}"; + // Store in maps + timesteps_[num_timesteps] = std::move(timesteps); + embeddings_[num_timesteps] = std::move(embeddings); - if (out.fail()) { - std::cerr << "Error while writing to file: " << filename << std::endl; - return false; - } - - std::cout << "Successfully saved embeddings to " << filename << std::endl; return true; } -std::vector TsEmbeddingParser::get_timestep_embedding( - int32_t steps, int32_t step_index) const { - auto steps_it = ts_embedding_seq_map_.find(steps); - if (steps_it == ts_embedding_seq_map_.end() || - step_index >= steps_it->second.size()) { - return std::vector(); +std::vector TsEmbeddingParser::get_timestep_embedding(int32_t steps, int32_t step_index) const { + auto emb_it = embeddings_.find(steps); + if (emb_it == embeddings_.end() || step_index >= emb_it->second.size()) { + return {}; } - return steps_it->second[step_index]; + return emb_it->second[step_index]; } std::vector TsEmbeddingParser::get_timesteps(int32_t steps) const { - auto steps_it = ts_seq_map_.find(steps); - if (steps_it == ts_seq_map_.end()) { - return std::vector(); + auto ts_it = timesteps_.find(steps); + if (ts_it == timesteps_.end()) { + return {}; } - return steps_it->second; + return ts_it->second; } bool EmbeddingManager::load_timestep_embeddings(const std::string& filename) { ts_parser_ = std::make_unique(); - - std::ifstream fin(filename, std::ios::binary); - if (!fin) { - std::cerr << "Failed to open file: " << filename << std::endl; - return false; - } - auto file_size = std::filesystem::file_size(filename); - bool success = ts_parser_->parse(fin, file_size); - if (!success) { - std::cerr << "Failed to parse timestep embeddings from " << filename - << std::endl; - ts_parser_.reset(); - } - return success; + return ts_parser_->parse_pickle(filename); } -std::vector EmbeddingManager::get_timestep_embedding( - int32_t timestep, int num_steps) const { - if (!ts_parser_) { - std::cerr << "No timestep parser initialized" << std::endl; - return std::vector(); - } +std::vector EmbeddingManager::get_timestep_embedding(int32_t timestep, int num_steps) const { + if (!ts_parser_) return {}; return ts_parser_->get_timestep_embedding(num_steps, timestep); } std::vector EmbeddingManager::get_timesteps(int num_steps) const { - if (!ts_parser_) { - std::cerr << "No timestep parser initialized" << std::endl; - return std::vector(); - } + if (!ts_parser_) return {}; return ts_parser_->get_timesteps(num_steps); } \ No newline at end of file diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h index 7b915aa8a..c69314fa8 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h @@ -1,37 +1,22 @@ #ifndef EMBEDDING_UTILS_H_ #define EMBEDDING_UTILS_H_ -#include #include #include -#include -#include #include #include -#include -#include -#include #include -using float32_t = float; -using tensor_data_float32_t = std::vector; - class TsEmbeddingParser { public: - bool parse(std::ifstream& fin, size_t file_size); - std::vector get_timestep_embedding(int32_t steps, - int32_t step_index) const; + bool parse_pickle(const std::string& filename); + std::vector get_timestep_embedding(int32_t steps, int32_t step_index) const; std::vector get_timesteps(int32_t steps) const; - bool saveToJson(const std::string& filename) const; private: - static std::string vectorToJsonArray(const std::vector& vec, - bool full_output = false); - static std::string vectorToJsonArray(const std::vector& vec); - - static constexpr size_t TS_EMBEDDING_ELEMENT_COUNT = 1280; - std::map> ts_seq_map_; - std::map>> ts_embedding_seq_map_; + static constexpr size_t EMBEDDING_DIM = 1280; + std::map> timesteps_; + std::map>> embeddings_; }; class EmbeddingManager { @@ -42,8 +27,7 @@ class EmbeddingManager { } bool load_timestep_embeddings(const std::string& filename); - std::vector get_timestep_embedding(int32_t timestep, - int num_steps) const; + std::vector get_timestep_embedding(int32_t timestep, int num_steps) const; std::vector get_timesteps(int num_steps) const; private: From 1cec8b348387b1d374241cfd89cf952c31067b51 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 3 Dec 2024 11:57:22 +0200 Subject: [PATCH 11/27] chore: linting --- .../cpp/backend_tflite/embedding_utils.cc | 13 +++++++++---- .../cpp/backend_tflite/embedding_utils.h | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc index 15f998880..a4760f1be 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc @@ -1,4 +1,5 @@ #include "embedding_utils.h" + #include bool TsEmbeddingParser::parse_pickle(const std::string& filename) { @@ -13,13 +14,15 @@ bool TsEmbeddingParser::parse_pickle(const std::string& filename) { uint32_t num_timesteps; file.read(reinterpret_cast(&num_timesteps), sizeof(uint32_t)); timesteps.resize(num_timesteps); - file.read(reinterpret_cast(timesteps.data()), num_timesteps * sizeof(int32_t)); + file.read(reinterpret_cast(timesteps.data()), + num_timesteps * sizeof(int32_t)); // Read embeddings array std::vector> embeddings(num_timesteps); for (auto& emb : embeddings) { emb.resize(EMBEDDING_DIM); - file.read(reinterpret_cast(emb.data()), EMBEDDING_DIM * sizeof(float)); + file.read(reinterpret_cast(emb.data()), + EMBEDDING_DIM * sizeof(float)); } // Store in maps @@ -29,7 +32,8 @@ bool TsEmbeddingParser::parse_pickle(const std::string& filename) { return true; } -std::vector TsEmbeddingParser::get_timestep_embedding(int32_t steps, int32_t step_index) const { +std::vector TsEmbeddingParser::get_timestep_embedding( + int32_t steps, int32_t step_index) const { auto emb_it = embeddings_.find(steps); if (emb_it == embeddings_.end() || step_index >= emb_it->second.size()) { return {}; @@ -50,7 +54,8 @@ bool EmbeddingManager::load_timestep_embeddings(const std::string& filename) { return ts_parser_->parse_pickle(filename); } -std::vector EmbeddingManager::get_timestep_embedding(int32_t timestep, int num_steps) const { +std::vector EmbeddingManager::get_timestep_embedding( + int32_t timestep, int num_steps) const { if (!ts_parser_) return {}; return ts_parser_->get_timestep_embedding(num_steps, timestep); } diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h index c69314fa8..f543c6332 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.h @@ -10,7 +10,8 @@ class TsEmbeddingParser { public: bool parse_pickle(const std::string& filename); - std::vector get_timestep_embedding(int32_t steps, int32_t step_index) const; + std::vector get_timestep_embedding(int32_t steps, + int32_t step_index) const; std::vector get_timesteps(int32_t steps) const; private: @@ -27,7 +28,8 @@ class EmbeddingManager { } bool load_timestep_embeddings(const std::string& filename); - std::vector get_timestep_embedding(int32_t timestep, int num_steps) const; + std::vector get_timestep_embedding(int32_t timestep, + int num_steps) const; std::vector get_timesteps(int num_steps) const; private: From 47c54fa6154f30655f70273f1328dd741e8b0a76 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 3 Dec 2024 13:53:40 +0200 Subject: [PATCH 12/27] fix: fixed lint issue in neuron --- mobile_back_tflite/cpp/backend_tflite/neuron/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD b/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD index 3bd677e6a..0f55136f2 100644 --- a/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD +++ b/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD @@ -73,6 +73,7 @@ cc_library( "//flutter/cpp:utils", "//flutter/cpp/c:headers", "//mobile_back_tflite/cpp/backend_tflite:tflite_settings", + "//mobile_back_tflite/cpp/backend_tflite:embedding_utils", "@org_tensorflow//tensorflow/core:tflite_portable_logging", "@org_tensorflow//tensorflow/lite/c:c_api", "@org_tensorflow//tensorflow/lite/c:common", From 1c97942abd9e3c2a41ef1533f716276f5b26ce72 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 3 Dec 2024 15:21:14 +0200 Subject: [PATCH 13/27] chore: BUILD cleanup --- mobile_back_tflite/cpp/backend_tflite/BUILD | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mobile_back_tflite/cpp/backend_tflite/BUILD b/mobile_back_tflite/cpp/backend_tflite/BUILD index b2d1d538e..7aa378cd5 100644 --- a/mobile_back_tflite/cpp/backend_tflite/BUILD +++ b/mobile_back_tflite/cpp/backend_tflite/BUILD @@ -36,6 +36,16 @@ pbtxt2header( ], ) +cc_library( + name = "embedding_utils", + srcs = ["embedding_utils.cc"], + hdrs = ["embedding_utils.h"], + deps = [ + "@org_tensorflow//tensorflow/core:tflite_portable_logging", + ], + visibility = ["//visibility:public"], +) + cc_library( name = "tflite_c", srcs = [ @@ -70,6 +80,7 @@ cc_library( }), deps = [ ":tflite_settings", + ":embedding_utils", "//flutter/cpp:utils", "//flutter/cpp/c:headers", "@org_tensorflow//tensorflow/core:tflite_portable_logging", From 4b675901b71e86354eb5db1014fc97d2f1051468 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 3 Dec 2024 16:47:42 +0200 Subject: [PATCH 14/27] chore: cleanup --- WORKSPACE | 4 ++-- mobile_back_tflite/cpp/backend_tflite/BUILD | 4 ++-- mobile_back_tflite/cpp/backend_tflite/neuron/BUILD | 2 +- .../cpp/backend_tflite/stable_diffusion_pipeline.cc | 3 +-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 9c0bb08ea..01c41391c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,7 +1,7 @@ workspace(name = "mlperf_app") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "bazel_skylib", @@ -49,11 +49,11 @@ http_archive( ], ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") load( "@org_tensorflow//tensorflow/tools/toolchains/python:python_repo.bzl", "python_repository", ) +load("@rules_python//python:repositories.bzl", "python_register_toolchains") python_repository(name = "python_version_repo") diff --git a/mobile_back_tflite/cpp/backend_tflite/BUILD b/mobile_back_tflite/cpp/backend_tflite/BUILD index 7aa378cd5..651e34eba 100644 --- a/mobile_back_tflite/cpp/backend_tflite/BUILD +++ b/mobile_back_tflite/cpp/backend_tflite/BUILD @@ -40,10 +40,10 @@ cc_library( name = "embedding_utils", srcs = ["embedding_utils.cc"], hdrs = ["embedding_utils.h"], + visibility = ["//visibility:public"], deps = [ "@org_tensorflow//tensorflow/core:tflite_portable_logging", ], - visibility = ["//visibility:public"], ) cc_library( @@ -79,8 +79,8 @@ cc_library( "//conditions:default": [], }), deps = [ - ":tflite_settings", ":embedding_utils", + ":tflite_settings", "//flutter/cpp:utils", "//flutter/cpp/c:headers", "@org_tensorflow//tensorflow/core:tflite_portable_logging", diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD b/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD index 0f55136f2..e9e8cf9a7 100644 --- a/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD +++ b/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD @@ -72,8 +72,8 @@ cc_library( ":tflite_settings", "//flutter/cpp:utils", "//flutter/cpp/c:headers", - "//mobile_back_tflite/cpp/backend_tflite:tflite_settings", "//mobile_back_tflite/cpp/backend_tflite:embedding_utils", + "//mobile_back_tflite/cpp/backend_tflite:tflite_settings", "@org_tensorflow//tensorflow/core:tflite_portable_logging", "@org_tensorflow//tensorflow/lite/c:c_api", "@org_tensorflow//tensorflow/lite/c:common", diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc index 6a5c04dd9..cf3f76ea5 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc @@ -112,8 +112,7 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( } std::string ts_embedding_path = - std::string(model_path) + - "/timestep_steps_20_int32_embedding_1x1280_float32.bin.ts"; + std::string(model_path) + "/timestep_embeddings_data.pkl"; if (!EmbeddingManager::getInstance().load_timestep_embeddings( ts_embedding_path)) { LOG(ERROR) << "Failed to load timestep embeddings from " From 8eb4f4abc629c7e5ff1e3ee22a4081447714ac45 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Fri, 6 Dec 2024 21:41:35 +0200 Subject: [PATCH 15/27] chore: ignore .fvm --- .fvm/flutter_sdk | 1 - .fvm/fvm_config.json | 3 --- .fvm/release | 1 - .fvm/version | 0 .fvm/versions/stable | 1 - .gitignore | 1 + 6 files changed, 1 insertion(+), 6 deletions(-) delete mode 120000 .fvm/flutter_sdk delete mode 100644 .fvm/fvm_config.json delete mode 100644 .fvm/release delete mode 100644 .fvm/version delete mode 120000 .fvm/versions/stable diff --git a/.fvm/flutter_sdk b/.fvm/flutter_sdk deleted file mode 120000 index 380767fb9..000000000 --- a/.fvm/flutter_sdk +++ /dev/null @@ -1 +0,0 @@ -/Users/sergijrylskyj/fvm/versions/stable \ No newline at end of file diff --git a/.fvm/fvm_config.json b/.fvm/fvm_config.json deleted file mode 100644 index 1c922f80f..000000000 --- a/.fvm/fvm_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "flutterSdkVersion": "stable" -} \ No newline at end of file diff --git a/.fvm/release b/.fvm/release deleted file mode 100644 index 870bbe4e5..000000000 --- a/.fvm/release +++ /dev/null @@ -1 +0,0 @@ -stable \ No newline at end of file diff --git a/.fvm/version b/.fvm/version deleted file mode 100644 index e69de29bb..000000000 diff --git a/.fvm/versions/stable b/.fvm/versions/stable deleted file mode 120000 index 380767fb9..000000000 --- a/.fvm/versions/stable +++ /dev/null @@ -1 +0,0 @@ -/Users/sergijrylskyj/fvm/versions/stable \ No newline at end of file diff --git a/.gitignore b/.gitignore index 17e98e56c..c1b1051db 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .ijwb .idea .vscode +.fvm /bazel-* /output /output_logs From 08351574ae10d5130d04337b5f465b359e38eb74 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Fri, 6 Dec 2024 21:57:37 +0200 Subject: [PATCH 16/27] chore: updated model paths and checksums for stable diffusion benchmark: tflite_settings_android.pbtxt --- .../tflite_settings_android.pbtxt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index 2c0b67ac7..c38490632 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -216,20 +216,20 @@ benchmark_setting { # accelerator_name: "npu" # accelerator_desc: "NPU" # model_file: { -# model_path: "local:///mlperf_models/stable-diffusion/decoder.tflite" -# model_checksum: "491385ad873880ba1876e1d097fcc0e3" +# model_path: "local:///mlperf_models/stable-diffusion/sd_text_encoder_dynamic.tflite" +# model_checksum: "b64effb0360f9ea49a117cdaf8a2fbdc" # } # model_file: { -# model_path: "local:///mlperf_models/stable-diffusion/text_encoder.tflite" -# model_checksum: "8985768b09fe31b805e66b6048da9125" +# model_path: "local:///mlperf_models/stable-diffusion/sd_diffusion_model_dynamic.tflite" +# model_checksum: "ccfd761a2f8186c3669948515d40a880" # } # model_file: { -# model_path: "local:///mlperf_models/stable-diffusion/first_model.tflite" -# model_checksum: "f0d6f45a2d702456a234c0a9b192816a" +# model_path: "local:///mlperf_models/stable-diffusion/sd_decoder_dynamic.tflite" +# model_checksum: "68acdb62f99e1dc2c7f5db8cdd0e007c" # } # model_file: { -# model_path: "local:///mlperf_models/stable-diffusion/second_model.tflite" -# model_checksum: "cea07208776347a8a5334106a09444fe" +# model_path: "local:///mlperf_models/stable-diffusion/timestep_embeddings_data.pkl" +# model_checksum: "c8762274fd95bdd40e60a1987e19e29e" # } # } # delegate_selected: "NNAPI" From b03576897deef5e97c75a14a723e64923e54d686 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Sun, 29 Dec 2024 20:30:10 +0200 Subject: [PATCH 17/27] chore: reverse timesteps and embeddings to support descending order of the timesteps and embeddings --- mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc index a4760f1be..99e9109a3 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc @@ -25,6 +25,10 @@ bool TsEmbeddingParser::parse_pickle(const std::string& filename) { EMBEDDING_DIM * sizeof(float)); } + // Reverse both timesteps and embeddings before storing + std::reverse(timesteps.begin(), timesteps.end()); + std::reverse(embeddings.begin(), embeddings.end()); + // Store in maps timesteps_[num_timesteps] = std::move(timesteps); embeddings_[num_timesteps] = std::move(embeddings); From f2f132a3279c07e1b81f37470aa61bef4e29b59a Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 31 Dec 2024 08:03:20 +0200 Subject: [PATCH 18/27] chore: fixed formatting --- mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc index 99e9109a3..9f25eb4e3 100644 --- a/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc +++ b/mobile_back_tflite/cpp/backend_tflite/embedding_utils.cc @@ -26,8 +26,8 @@ bool TsEmbeddingParser::parse_pickle(const std::string& filename) { } // Reverse both timesteps and embeddings before storing - std::reverse(timesteps.begin(), timesteps.end()); - std::reverse(embeddings.begin(), embeddings.end()); + std::reverse(timesteps.begin(), timesteps.end()); + std::reverse(embeddings.begin(), embeddings.end()); // Store in maps timesteps_[num_timesteps] = std::move(timesteps); From 336bd7667e012fe2e190d99c749c1fa01633c2f9 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Fri, 3 Jan 2025 19:19:31 +0200 Subject: [PATCH 19/27] chore: added links to the sd models and timestep embeddings file --- .../backend_settings/tflite_settings_android.pbtxt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index 4aab5dbb3..bfec84e5f 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -215,17 +215,21 @@ benchmark_setting { accelerator_name: "npu" accelerator_desc: "NPU" model_file: { - model_path: "local:///mlperf_models/sd_decoder_dynamic.tflite" + model_path: "https://github.com/RSMNYS/SD/releases/download/SD/sd_decoder_dynamic.tflite" model_checksum: "68acdb62f99e1dc2c7f5db8cdd0e007c" } model_file: { - model_path: "local:///mlperf_models/sd_diffusion_model_dynamic.tflite" - model_checksum: "7cbdadf5282b71561ce5eda75e868c19" + model_path: "https://github.com/RSMNYS/SD/releases/download/SD/sd_diffusion_model_dynamic.tflite" + model_checksum: "309e95f76ac8de01130942037a28aa8f" } model_file: { - model_path: "local:///mlperf_models/sd_text_encoder_dynamic.tflite" + model_path: "https://github.com/RSMNYS/SD/releases/download/SD/sd_text_encoder_dynamic.tflite" model_checksum: "b64effb0360f9ea49a117cdaf8a2fbdc" } + model_file: { + model_path: "https://github.com/RSMNYS/SD/releases/download/SD/timestep_embeddings_data.bin.ts" + model_checksum: "798b772155a69de5df44b304327bb3cc" + } } delegate_selected: "NNAPI" custom_setting { From f222a1488504d6eeb233c53d319cdb914eabfa8a Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Mon, 6 Jan 2025 09:51:52 +0200 Subject: [PATCH 20/27] chore: add the proper name for the embedding_timesteps file --- .../cpp/backend_tflite/stable_diffusion_pipeline.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc index cf3f76ea5..de7ddba57 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc @@ -112,7 +112,7 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( } std::string ts_embedding_path = - std::string(model_path) + "/timestep_embeddings_data.pkl"; + std::string(model_path) + "/timestep_embeddings_data.bin.ts"; if (!EmbeddingManager::getInstance().load_timestep_embeddings( ts_embedding_path)) { LOG(ERROR) << "Failed to load timestep embeddings from " From b14424ee8a7d3b93886a6d7680f9b3d57cfb7251 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Mon, 6 Jan 2025 09:54:12 +0200 Subject: [PATCH 21/27] chore: added missed declaration for backend_convert_outputs --- .../cpp/backend_tflite/stable_diffusion_pipeline.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h index adf460530..2f4af4e5c 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h @@ -91,6 +91,10 @@ class StableDiffusionPipeline : public Pipeline { void backend_convert_inputs(mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, uint8_t *data) override; + void backend_convert_outputs( + mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, + uint8_t* data) override; + void *backend_get_buffer(size_t n) override; void backend_release_buffer(void *p) override; From 3e5a5e264745018955c9e22e31e046e27982f973 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Mon, 6 Jan 2025 10:19:06 +0200 Subject: [PATCH 22/27] chore: clang formatting --- .../cpp/backend_tflite/stable_diffusion_pipeline.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h index 2f4af4e5c..17070a286 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.h @@ -91,9 +91,8 @@ class StableDiffusionPipeline : public Pipeline { void backend_convert_inputs(mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, uint8_t *data) override; - void backend_convert_outputs( - mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, - uint8_t* data) override; + void backend_convert_outputs(mlperf_backend_ptr_t backend_ptr, int bytes, + int width, int height, uint8_t *data) override; void *backend_get_buffer(size_t n) override; From 809b11d9ae1286277f1d68b74de26520c6316e6a Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 7 Jan 2025 10:42:38 +0200 Subject: [PATCH 23/27] chore: added missed files --- mobile_back_pixel/cpp/backend_tflite/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile_back_pixel/cpp/backend_tflite/BUILD b/mobile_back_pixel/cpp/backend_tflite/BUILD index 54947f346..38624dc2c 100644 --- a/mobile_back_pixel/cpp/backend_tflite/BUILD +++ b/mobile_back_pixel/cpp/backend_tflite/BUILD @@ -60,6 +60,7 @@ cc_library( hdrs = [ "tflite_settings_pixel.h", "thread_pool.h", + "//mobile_back_tflite/cpp/backend_tflite:embedding_utils.h", "//mobile_back_tflite/cpp/backend_tflite:pipeline.h", "//mobile_back_tflite/cpp/backend_tflite:sd_utils.h", "//mobile_back_tflite/cpp/backend_tflite:single_model_pipeline.h", From ddedee4398cceb18e68ce644612dfe6ee97916d5 Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 7 Jan 2025 11:17:18 +0200 Subject: [PATCH 24/27] chore: fixed build file for the pixel backend --- mobile_back_pixel/cpp/backend_tflite/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile_back_pixel/cpp/backend_tflite/BUILD b/mobile_back_pixel/cpp/backend_tflite/BUILD index 38624dc2c..6e071660e 100644 --- a/mobile_back_pixel/cpp/backend_tflite/BUILD +++ b/mobile_back_pixel/cpp/backend_tflite/BUILD @@ -56,6 +56,7 @@ cc_library( "//mobile_back_tflite/cpp/backend_tflite:sd_utils.cc", "//mobile_back_tflite/cpp/backend_tflite:stable_diffusion_invoker.cc", "//mobile_back_tflite/cpp/backend_tflite:stable_diffusion_pipeline.cc", + "//mobile_back_tflite/cpp/backend_tflite:embedding_utils.cc" ], hdrs = [ "tflite_settings_pixel.h", From 3d58e5a05efc5290f285727153ff04505287486e Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 7 Jan 2025 11:23:34 +0200 Subject: [PATCH 25/27] chore: bazel formatting --- mobile_back_pixel/cpp/backend_tflite/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_back_pixel/cpp/backend_tflite/BUILD b/mobile_back_pixel/cpp/backend_tflite/BUILD index 6e071660e..8b59fc413 100644 --- a/mobile_back_pixel/cpp/backend_tflite/BUILD +++ b/mobile_back_pixel/cpp/backend_tflite/BUILD @@ -53,10 +53,10 @@ cc_library( srcs = [ "pixel_single_model_pipeline.cc", "tflite_pixel.cc", + "//mobile_back_tflite/cpp/backend_tflite:embedding_utils.cc", "//mobile_back_tflite/cpp/backend_tflite:sd_utils.cc", "//mobile_back_tflite/cpp/backend_tflite:stable_diffusion_invoker.cc", "//mobile_back_tflite/cpp/backend_tflite:stable_diffusion_pipeline.cc", - "//mobile_back_tflite/cpp/backend_tflite:embedding_utils.cc" ], hdrs = [ "tflite_settings_pixel.h", From 61706274a7d7f2d6ddeffa4020a56880f047853d Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 7 Jan 2025 14:40:09 +0200 Subject: [PATCH 26/27] fix: added missed interface implementation for pixel --- .../cpp/backend_tflite/pixel_single_model_pipeline.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mobile_back_pixel/cpp/backend_tflite/pixel_single_model_pipeline.cc b/mobile_back_pixel/cpp/backend_tflite/pixel_single_model_pipeline.cc index 1dc201cfb..6be9a726e 100644 --- a/mobile_back_pixel/cpp/backend_tflite/pixel_single_model_pipeline.cc +++ b/mobile_back_pixel/cpp/backend_tflite/pixel_single_model_pipeline.cc @@ -413,6 +413,10 @@ void SingleModelPipeline::backend_convert_inputs( mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, uint8_t* data) {} +void SingleModelPipeline::backend_convert_outputs( + mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, + uint8_t *data) {} + void* SingleModelPipeline::backend_get_buffer(size_t n) { return ::operator new(n); } From f29dccd30e6b6d07d6aa957fe56e79017ea42d5f Mon Sep 17 00:00:00 2001 From: Sergij Rylskyj Date: Tue, 7 Jan 2025 15:12:19 +0200 Subject: [PATCH 27/27] chore: clang formatting --- .../cpp/backend_tflite/pixel_single_model_pipeline.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_back_pixel/cpp/backend_tflite/pixel_single_model_pipeline.cc b/mobile_back_pixel/cpp/backend_tflite/pixel_single_model_pipeline.cc index 6be9a726e..1d44b411f 100644 --- a/mobile_back_pixel/cpp/backend_tflite/pixel_single_model_pipeline.cc +++ b/mobile_back_pixel/cpp/backend_tflite/pixel_single_model_pipeline.cc @@ -415,7 +415,7 @@ void SingleModelPipeline::backend_convert_inputs( void SingleModelPipeline::backend_convert_outputs( mlperf_backend_ptr_t backend_ptr, int bytes, int width, int height, - uint8_t *data) {} + uint8_t* data) {} void* SingleModelPipeline::backend_get_buffer(size_t n) { return ::operator new(n);