From 14eee1a076aa1af7ec1ae3c752be79ae2604a708 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalev Date: Thu, 5 Nov 2020 15:42:32 -0800 Subject: [PATCH] Release Frogfish --- .gitignore | 1 - BUILD | 7 + Makefile | 39 ++- WORKSPACE | 23 ++ api/BUILD | 34 +-- api/driver.h | 3 +- api/layer_information.cc | 60 ++-- bazel/WORKSPACE | 63 ----- bazel/WORKSPACE.darwin | 13 - bazel/WORKSPACE.linux | 12 - bazel/WORKSPACE.windows | 21 -- build.bat | 35 ++- debian/changelog | 6 + debian/compat | 1 + debian/control | 46 +++ debian/copyright | 6 + debian/edgetpu-accelerator.rules | 2 + debian/libedgetpu-dev.install | 2 + debian/libedgetpu1-max.lintian-overrides | 4 + debian/libedgetpu1-max.preinst | 24 ++ debian/libedgetpu1-max.templates | 21 ++ debian/libedgetpu1-max.triggers | 1 + debian/libedgetpu1-max.udev | 1 + debian/libedgetpu1-std.lintian-overrides | 2 + debian/libedgetpu1-std.triggers | 1 + debian/libedgetpu1-std.udev | 1 + debian/rules | 46 +++ docker/Dockerfile | 15 + docker/Dockerfile.windows | 13 + docker/update_sources.sh | 13 + driver/BUILD | 69 ++--- driver/beagle/BUILD | 264 ++++++++++-------- driver/beagle/beagle_ioctl.h | 5 +- .../beagle/beagle_kernel_top_level_handler.cc | 11 +- .../beagle/beagle_kernel_top_level_handler.h | 3 +- driver/beagle/beagle_pci_driver_provider.cc | 42 +-- driver/beagle/beagle_pci_driver_provider.h | 54 ++++ .../beagle_pci_driver_provider_linux.cc | 63 +++++ .../beagle_pci_driver_provider_windows.cc | 64 +++++ driver/beagle/beagle_usb_driver_provider.cc | 4 +- driver/config/BUILD | 9 +- driver/config/beagle/BUILD | 6 +- driver/config/beagle/beagle_chip_structures.h | 1 + driver/config/beagle/beagle_csr_offsets.h | 32 +++ driver/config/chip_config.h | 1 + driver/config/chip_structures.h | 2 + driver/config/common_csr_helper.h | 1 + driver/config/lpm_csr_offsets.h | 41 +++ driver/config/scalar_core_csr_offsets.h | 29 ++ driver/config/tile_csr_offsets.h | 8 + driver/driver.cc | 3 +- driver/driver.h | 8 +- driver/driver_factory_windows.cc | 56 +++- driver/interrupt/BUILD | 16 +- driver/interrupt/wire_interrupt_handler.cc | 3 - driver/kernel/BUILD | 45 +-- ...gasket_ioctl.h => common_gasket_ioctl.inc} | 26 +- driver/kernel/gasket_ioctl.h | 35 +++ driver/kernel/kernel_coherent_allocator.cc | 39 ++- driver/kernel/kernel_coherent_allocator.h | 20 +- driver/kernel/kernel_event.h | 29 +- driver/kernel/kernel_event_handler.cc | 41 +-- driver/kernel/kernel_event_handler.h | 33 ++- driver/kernel/kernel_interrupt_handler.cc | 11 +- driver/kernel/kernel_interrupt_handler.h | 5 +- driver/kernel/kernel_mmu_mapper.cc | 27 +- driver/kernel/kernel_mmu_mapper.h | 12 +- driver/kernel/kernel_registers.cc | 73 +++-- driver/kernel/kernel_registers.h | 17 +- .../kernel/kernel_wire_interrupt_handler.cc | 15 +- driver/kernel/kernel_wire_interrupt_handler.h | 5 +- driver/kernel/linux/BUILD | 83 ++++++ .../linux/kernel_coherent_allocator_linux.cc | 62 ++++ .../linux/kernel_coherent_allocator_linux.h | 44 +++ .../linux/kernel_event_handler_linux.cc | 69 +++++ .../kernel/linux/kernel_event_handler_linux.h | 52 ++++ .../kernel_event_linux.cc} | 39 ++- driver/kernel/linux/kernel_event_linux.h | 66 +++++ driver/kernel/linux/kernel_registers_linux.cc | 71 +++++ driver/kernel/linux/kernel_registers_linux.h | 51 ++++ driver/kernel/windows/BUILD | 113 ++++++++ .../kernel_coherent_allocator_windows.cc | 62 ++++ .../kernel_coherent_allocator_windows.h | 44 +++ .../windows/kernel_event_handler_windows.cc | 119 ++++++++ .../windows/kernel_event_handler_windows.h | 52 ++++ driver/kernel/windows/kernel_event_windows.cc | 98 +++++++ driver/kernel/windows/kernel_event_windows.h | 66 +++++ .../windows/kernel_registers_windows.cc | 83 ++++++ .../kernel/windows/kernel_registers_windows.h | 51 ++++ .../kernel/windows/windows_gasket_ioctl.inc | 168 +++++++++++ driver/memory/BUILD | 30 +- driver/mmio/BUILD | 6 +- driver/mmio/host_queue.h | 48 ++++ driver/mmio_driver.cc | 17 +- driver/mmio_driver.h | 4 +- driver/real_time_dma_scheduler.h | 6 +- driver/registers/BUILD | 11 +- driver/registers/registers.h | 14 +- driver/request.cc | 11 +- driver/request.h | 6 +- driver/usb/BUILD | 28 +- driver/usb/apex_latest_multi_ep.bin | Bin driver/usb/apex_latest_single_ep.bin | Bin driver/usb/usb_driver.cc | 11 +- driver/usb/usb_driver.h | 6 +- driver_shared/BUILD | 29 ++ driver_shared/registers.h | 65 +++++ {driver => driver_shared}/time_stamper/BUILD | 44 ++- .../time_stamper/driver_time_stamper.cc | 7 +- .../time_stamper/driver_time_stamper.h | 13 +- .../driver_time_stamper_factory.h | 16 +- .../time_stamper/time_stamper.h | 11 +- .../time_stamper/time_stamper_factory.h | 14 +- executable/BUILD | 70 ----- executable/executable.fbs | 8 +- libedgetpu_cc_rules.bzl | 114 ++++++++ port/BUILD | 137 +++++++-- port/aligned_malloc.h | 2 + port/array_slice.h | 2 + port/builddata.h | 2 + port/casts.h | 2 + port/default/BUILD | 67 ++++- port/default/builddata.cc | 35 +++ port/default/builddata.h | 5 +- port/default/dma_manager.h | 70 +++++ port/default/port_from_tf/BUILD | 8 +- port/default/port_from_tf/logging.h | 2 + port/default/port_from_tf/statusor.cc | 6 + port/default/port_from_tf/statusor.h | 122 +++++++- .../default/port_from_tf/statusor_internals.h | 2 + port/default/semaphore.h | 70 +++-- port/default/status_macros.h | 17 +- port/dma_manager.h | 38 +++ port/errors.h | 2 + port/fileio.h | 24 ++ port/fileio_linux.h | 31 ++ port/fileio_windows.cc | 75 +++++ port/fileio_windows.h | 49 ++++ port/integral_types.h | 2 + port/logging.h | 2 + port/macros.h | 2 + port/math_util.h | 2 + port/ptr_util.h | 2 + port/status.h | 2 + port/status_macros.h | 2 + port/statusor.h | 11 +- port/stringprintf.h | 2 + port/tracing.h | 123 +++----- scripts/install.sh | 185 ++++++++++++ scripts/uninstall.sh | 103 +++++++ tflite/BUILD | 29 +- tflite/edgetpu_context_factory.cc | 9 +- ...pu_delegate_for_custom_op_tflite_plugin.cc | 60 ++-- tflite/edgetpu_manager_direct.cc | 12 +- tflite/public/BUILD | 204 +++++++++++--- tflite/public/edgetpu.h | 203 ++++++++------ tflite/public/edgetpu.rc | 42 +++ tflite/public/edgetpu_c.h | 2 +- tflite/public/libedgetpu.lds | 27 ++ workspace.bzl | 109 ++++++++ 160 files changed, 4484 insertions(+), 1145 deletions(-) create mode 100644 WORKSPACE delete mode 100644 bazel/WORKSPACE delete mode 100644 bazel/WORKSPACE.darwin delete mode 100644 bazel/WORKSPACE.linux delete mode 100644 bazel/WORKSPACE.windows create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/edgetpu-accelerator.rules create mode 100644 debian/libedgetpu-dev.install create mode 100644 debian/libedgetpu1-max.lintian-overrides create mode 100644 debian/libedgetpu1-max.preinst create mode 100644 debian/libedgetpu1-max.templates create mode 100644 debian/libedgetpu1-max.triggers create mode 120000 debian/libedgetpu1-max.udev create mode 100644 debian/libedgetpu1-std.lintian-overrides create mode 100644 debian/libedgetpu1-std.triggers create mode 120000 debian/libedgetpu1-std.udev create mode 100755 debian/rules create mode 100644 driver/beagle/beagle_pci_driver_provider.h create mode 100644 driver/beagle/beagle_pci_driver_provider_linux.cc create mode 100644 driver/beagle/beagle_pci_driver_provider_windows.cc create mode 100644 driver/config/lpm_csr_offsets.h rename driver/kernel/{linux_gasket_ioctl.h => common_gasket_ioctl.inc} (86%) create mode 100644 driver/kernel/gasket_ioctl.h create mode 100644 driver/kernel/linux/BUILD create mode 100644 driver/kernel/linux/kernel_coherent_allocator_linux.cc create mode 100644 driver/kernel/linux/kernel_coherent_allocator_linux.h create mode 100644 driver/kernel/linux/kernel_event_handler_linux.cc create mode 100644 driver/kernel/linux/kernel_event_handler_linux.h rename driver/kernel/{kernel_event.cc => linux/kernel_event_linux.cc} (80%) create mode 100644 driver/kernel/linux/kernel_event_linux.h create mode 100644 driver/kernel/linux/kernel_registers_linux.cc create mode 100644 driver/kernel/linux/kernel_registers_linux.h create mode 100644 driver/kernel/windows/BUILD create mode 100644 driver/kernel/windows/kernel_coherent_allocator_windows.cc create mode 100644 driver/kernel/windows/kernel_coherent_allocator_windows.h create mode 100644 driver/kernel/windows/kernel_event_handler_windows.cc create mode 100644 driver/kernel/windows/kernel_event_handler_windows.h create mode 100644 driver/kernel/windows/kernel_event_windows.cc create mode 100644 driver/kernel/windows/kernel_event_windows.h create mode 100644 driver/kernel/windows/kernel_registers_windows.cc create mode 100644 driver/kernel/windows/kernel_registers_windows.h create mode 100644 driver/kernel/windows/windows_gasket_ioctl.inc mode change 100755 => 100644 driver/usb/apex_latest_multi_ep.bin mode change 100755 => 100644 driver/usb/apex_latest_single_ep.bin create mode 100644 driver_shared/BUILD create mode 100644 driver_shared/registers.h rename {driver => driver_shared}/time_stamper/BUILD (60%) rename {driver => driver_shared}/time_stamper/driver_time_stamper.cc (87%) rename {driver => driver_shared}/time_stamper/driver_time_stamper.h (74%) rename {driver => driver_shared}/time_stamper/driver_time_stamper_factory.h (69%) rename {driver => driver_shared}/time_stamper/time_stamper.h (91%) rename {driver => driver_shared}/time_stamper/time_stamper_factory.h (78%) create mode 100644 libedgetpu_cc_rules.bzl create mode 100644 port/default/builddata.cc create mode 100644 port/default/dma_manager.h create mode 100644 port/dma_manager.h create mode 100644 port/fileio.h create mode 100644 port/fileio_linux.h create mode 100644 port/fileio_windows.cc create mode 100644 port/fileio_windows.h create mode 100755 scripts/install.sh create mode 100755 scripts/uninstall.sh create mode 100644 tflite/public/edgetpu.rc create mode 100644 workspace.bzl diff --git a/.gitignore b/.gitignore index e6c73c5..4a44468 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ bazel-* out/ -WORKSPACE diff --git a/BUILD b/BUILD index b7f8359..de1eb14 100644 --- a/BUILD +++ b/BUILD @@ -25,6 +25,13 @@ exports_files([ "LICENSE", ]) +config_setting( + name = "opt", + values = { + "compilation_mode": "opt", + }, +) + # If --define darwinn_portable=1, compile without google3 deps. config_setting( name = "darwinn_portable", diff --git a/Makefile b/Makefile index b6f8edf..0457986 100644 --- a/Makefile +++ b/Makefile @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. SHELL := /bin/bash +PYTHON3 ?= python3 MAKEFILE_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) OUT_DIR := $(MAKEFILE_DIR)/out OS := $(shell uname -s) ifeq ($(OS),Linux) CPU ?= k8 -WORKSPACE_PLATFORM_FILE := WORKSPACE.linux else ifeq ($(OS),Darwin) CPU ?= darwin -WORKSPACE_PLATFORM_FILE := WORKSPACE.darwin else $(error $(OS) is not supported) endif @@ -40,23 +39,16 @@ BAZEL_OUT_DIR := $(MAKEFILE_DIR)/bazel-out/$(CPU)-$(COMPILATION_MODE)/bin # Linux-specific parameters BAZEL_BUILD_TARGET_Linux := //tflite/public:libedgetpu_direct_all.so BAZEL_BUILD_FLAGS_Linux := --crosstool_top=@crosstool//:toolchains \ - --compiler=gcc \ - --linkopt=-l:libusb-1.0.so + --compiler=gcc BAZEL_BUILD_OUTPUT_FILE_Linux := libedgetpu.so.1.0 BAZEL_BUILD_OUTPUT_SYMLINK_Linux := libedgetpu.so.1 -ifeq ($(COMPILATION_MODE), opt) -BAZEL_BUILD_FLAGS_Linux += --linkopt=-Wl,--strip-all -endif ifeq ($(CPU), armv6) BAZEL_BUILD_FLAGS_Linux += --linkopt=-L/usr/lib/arm-linux-gnueabihf/ endif # Darwin-specific parameters BAZEL_BUILD_TARGET_Darwin := //tflite/public:libedgetpu_direct_usb.dylib -BAZEL_BUILD_FLAGS_Darwin := --linkopt=-L/opt/local/lib \ - --linkopt=-lusb-1.0 \ - --copt=-fvisibility=hidden BAZEL_BUILD_OUTPUT_FILE_Darwin := libedgetpu.1.0.dylib BAZEL_BUILD_OUTPUT_SYMLINK_Darwin := libedgetpu.1.dylib @@ -65,11 +57,10 @@ BAZEL_BUILD_FLAGS := --sandbox_debug --subcommands \ --experimental_repo_remote_exec \ --compilation_mode=$(COMPILATION_MODE) \ --define darwinn_portable=1 \ - --copt=-DSTRIP_LOG=1 \ - --copt=-fno-rtti \ - --copt=-fno-exceptions \ - --copt='-D__FILE__=""' \ - --cpu=$(CPU) + --action_env PYTHON_BIN_PATH=$(shell which $(PYTHON3)) \ + --cpu=$(CPU) \ + --embed_label='TENSORFLOW_COMMIT=$(shell grep "TENSORFLOW_COMMIT =" $(MAKEFILE_DIR)/workspace.bzl | grep -o '[0-9a-f]\{40\}')' \ + --stamp BAZEL_BUILD_FLAGS += $(BAZEL_BUILD_FLAGS_$(OS)) BAZEL_BUILD_TARGET := $(BAZEL_BUILD_TARGET_$(OS)) BAZEL_BUILD_OUTPUT_FILE := $(BAZEL_BUILD_OUTPUT_FILE_$(OS)) @@ -95,23 +86,31 @@ endif .PHONY: libedgetpu \ libedgetpu-direct \ libedgetpu-throttled \ - workspace \ + deb \ + deb-armhf \ + deb-arm64 \ clean libedgetpu: libedgetpu-direct libedgetpu-throttled -libedgetpu-direct: workspace +libedgetpu-direct: bazel build $(BAZEL_BUILD_FLAGS) $(BAZEL_BUILD_TARGET) $(call copy_out,direct) $(call strip_out,direct) -libedgetpu-throttled: workspace +libedgetpu-throttled: bazel build $(BAZEL_BUILD_FLAGS) --copt=-DTHROTTLE_EDGE_TPU $(BAZEL_BUILD_TARGET) $(call copy_out,throttled) $(call strip_out,throttled) -workspace: bazel/WORKSPACE bazel/$(WORKSPACE_PLATFORM_FILE) - cat $^ > WORKSPACE +deb: + dpkg-buildpackage -rfakeroot -us -uc -tc -b + +deb-armhf: + dpkg-buildpackage -rfakeroot -us -uc -tc -b -a armhf -d + +deb-arm64: + dpkg-buildpackage -rfakeroot -us -uc -tc -b -a arm64 -d clean: rm -rf $(OUT_DIR) diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..ef1776f --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,23 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +workspace(name = "libedgetpu") + +load(":workspace.bzl", "libedgetpu_dependencies") + +libedgetpu_dependencies() +load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace") +tf_workspace(tf_repo_name = "org_tensorflow") + +load("@coral_crosstool//:configure.bzl", "cc_crosstool") +cc_crosstool(name = "crosstool") diff --git a/api/BUILD b/api/BUILD index 9f06da5..94da6a4 100644 --- a/api/BUILD +++ b/api/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + # Description: # Darwinn API headers load( @@ -23,12 +25,12 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "chip", hdrs = ["chip.h"], ) -cc_library( +libedgetpu_cc_library( name = "tensor_util", srcs = ["tensor_util.cc"], hdrs = ["tensor_util.h"], @@ -39,7 +41,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "layer_information", srcs = ["layer_information.cc"], hdrs = ["layer_information.h"], @@ -51,7 +53,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "request", hdrs = ["request.h"], deps = [ @@ -60,7 +62,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "driver", hdrs = ["driver.h"], deps = [ @@ -73,7 +75,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "driver_options_helper", srcs = ["driver_options_helper.cc"], hdrs = ["driver_options_helper.h"], @@ -83,20 +85,20 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "allocated_buffer", srcs = ["allocated_buffer.cc"], hdrs = ["allocated_buffer.h"], deps = ["//port"], ) -cc_library( +libedgetpu_cc_library( name = "dram_buffer", hdrs = ["dram_buffer.h"], deps = ["//port"], ) -cc_library( +libedgetpu_cc_library( name = "buffer", srcs = ["buffer.cc"], hdrs = ["buffer.h"], @@ -107,7 +109,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "driver_factory", srcs = ["driver_factory.cc"], hdrs = ["driver_factory.h"], @@ -119,7 +121,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "package_reference", hdrs = ["package_reference.h"], deps = [ @@ -130,7 +132,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "runtime_version", hdrs = ["runtime_version.h"], ) @@ -141,7 +143,7 @@ flatbuffer_cc_library( flatc_args = [""], ) -cc_library( +libedgetpu_cc_library( name = "timing", hdrs = ["timing.h"], deps = [ @@ -149,7 +151,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "watchdog", srcs = ["watchdog.cc"], hdrs = ["watchdog.h"], @@ -161,7 +163,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "telemeter_interface", hdrs = [ "telemeter_interface.h", @@ -171,7 +173,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "execution_context_interface", hdrs = [ "execution_context_interface.h", diff --git a/api/driver.h b/api/driver.h index 58aa671..05815a9 100644 --- a/api/driver.h +++ b/api/driver.h @@ -85,8 +85,7 @@ class Driver { // integer values may be different from those in NNAPI or other APIs. The // values here are defined in the order of priority when there are multiple // models requesting different preferences (e.g. sustained speed takes - // priority over low power). For more information, please see: - // http://go/noronha-execution-preference + // priority over low power). enum class ExecutionPreference { // Run at the absolute maximum performance. kSingleFastAnswer = 0, diff --git a/api/layer_information.cc b/api/layer_information.cc index f6d251c..c589fc6 100644 --- a/api/layer_information.cc +++ b/api/layer_information.cc @@ -203,30 +203,41 @@ bool OutputLayerInformation::NeedsRelayout() const { // TODO Add unit tests for this method. util::Status OutputLayerInformation::Relayout(unsigned char* dest, const unsigned char* src) const { - // TODO: re-use the same buffer and avoid an unnecessary memcopy - // when relayout is not needed. - if (!NeedsRelayout()) { - memcpy(dest, src, - batch_dim() * y_dim() * x_dim() * z_dim() * DataTypeSize()); - return util::OkStatus(); - } - - if (output_layer_->shape_info()) { - // If output shape info exists in the executable, use the new re-layout - // function. Currently, this is only enabled for models with multiple - // batches. - return RelayoutWithShapeInformation(dest, src); - } - const auto data_type_size = DataTypeSize(); const int z_bytes = z_dim() * data_type_size; + const int executions = execution_count_per_inference(); + + if (executions == 1) { + // Handle case when execution count is equal to 1, since if execution count + // is greater than 1, there might be padding data in-between. + + // TODO: re-use the same buffer and avoid an unnecessary + // memcopy when relayout is not needed. + if (!NeedsRelayout()) { + memcpy(dest, src, batch_dim() * y_dim() * x_dim() * z_bytes); + return util::OkStatus(); + } + + if (output_layer_->shape_info()) { + // If output shape info exists in the executable, use the new re-layout + // function. Currently, this is only enabled for models with multiple + // batches. + return RelayoutWithShapeInformation(dest, src); + } + } else if (PaddedSizeBytes() == ActualSizeBytes() && !NeedsRelayout()) { + // Use memcpy if `executions` is greater than 1 and there is no internal + // padding between iterations. + if (dest != src) { + memcpy(dest, src, ActualSizeBytes()); + } + return util::OkStatus(); + } if (y_dim() == 1 && x_dim() == 1) { // One dimensional output (only z-dimension). if (src != dest) { const int padded_size_bytes = PaddedSizeBytes(); const int actual_size_bytes = ActualSizeBytes(); - const int executions = execution_count_per_inference(); if (executions == 1 || padded_size_bytes == actual_size_bytes) { memcpy(dest, src, z_bytes * executions); } else { @@ -276,8 +287,7 @@ util::Status OutputLayerInformation::Relayout(unsigned char* dest, // provided we have a guaranteed way of ensuring this function would be inlined // so that the compiler optimizations based on compile-time-constants can kick // in. -#define RELAYOUT_WITH_Z_BYTES_SPECIALIZATION( \ - num_z_bytes, num_z_bytes_padded) \ +#define RELAYOUT_WITH_Z_BYTES_SPECIALIZATION(num_z_bytes, num_z_bytes_padded) \ do { \ for (int y = 0; y < y_dim(); ++y) { \ const auto y_buffer_index = GetYBufferIndex(y); \ @@ -330,6 +340,12 @@ util::Status OutputLayerInformation::Relayout(unsigned char* dest, active_tile_x_sizes.size() > 1 || first_y_tile != last_y_tile; if (need_relayout) { + // TODO: If iteration count is more than 1, we need to make + // sure we advance 'src' and 'dest' correctly due to padding issue. We + // don't have test case now. + CHECK_EQ(executions, 1) + << "Verification is missing if execution count is greater than 1"; + // If there's no z padding, copy one xz block on one tile at a time. for (int y = 0; y < y_dim(); ++y) { const auto y_buffer_index = GetYBufferIndex(y); @@ -347,17 +363,11 @@ util::Status OutputLayerInformation::Relayout(unsigned char* dest, } } else { // TODO: avoid copy and assign in caller directly. - memcpy(dest, src, x_dim() * y_dim() * z_bytes); + memcpy(dest, src, x_dim() * y_dim() * z_bytes * executions); } } #undef RELAYOUT_WITH_Z_BYTES_SPECIALIZATION - - // TODO: If iteration count is more than 1, we need to make sure we - // advance 'src' and 'dest' correctly due to padding issue. We don't have - // test case now. - CHECK_EQ(execution_count_per_inference(), 1) - << "Verification is missing if execution count is greater than 1"; } return util::OkStatus(); diff --git a/bazel/WORKSPACE b/bazel/WORKSPACE deleted file mode 100644 index f9cb049..0000000 --- a/bazel/WORKSPACE +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -workspace(name = "libedgetpu") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -http_archive( - name = "io_bazel_rules_closure", - sha256 = "5b00383d08dd71f28503736db0500b6fb4dda47489ff5fc6bed42557c07c6ba9", - strip_prefix = "rules_closure-308b05b2419edb5c8ee0471b67a40403df940149", - urls = [ - "http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz", - "https://github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz", # 2019-06-13 - ], -) - -# Be consistent with tensorflow/WORKSPACE. -http_archive( - name = "bazel_skylib", - sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0", - urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz"], -) # https://github.com/bazelbuild/bazel-skylib/releases - -# The TF commit # here must be in sync with that specified under Gob edgetpu -# repo WORKSPACE file. -# TODO: figure out a way to keep single source of truth of the -# TF commit # used. -TENSORFLOW_COMMIT = "f394a768719a55b5c351ed1ecab2ec6f16f99dd4"; -# Command to calculate: curl -OL | sha256sum | awk '{print $1}' -TENSORFLOW_SHA256 = "cb286abee7ee9cf5c8701d85fcc88f0fd59e72492ec4f254156de486e3e905c1" -http_archive( - name = "org_tensorflow", - sha256 = TENSORFLOW_SHA256, - strip_prefix = "tensorflow-" + TENSORFLOW_COMMIT, - urls = [ - "https://github.com/tensorflow/tensorflow/archive/" + TENSORFLOW_COMMIT + ".tar.gz", - ], -) - -load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace") -tf_workspace(tf_repo_name = "org_tensorflow") - -http_archive( - name = "coral_crosstool", - sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb", - strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229", - urls = [ - "https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz", - ], -) -load("@coral_crosstool//:configure.bzl", "cc_crosstool") -cc_crosstool(name = "crosstool") diff --git a/bazel/WORKSPACE.darwin b/bazel/WORKSPACE.darwin deleted file mode 100644 index ad08cb5..0000000 --- a/bazel/WORKSPACE.darwin +++ /dev/null @@ -1,13 +0,0 @@ -# Use libusb from MacPorts. -new_local_repository( - name = "libusb", - path = "/opt/local/include/", - build_file_content = """ -cc_library( - name = "headers", - includes = ["."], - hdrs = ["libusb-1.0/libusb.h"], - visibility = ["//visibility:public"], -) -""" -) diff --git a/bazel/WORKSPACE.linux b/bazel/WORKSPACE.linux deleted file mode 100644 index c33e185..0000000 --- a/bazel/WORKSPACE.linux +++ /dev/null @@ -1,12 +0,0 @@ -new_local_repository( - name = "libusb", - path = "/usr/include/", - build_file_content = """ -cc_library( - name = "headers", - includes = ["."], - hdrs = ["libusb-1.0/libusb.h"], - visibility = ["//visibility:public"], -) -""" -) diff --git a/bazel/WORKSPACE.windows b/bazel/WORKSPACE.windows deleted file mode 100644 index 9994174..0000000 --- a/bazel/WORKSPACE.windows +++ /dev/null @@ -1,21 +0,0 @@ -# For Windows, extract libusb archive to ../libusb-1.0.22 -# Unfortunately, can't use http_archive here, as bazel doesn't support 7z. -# https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.7z -new_local_repository( - name = "libusb", - path = "../libusb-1.0.22", - build_file_content = """ -cc_library( - name = "headers", - includes = ["include"], - hdrs = ["include/libusb-1.0/libusb.h"], - visibility = ["//visibility:public"], -) -cc_import( - name = "shared", - interface_library = "MS64/dll/libusb-1.0.lib", - shared_library = "MS64/dll/libusb-1.0.dll", - visibility = ["//visibility:public"], -) -""" -) \ No newline at end of file diff --git a/build.bat b/build.bat index af53532..8841c43 100644 --- a/build.bat +++ b/build.bat @@ -15,7 +15,12 @@ echo off setlocal -type bazel\WORKSPACE bazel\WORKSPACE.windows > WORKSPACE 2>NUL +if not defined PYTHON set PYTHON=python +set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools +set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC +call "%BAZEL_VC%\Auxiliary\Build\vcvars64.bat" + +for /f %%i in ('%PYTHON% -c "import sys;print(sys.executable)"') do set PYTHON_BIN_PATH=%%i set THROTTLED=0 set COMPILATION_MODE=opt @@ -40,38 +45,30 @@ for /f %%i in ('bazel info %BAZEL_INFO_FLAGS% output_path') do set "BAZEL_OUTPUT set BAZEL_OUTPUT_PATH=%BAZEL_OUTPUT_PATH:/=\% set BAZEL_OUT_DIR=%BAZEL_OUTPUT_PATH%\%CPU%-%COMPILATION_MODE%\bin +:: Supported EdgeTPU devices, one of: usb, pci or all. +set EDGETPU_BUS=all -set TARGET=//tflite/public:edgetpu_direct_usb.dll -set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools -set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC +set TARGET=//tflite/public:edgetpu_direct_%EDGETPU_BUS%.dll set BAZEL_BUILD_FLAGS= ^ --experimental_repo_remote_exec ^ --compilation_mode %COMPILATION_MODE% ^ --define darwinn_portable=1 ^ ---copt=/DSTRIP_LOG=1 ^ ---copt=/DABSL_FLAGS_STRIP_NAMES ^ --copt=/D_HAS_DEPRECATED_RESULT_OF ^ ---copt=/D_HAS_DEPRECATED_ADAPTOR_TYPEDEFS ^ ---copt=/GR- ^ ---copt=/DWIN32_LEAN_AND_MEAN ^ ---copt=/D_WINSOCKAPI_ ^ --copt=/std:c++latest -call "%BAZEL_VC%\Auxiliary\Build\vcvars64.bat" - bazel build %BAZEL_BUILD_FLAGS% %LEFTOVER_ARGS% %TARGET% md %OUT_DIR%\direct\%CPU% -copy %BAZEL_OUT_DIR%\tflite\public\edgetpu_direct_usb.dll ^ +copy %BAZEL_OUT_DIR%\tflite\public\edgetpu_direct_%EDGETPU_BUS%.dll ^ %OUT_DIR%\direct\%CPU%\ -python %~dp0\rename_library.py ^ - --input_dll %OUT_DIR%\direct\%CPU%\edgetpu_direct_usb.dll ^ +%PYTHON_BIN_PATH% %~dp0\rename_library.py ^ + --input_dll %OUT_DIR%\direct\%CPU%\edgetpu_direct_%EDGETPU_BUS%.dll ^ --output_dll %OUT_DIR%\direct\%CPU%\edgetpu.dll set BAZEL_BUILD_FLAGS=%BAZEL_BUILD_FLAGS% --copt=/DTHROTTLE_EDGE_TPU bazel build %BAZEL_BUILD_FLAGS% %LEFTOVER_ARGS% %TARGET% md %OUT_DIR%\throttled\%CPU% -copy %BAZEL_OUT_DIR%\tflite\public\edgetpu_direct_usb.dll ^ +copy %BAZEL_OUT_DIR%\tflite\public\edgetpu_direct_%EDGETPU_BUS%.dll ^ %OUT_DIR%\throttled\%CPU%\ -python %~dp0\rename_library.py ^ - --input_dll %OUT_DIR%\throttled\%CPU%\edgetpu_direct_usb.dll ^ - --output_dll %OUT_DIR%\throttled\%CPU%\edgetpu.dll \ No newline at end of file +%PYTHON_BIN_PATH% %~dp0\rename_library.py ^ + --input_dll %OUT_DIR%\throttled\%CPU%\edgetpu_direct_%EDGETPU_BUS%.dll ^ + --output_dll %OUT_DIR%\throttled\%CPU%\edgetpu.dll diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..c3b8b9d --- /dev/null +++ b/debian/changelog @@ -0,0 +1,6 @@ +libedgetpu (14.0) stable; urgency=medium + + * Initial release. + + -- Coral Wed, 25 Mar 2020 14:25:24 -0700 + diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +10 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..d019540 --- /dev/null +++ b/debian/control @@ -0,0 +1,46 @@ +Source: libedgetpu +Maintainer: Coral +Section: misc +Priority: optional +Build-Depends: debhelper (>= 9) +Standards-Version: 3.9.6 +Homepage: https://coral.ai/ + +Package: libedgetpu1-std +Provides: libedgetpu1 (= ${binary:Version}) +Conflicts: libedgetpu1 +Section: misc +Priority: optional +Architecture: any +Multi-Arch: same +Depends: libc6, + libgcc1, + libstdc++6, + libusb-1.0-0, + ${misc:Depends} +Description: Support library for Edge TPU + Support library (standard speed) for the Edge TPU + +Package: libedgetpu1-max +Provides: libedgetpu1 (= ${binary:Version}) +Conflicts: libedgetpu1 +Section: misc +Priority: optional +Architecture: any +Multi-Arch: same +Depends: libc6, + libgcc1, + libstdc++6, + libusb-1.0-0, + ${misc:Depends} +Description: Support library for Edge TPU + Support library (max speed) for the Edge TPU + +Package:libedgetpu-dev +Section: libdevel +Priority: optional +Architecture: any +Depends: libedgetpu1-std (= ${binary:Version}) | libedgetpu1 (= ${binary:Version}), + ${misc:Depends} +Description: Development files for libedgetpu + This package contains C++ Header files for libedgetpu.so diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..699aad1 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,6 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Source: https://github.com/google-coral/libedgetpu + +Files: * +Copyright: Copyright 2018 Google, LLC +License: Apache-2.0 diff --git a/debian/edgetpu-accelerator.rules b/debian/edgetpu-accelerator.rules new file mode 100644 index 0000000..60e034c --- /dev/null +++ b/debian/edgetpu-accelerator.rules @@ -0,0 +1,2 @@ +SUBSYSTEM=="usb",ATTRS{idVendor}=="1a6e",GROUP="plugdev" +SUBSYSTEM=="usb",ATTRS{idVendor}=="18d1",GROUP="plugdev" diff --git a/debian/libedgetpu-dev.install b/debian/libedgetpu-dev.install new file mode 100644 index 0000000..2f5c2e7 --- /dev/null +++ b/debian/libedgetpu-dev.install @@ -0,0 +1,2 @@ +tflite/public/edgetpu.h /usr/include +tflite/public/edgetpu_c.h /usr/include diff --git a/debian/libedgetpu1-max.lintian-overrides b/debian/libedgetpu1-max.lintian-overrides new file mode 100644 index 0000000..3591ed2 --- /dev/null +++ b/debian/libedgetpu1-max.lintian-overrides @@ -0,0 +1,4 @@ +# We provide two conflicting package variants with the same soname inside. +libedgetpu1-max: package-name-doesnt-match-sonames libedgetpu1 +libedgetpu1-max: missing-debconf-dependency-for-preinst +libedgetpu1-max: too-long-short-description-in-templates libedgetpu/accepted-eula diff --git a/debian/libedgetpu1-max.preinst b/debian/libedgetpu1-max.preinst new file mode 100644 index 0000000..5a7a763 --- /dev/null +++ b/debian/libedgetpu1-max.preinst @@ -0,0 +1,24 @@ +#!/bin/sh -e + +. /usr/share/debconf/confmodule + +db_version 2.0 + +db_get libedgetpu/accepted-eula +if [ "$RET" = "true" ]; then + exit 0 # already accepted +fi + +db_fset libedgetpu/accepted-eula seen false +db_input critical libedgetpu/accepted-eula +db_go || true + +db_get libedgetpu/accepted-eula +if [ "$RET" = "true" ]; then + exit 0 # accepted +fi + +db_input critical libedgetpu/error-eula +db_go || true + +exit 1 # not accepted diff --git a/debian/libedgetpu1-max.templates b/debian/libedgetpu1-max.templates new file mode 100644 index 0000000..6c276be --- /dev/null +++ b/debian/libedgetpu1-max.templates @@ -0,0 +1,21 @@ +Template: libedgetpu/accepted-eula +Type: boolean +Default: false +Description: Continue to install the Edge TPU runtime that runs at the maximum operating frequency? + You're about to install the Edge TPU runtime that runs at the maximum operating frequency. + . + Warning: If you're using the Coral USB Accelerator, it may heat up during operation, depending + on the computation workloads and operating frequency. Touching the metal part of the USB + Accelerator after it has been operating for an extended period of time may lead to discomfort + and/or skin burns. As such, if you install the Edge TPU runtime using the maximum operating + frequency, the USB Accelerator should be operated at an ambient temperature of 25°C or less. + (If you instead install the Edge TPU runtime using the reduced operating frequency, then the + device is intended to safely operate at an ambient temperature of 35°C or less.) + . + Google does not accept any responsibility for any loss or damage if the device is operated + outside of the recommended ambient temperature range. + +Template: libedgetpu/error-eula +Type: error +Description: Install aborted. + For help setting up your device, see g.co/coral/setup. diff --git a/debian/libedgetpu1-max.triggers b/debian/libedgetpu1-max.triggers new file mode 100644 index 0000000..dd86603 --- /dev/null +++ b/debian/libedgetpu1-max.triggers @@ -0,0 +1 @@ +activate-noawait ldconfig diff --git a/debian/libedgetpu1-max.udev b/debian/libedgetpu1-max.udev new file mode 120000 index 0000000..2500f6f --- /dev/null +++ b/debian/libedgetpu1-max.udev @@ -0,0 +1 @@ +edgetpu-accelerator.rules \ No newline at end of file diff --git a/debian/libedgetpu1-std.lintian-overrides b/debian/libedgetpu1-std.lintian-overrides new file mode 100644 index 0000000..cfa2624 --- /dev/null +++ b/debian/libedgetpu1-std.lintian-overrides @@ -0,0 +1,2 @@ +# We provide two conflicting package variants with the same soname inside. +libedgetpu1-std: package-name-doesnt-match-sonames libedgetpu1 diff --git a/debian/libedgetpu1-std.triggers b/debian/libedgetpu1-std.triggers new file mode 100644 index 0000000..dd86603 --- /dev/null +++ b/debian/libedgetpu1-std.triggers @@ -0,0 +1 @@ +activate-noawait ldconfig diff --git a/debian/libedgetpu1-std.udev b/debian/libedgetpu1-std.udev new file mode 120000 index 0000000..2500f6f --- /dev/null +++ b/debian/libedgetpu1-std.udev @@ -0,0 +1 @@ +edgetpu-accelerator.rules \ No newline at end of file diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..ed125e6 --- /dev/null +++ b/debian/rules @@ -0,0 +1,46 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +# export DH_VERBOSE=1 +FILENAME := libedgetpu.so.1.0 +SONAME := libedgetpu.so.1 +LIB_DEV := debian/libedgetpu-dev/usr/lib/$(DEB_HOST_GNU_TYPE) +LIB_STD := debian/libedgetpu1-std/usr/lib/$(DEB_HOST_GNU_TYPE) +LIB_MAX := debian/libedgetpu1-max/usr/lib/$(DEB_HOST_GNU_TYPE) + +ifeq ($(DEB_TARGET_ARCH),armhf) + CPU := armv7a +else ifeq ($(DEB_TARGET_ARCH),arm64) + CPU := aarch64 +else ifeq ($(DEB_TARGET_ARCH),amd64) + CPU := k8 +endif + +build: + +%: + dh $@ + +override_dh_auto_install: + mkdir -p $(LIB_DEV) + ln -fs $(FILENAME) $(LIB_DEV)/libedgetpu.so + + mkdir -p $(LIB_STD) + cp -f out/throttled/$(CPU)/$(FILENAME) $(LIB_STD)/$(FILENAME) + ln -fs $(FILENAME) $(LIB_STD)/$(SONAME) + + mkdir -p $(LIB_MAX) + cp -f out/direct/$(CPU)/$(FILENAME) $(LIB_MAX)/$(FILENAME) + ln -fs $(FILENAME) $(LIB_MAX)/$(SONAME) + +# Skip auto build and auto clean. +override_dh_auto_clean: +override_dh_auto_build: + +# Skip .so post processing. +override_dh_strip: +override_dh_shlibdeps: + +# Skip tests. +override_dh_auto_test: diff --git a/docker/Dockerfile b/docker/Dockerfile index 1c5fe0e..00cb8d9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,16 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ARG IMAGE FROM ${IMAGE} @@ -10,6 +23,8 @@ RUN apt-get update && apt-get install -y \ python \ python-future \ python-numpy \ + python3-all \ + python3-numpy \ build-essential \ crossbuild-essential-armhf \ crossbuild-essential-arm64 \ diff --git a/docker/Dockerfile.windows b/docker/Dockerfile.windows index 7f417c6..89bf666 100644 --- a/docker/Dockerfile.windows +++ b/docker/Dockerfile.windows @@ -1,3 +1,16 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. FROM mcr.microsoft.com/windows/servercore:1903 SHELL ["powershell", "-command"] diff --git a/docker/update_sources.sh b/docker/update_sources.sh index 50fc6cb..19ddaa1 100755 --- a/docker/update_sources.sh +++ b/docker/update_sources.sh @@ -1,4 +1,17 @@ #!/bin/bash +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. . /etc/os-release [[ "${NAME}" == "Ubuntu" ]] || exit 0 diff --git a/driver/BUILD b/driver/BUILD index 49d6e9d..430d8af 100644 --- a/driver/BUILD +++ b/driver/BUILD @@ -15,6 +15,8 @@ # Description: # Portable DarwiNN driver implementation. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package( default_visibility = ["//visibility:public"], ) @@ -24,7 +26,7 @@ licenses(["notice"]) exports_files(["libdarwinn_driver.lds"]) # Utility functions. -cc_library( +libedgetpu_cc_library( name = "util", hdrs = [ "bitfield.h", @@ -35,7 +37,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "allocator", srcs = [ "aligned_allocator.cc", @@ -52,7 +54,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "device_buffer", srcs = ["device_buffer.cc"], hdrs = ["device_buffer.h"], @@ -61,7 +63,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "executable_util", srcs = ["executable_util.cc"], hdrs = ["executable_util.h"], @@ -73,7 +75,7 @@ cc_library( ) # Driver Factory. -cc_library( +libedgetpu_cc_library( name = "driver_factory", srcs = ["driver_factory.cc"] + select({ "//:windows": ["driver_factory_windows.cc"], @@ -91,13 +93,15 @@ cc_library( "//api:driver_options_fbs", "//api:driver_options_helper", "//driver/config", + "//driver/kernel:gasket_ioctl", "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", ], ) -cc_library( +libedgetpu_cc_library( name = "driver_helper", srcs = ["driver_helper.cc"], hdrs = ["driver_helper.h"], @@ -119,7 +123,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "instruction_buffers", srcs = ["instruction_buffers.cc"], hdrs = ["instruction_buffers.h"], @@ -134,7 +138,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "run_controller", srcs = ["run_controller.cc"], hdrs = ["run_controller.h"], @@ -147,7 +151,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "scalar_core_controller", srcs = ["scalar_core_controller.cc"], hdrs = ["scalar_core_controller.h"], @@ -161,7 +165,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "hardware_structures", hdrs = ["hardware_structures.h"], deps = [ @@ -170,7 +174,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "driver", srcs = ["driver.cc"], hdrs = ["driver.h"], @@ -189,7 +193,7 @@ cc_library( "//api:request", "//api:telemeter_interface", "//driver/memory:dma_direction", - "//driver/time_stamper", + "//driver_shared/time_stamper", "//executable:executable_fbs", "//port", "//port:blocking_counter", @@ -200,7 +204,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "default_telemeter", hdrs = ["default_telemeter.h"], deps = [ @@ -208,7 +212,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "mmio_driver", srcs = ["mmio_driver.cc"], hdrs = ["mmio_driver.h"], @@ -241,17 +245,18 @@ cc_library( "//driver/memory:mmu_mapper", "//driver/mmio:host_queue", "//driver/registers", - "//driver/time_stamper", - "//driver/time_stamper:driver_time_stamper", + "//driver_shared/time_stamper", + "//driver_shared/time_stamper:driver_time_stamper", "//executable:executable_fbs", "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", "//port:tracing", ], ) -cc_library( +libedgetpu_cc_library( name = "dma_info", srcs = ["dma_info.cc"], hdrs = ["dma_info.h"], @@ -261,7 +266,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "device_buffer_mapper", srcs = ["device_buffer_mapper.cc"], hdrs = ["device_buffer_mapper.h"], @@ -277,7 +282,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "dma_info_extractor", srcs = ["dma_info_extractor.cc"], hdrs = ["dma_info_extractor.h"], @@ -291,7 +296,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "dma_scheduler", hdrs = ["dma_scheduler.h"], deps = [ @@ -302,7 +307,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "single_queue_dma_scheduler", srcs = ["single_queue_dma_scheduler.cc"], hdrs = ["single_queue_dma_scheduler.h"], @@ -319,7 +324,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "real_time_dma_scheduler", srcs = ["real_time_dma_scheduler.cc"], hdrs = ["real_time_dma_scheduler.h"], @@ -334,14 +339,14 @@ cc_library( "//api:package_reference", "//api:timing", "//api:watchdog", - "//driver/time_stamper", + "//driver_shared/time_stamper", "//port", "//port:std_mutex_lock", "//port:thread_annotations", ], ) -cc_library( +libedgetpu_cc_library( name = "dma_chunker", srcs = ["dma_chunker.cc"], hdrs = ["dma_chunker.h"], @@ -351,13 +356,13 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "top_level_handler", hdrs = ["top_level_handler.h"], deps = ["//port"], ) -cc_library( +libedgetpu_cc_library( name = "package_registry", srcs = ["package_registry.cc"], hdrs = ["package_registry.h"], @@ -382,7 +387,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "package_verifier", srcs = ["package_verifier.cc"], hdrs = ["package_verifier.h"], @@ -392,7 +397,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "tpu_request", hdrs = ["tpu_request.h"], deps = [ @@ -404,7 +409,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "single_tpu_request", srcs = ["single_tpu_request.cc"], hdrs = ["single_tpu_request.h"], @@ -432,15 +437,15 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "request", srcs = ["request.cc"], hdrs = ["request.h"], deps = [ ":tpu_request", "//api:request", - "//driver/time_stamper", - "//driver/time_stamper:driver_time_stamper_factory", + "//driver_shared/time_stamper", + "//driver_shared/time_stamper:driver_time_stamper_factory", "//port", "//port:std_mutex_lock", "//port:thread_annotations", diff --git a/driver/beagle/BUILD b/driver/beagle/BUILD index b38a989..f905538 100644 --- a/driver/beagle/BUILD +++ b/driver/beagle/BUILD @@ -15,169 +15,197 @@ # Description: # Beagle Specific functionality. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_binary", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) -licenses(["notice"]) +# Google Owned Code +licenses(["unencumbered"]) + +BEAGLE_USB_DRIVER_DEPS = [ + ":beagle_top_level_handler", + ":beagle_top_level_interrupt_manager", + "@com_google_absl//absl/strings", + "//api:chip", + "//api:driver", + "//api:driver_options_fbs", + "//driver:allocator", + "//driver:driver_factory", + "//driver:package_registry", + "//driver:package_verifier", + "//driver:run_controller", + "//driver:scalar_core_controller", + "//driver/config/beagle:beagle_chip_config", + "//driver/interrupt:grouped_interrupt_controller", + "//driver/interrupt:interrupt_controller", + "//driver/interrupt:interrupt_controller_interface", + "//driver/memory:null_dram_allocator", + "//driver_shared/time_stamper:driver_time_stamper", + "//driver/usb:usb_device_interface", + "//driver/usb:usb_driver", + "//driver/usb:usb_ml_commands", + "//driver/usb:usb_registers", + "//port", + "//port:tracing", +] # Provides Beagle USB Driver for internal (=no_external_release) usage. # libUSB is statically linked in this version. -cc_library( +libedgetpu_cc_library( name = "beagle_usb_driver_provider_no_external_release", srcs = ["beagle_usb_driver_provider.cc"], - deps = [ - ":beagle_top_level_handler", - ":beagle_top_level_interrupt_manager", - "@com_google_absl//absl/strings", - "//api:chip", - "//api:driver", - "//api:driver_options_fbs", - "//driver:allocator", - "//driver:driver_factory", - "//driver:package_registry", - "//driver:package_verifier", - "//driver:run_controller", - "//driver:scalar_core_controller", - "//driver/config/beagle:beagle_chip_config", - "//driver/interrupt:grouped_interrupt_controller", - "//driver/interrupt:interrupt_controller", - "//driver/interrupt:interrupt_controller_interface", - "//driver/memory:null_dram_allocator", - "//driver/time_stamper:driver_time_stamper", + deps = BEAGLE_USB_DRIVER_DEPS + [ "//driver/usb:local_usb_device_no_external_release", - "//driver/usb:usb_device_interface", - "//driver/usb:usb_driver", - "//driver/usb:usb_ml_commands", - "//driver/usb:usb_registers", - "//port", - "//port:tracing", ], alwayslink = 1, ) # Provides Beagle USB Driver. # libUSB is dynamically linked in this version. -cc_library( +libedgetpu_cc_library( name = "beagle_usb_driver_provider", srcs = ["beagle_usb_driver_provider.cc"], - deps = [ + deps = BEAGLE_USB_DRIVER_DEPS + [ + "//driver/usb:local_usb_device", + ], + alwayslink = 1, +) + +BEAGLE_PCI_DRIVER_PROVIDER_DEPS = [ + ":beagle_kernel_top_level_handler", + "//api:chip", + "//api:driver", + "//driver:allocator", + "//driver:driver_factory", + "//driver:hardware_structures", + "//driver:mmio_driver", + "//driver:package_registry", + "//driver:package_verifier", + "//driver:run_controller", + "//driver:scalar_core_controller", + "//driver/config", + "//driver/config/beagle:beagle_chip_config", + "//driver/interrupt:dummy_interrupt_controller", + "//driver/interrupt:grouped_interrupt_controller", + "//driver/interrupt:interrupt_handler", + "//driver/interrupt:top_level_interrupt_manager", + "//driver/kernel:kernel_coherent_allocator", + "//driver/kernel:kernel_event_handler", + "//driver/kernel:kernel_interrupt_handler", + "//driver/kernel:kernel_mmu_mapper", + "//driver/kernel:kernel_registers", + "//driver/kernel:kernel_wire_interrupt_handler", + "//driver/memory:dual_address_space", + "//driver/memory:null_dram_allocator", + "//driver/mmio:host_queue", + "//driver_shared/time_stamper:driver_time_stamper", + "//port", + "//port:fileio", +] + +libedgetpu_cc_library( + name = "beagle_pci_driver_provider", + srcs = ["beagle_pci_driver_provider.cc"], + hdrs = ["beagle_pci_driver_provider.h"], + deps = BEAGLE_PCI_DRIVER_PROVIDER_DEPS, +) + +# Provides Beagle PCI Driver for Linux. +libedgetpu_cc_library( + name = "beagle_pci_driver_provider_linux", + srcs = ["beagle_pci_driver_provider_linux.cc"], + deps = BEAGLE_PCI_DRIVER_PROVIDER_DEPS + [ + "//driver/beagle:beagle_pci_driver_provider", + "//driver/kernel/linux:kernel_coherent_allocator_linux", + "//driver/kernel/linux:kernel_event_handler_linux", + "//driver/kernel/linux:kernel_registers_linux", + ], + alwayslink = 1, +) + +# Provides Beagle PCI Driver for Windows. +libedgetpu_cc_library( + name = "beagle_pci_driver_provider_windows", + srcs = ["beagle_pci_driver_provider_windows.cc"], + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = BEAGLE_PCI_DRIVER_PROVIDER_DEPS + [ + "//driver/beagle:beagle_pci_driver_provider", + "//driver/kernel/windows:kernel_coherent_allocator_windows", + "//driver/kernel/windows:kernel_event_handler_windows", + "//driver/kernel/windows:kernel_registers_windows", + ], + alwayslink = 1, +) + +# Provides Beagle USB/PCI Driver for Linux. Used by an Android side script to +# populate dependencies for a unified Beagle provider in Beagle NNAPI HAL. +libedgetpu_cc_library( + name = "beagle_all_driver_provider_linux", + srcs = [ + "beagle_pci_driver_provider.cc", + "beagle_pci_driver_provider_linux.cc", + "beagle_usb_driver_provider.cc", + ], + hdrs = ["beagle_pci_driver_provider.h"], + deps = BEAGLE_PCI_DRIVER_PROVIDER_DEPS + [ ":beagle_top_level_handler", ":beagle_top_level_interrupt_manager", "@com_google_absl//absl/strings", - "//api:chip", - "//api:driver", "//api:driver_options_fbs", - "//driver:allocator", - "//driver:driver_factory", - "//driver:package_registry", - "//driver:package_verifier", - "//driver:run_controller", - "//driver:scalar_core_controller", - "//driver/config/beagle:beagle_chip_config", - "//driver/interrupt:grouped_interrupt_controller", "//driver/interrupt:interrupt_controller", "//driver/interrupt:interrupt_controller_interface", - "//driver/memory:null_dram_allocator", - "//driver/time_stamper:driver_time_stamper", + "//driver/kernel/linux:kernel_coherent_allocator_linux", + "//driver/kernel/linux:kernel_event_handler_linux", + "//driver/kernel/linux:kernel_registers_linux", "//driver/usb:local_usb_device", "//driver/usb:usb_device_interface", "//driver/usb:usb_driver", "//driver/usb:usb_ml_commands", "//driver/usb:usb_registers", - "//port", "//port:tracing", ], alwayslink = 1, ) -# Provides Beagle PCI Driver. -cc_library( - name = "beagle_pci_driver_provider", - srcs = ["beagle_pci_driver_provider.cc"], - deps = [ - ":beagle_kernel_top_level_handler", - "//api:chip", - "//api:driver", - "//driver:allocator", - "//driver:driver_factory", - "//driver:hardware_structures", - "//driver:mmio_driver", - "//driver:package_registry", - "//driver:package_verifier", - "//driver:run_controller", - "//driver:scalar_core_controller", - "//driver/config", - "//driver/config/beagle:beagle_chip_config", - "//driver/interrupt:dummy_interrupt_controller", - "//driver/interrupt:grouped_interrupt_controller", - "//driver/interrupt:interrupt_handler", - "//driver/interrupt:top_level_interrupt_manager", - "//driver/kernel:kernel_coherent_allocator", - "//driver/kernel:kernel_interrupt_handler", - "//driver/kernel:kernel_mmu_mapper", - "//driver/kernel:kernel_registers", - "//driver/kernel:kernel_wire_interrupt_handler", - "//driver/memory:dual_address_space", - "//driver/memory:null_dram_allocator", - "//driver/mmio:host_queue", - "//driver/time_stamper:driver_time_stamper", - "//port", - ], - alwayslink = 1, -) - -# Provides Beagle USB/PCI Driver. Used by an Android side script to -# populate dependencies for a unified Beagle provider in Beagle NNAPI HAL. -cc_library( - name = "beagle_all_driver_provider", +# Provides Beagle USB/PCI Driver for Windows. +libedgetpu_cc_library( + name = "beagle_all_driver_provider_windows", srcs = [ "beagle_pci_driver_provider.cc", + "beagle_pci_driver_provider_windows.cc", "beagle_usb_driver_provider.cc", ], - deps = [ - ":beagle_kernel_top_level_handler", + hdrs = ["beagle_pci_driver_provider.h"], + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = BEAGLE_PCI_DRIVER_PROVIDER_DEPS + [ ":beagle_top_level_handler", ":beagle_top_level_interrupt_manager", "@com_google_absl//absl/strings", - "//api:chip", - "//api:driver", "//api:driver_options_fbs", - "//driver:allocator", - "//driver:driver_factory", - "//driver:hardware_structures", - "//driver:mmio_driver", - "//driver:package_registry", - "//driver:package_verifier", - "//driver:run_controller", - "//driver:scalar_core_controller", - "//driver/config", - "//driver/config/beagle:beagle_chip_config", - "//driver/interrupt:dummy_interrupt_controller", - "//driver/interrupt:grouped_interrupt_controller", "//driver/interrupt:interrupt_controller", "//driver/interrupt:interrupt_controller_interface", - "//driver/interrupt:interrupt_handler", - "//driver/interrupt:top_level_interrupt_manager", - "//driver/kernel:kernel_coherent_allocator", - "//driver/kernel:kernel_interrupt_handler", - "//driver/kernel:kernel_mmu_mapper", - "//driver/kernel:kernel_registers", - "//driver/kernel:kernel_wire_interrupt_handler", - "//driver/memory:dual_address_space", - "//driver/memory:null_dram_allocator", - "//driver/mmio:host_queue", - "//driver/time_stamper:driver_time_stamper", + "//driver/kernel/windows:kernel_coherent_allocator_windows", + "//driver/kernel/windows:kernel_event_handler_windows", + "//driver/kernel/windows:kernel_registers_windows", "//driver/usb:local_usb_device", "//driver/usb:usb_device_interface", "//driver/usb:usb_driver", "//driver/usb:usb_ml_commands", "//driver/usb:usb_registers", - "//port", "//port:tracing", ], alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "beagle_top_level_interrupt_manager", srcs = ["beagle_top_level_interrupt_manager.cc"], hdrs = ["beagle_top_level_interrupt_manager.h"], @@ -187,10 +215,11 @@ cc_library( "//driver/interrupt:top_level_interrupt_manager", "//driver/registers", "//port", + "//port:fileio", ], ) -cc_library( +libedgetpu_cc_library( name = "beagle_top_level_handler", srcs = ["beagle_top_level_handler.cc"], hdrs = ["beagle_top_level_handler.h"], @@ -200,15 +229,17 @@ cc_library( "//driver/config", "//driver/registers", "//port", + "//port:fileio", ], ) -cc_library( +libedgetpu_cc_library( name = "beagle_ioctl", hdrs = ["beagle_ioctl.h"], + deps = ["//driver/kernel:gasket_ioctl"], ) -cc_library( +libedgetpu_cc_library( name = "beagle_kernel_top_level_handler", srcs = ["beagle_kernel_top_level_handler.cc"], hdrs = ["beagle_kernel_top_level_handler.h"], @@ -217,13 +248,14 @@ cc_library( "//api:driver_options_fbs", "//driver:top_level_handler", "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", ], ) # Standalone shared library that can be linked into a binary built elsewhere. -cc_binary( +libedgetpu_cc_binary( name = "libbeagle-usb.so", # Setting DT_SONAME for the result .so. @@ -241,7 +273,7 @@ cc_binary( ) # Standalone shared library that can be linked into a binary built elsewhere. -cc_binary( +libedgetpu_cc_binary( name = "libbeagle-pci.so", # Setting DT_SONAME for the result .so. diff --git a/driver/beagle/beagle_ioctl.h b/driver/beagle/beagle_ioctl.h index 5f2e44e..eb5eeba 100644 --- a/driver/beagle/beagle_ioctl.h +++ b/driver/beagle/beagle_ioctl.h @@ -18,10 +18,7 @@ #ifndef __APEX_IOCTL_H__ #define __APEX_IOCTL_H__ -#include -#ifndef __KERNEL__ -#include -#endif +#include "driver/kernel/gasket_ioctl.h" /* Clock Gating ioctl. */ struct apex_gate_clock_ioctl { diff --git a/driver/beagle/beagle_kernel_top_level_handler.cc b/driver/beagle/beagle_kernel_top_level_handler.cc index fd0df0d..2a020be 100644 --- a/driver/beagle/beagle_kernel_top_level_handler.cc +++ b/driver/beagle/beagle_kernel_top_level_handler.cc @@ -14,11 +14,6 @@ #include "driver/beagle/beagle_kernel_top_level_handler.h" -#include -#include -#include -#include - #include "api/driver_options_generated.h" #include "driver/beagle/beagle_ioctl.h" #include "port/errors.h" @@ -80,7 +75,7 @@ util::Status BeagleKernelTopLevelHandler::EnableSoftwareClockGate() { util::Status BeagleKernelTopLevelHandler::Open() { StdMutexLock lock(&mutex_); - if (fd_ != -1) { + if (fd_ != INVALID_FD_VALUE) { return util::FailedPreconditionError("Device already open."); } @@ -95,12 +90,12 @@ util::Status BeagleKernelTopLevelHandler::Open() { util::Status BeagleKernelTopLevelHandler::Close() { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } close(fd_); - fd_ = -1; + fd_ = INVALID_FD_VALUE; return util::Status(); // OK } diff --git a/driver/beagle/beagle_kernel_top_level_handler.h b/driver/beagle/beagle_kernel_top_level_handler.h index 085cd01..3c6a19a 100644 --- a/driver/beagle/beagle_kernel_top_level_handler.h +++ b/driver/beagle/beagle_kernel_top_level_handler.h @@ -19,6 +19,7 @@ #include "api/driver_options_generated.h" #include "driver/top_level_handler.h" +#include "port/fileio.h" #include "port/status.h" #include "port/thread_annotations.h" @@ -45,7 +46,7 @@ class BeagleKernelTopLevelHandler : public TopLevelHandler { const std::string device_path_; // File descriptor of the opened device. - int fd_ GUARDED_BY(mutex_){-1}; + FileDescriptor fd_ GUARDED_BY(mutex_){INVALID_FD_VALUE}; // Mutex that guards fd_. std::mutex mutex_; diff --git a/driver/beagle/beagle_pci_driver_provider.cc b/driver/beagle/beagle_pci_driver_provider.cc index a080221..14f2ad5 100644 --- a/driver/beagle/beagle_pci_driver_provider.cc +++ b/driver/beagle/beagle_pci_driver_provider.cc @@ -12,27 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "driver/beagle/beagle_pci_driver_provider.h" + #include #include #include #include -#include "api/chip.h" -#include "api/driver.h" #include "driver/aligned_allocator.h" #include "driver/beagle/beagle_kernel_top_level_handler.h" #include "driver/config/beagle/beagle_chip_config.h" #include "driver/config/chip_structures.h" -#include "driver/driver_factory.h" #include "driver/hardware_structures.h" #include "driver/interrupt/dummy_interrupt_controller.h" #include "driver/interrupt/grouped_interrupt_controller.h" #include "driver/interrupt/interrupt_handler.h" #include "driver/interrupt/top_level_interrupt_manager.h" -#include "driver/kernel/kernel_coherent_allocator.h" -#include "driver/kernel/kernel_interrupt_handler.h" #include "driver/kernel/kernel_mmu_mapper.h" -#include "driver/kernel/kernel_registers.h" #include "driver/kernel/kernel_wire_interrupt_handler.h" #include "driver/memory/dual_address_space.h" #include "driver/memory/null_dram_allocator.h" @@ -42,9 +38,7 @@ #include "driver/package_verifier.h" #include "driver/run_controller.h" #include "driver/scalar_core_controller.h" -#include "driver/time_stamper/driver_time_stamper.h" -#include "port/integral_types.h" -#include "port/ptr_util.h" +#include "driver_shared/time_stamper/driver_time_stamper.h" namespace platforms { namespace darwinn { @@ -56,25 +50,6 @@ using platforms::darwinn::api::Device; } // namespace -class BeaglePciDriverProvider : public DriverProvider { - public: - static std::unique_ptr CreateDriverProvider() { - return gtl::WrapUnique(new BeaglePciDriverProvider()); - } - - ~BeaglePciDriverProvider() override = default; - - std::vector Enumerate() override; - bool CanCreate(const Device& device) override; - util::StatusOr> CreateDriver( - const Device& device, const api::DriverOptions& options) override; - - private: - BeaglePciDriverProvider() = default; -}; - -REGISTER_DRIVER_PROVIDER(BeaglePciDriverProvider); - std::vector BeaglePciDriverProvider::Enumerate() { return EnumerateSysfs("apex", Chip::kBeagle, Device::Type::PCI); } @@ -111,10 +86,9 @@ BeaglePciDriverProvider::CreateDriver(const Device& device, {kScalarCoreOffset, kSectionSize}, {kUserHibOffset, kSectionSize}, }; - auto registers = gtl::MakeUnique(device.path, regions, - /*read_only=*/false); - - auto interrupt_handler = gtl::MakeUnique(device.path); + auto registers = + CreateKernelRegisters(device.path, regions, /*read_only=*/false); + auto interrupt_handler = CreateKernelInterruptHandler(device.path); auto top_level_handler = gtl::MakeUnique( device.path, options.performance_expectation()); auto mmu_mapper = gtl::MakeUnique(device.path); @@ -124,7 +98,7 @@ BeaglePciDriverProvider::CreateDriver(const Device& device, config->GetChipStructures().allocation_alignment_bytes; auto allocator = gtl::MakeUnique(allocation_alignment_bytes); - auto coherent_allocator = gtl::MakeUnique( + auto coherent_allocator = CreateKernelCoherentAllocator( device.path, allocation_alignment_bytes, kCoherentAllocatorMaxSizeByte); auto host_queue = gtl::MakeUnique>( @@ -160,7 +134,7 @@ BeaglePciDriverProvider::CreateDriver(const Device& device, MakeExecutableVerifier(flatbuffers::GetString(options.public_key()))); auto executable_registry = gtl::MakeUnique( device.chip, std::move(verifier), dram_allocator.get()); - auto time_stamper = gtl::MakeUnique(); + auto time_stamper = gtl::MakeUnique(); return {gtl::MakeUnique( options, std::move(config), std::move(registers), diff --git a/driver/beagle/beagle_pci_driver_provider.h b/driver/beagle/beagle_pci_driver_provider.h new file mode 100644 index 0000000..9e41c7e --- /dev/null +++ b/driver/beagle/beagle_pci_driver_provider.h @@ -0,0 +1,54 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_BEAGLE_BEAGLE_PCI_DRIVER_PROVIDER_H_ +#define DARWINN_DRIVER_BEAGLE_BEAGLE_PCI_DRIVER_PROVIDER_H_ + +#include "api/chip.h" +#include "api/driver.h" +#include "driver/driver_factory.h" +#include "driver/kernel/kernel_coherent_allocator.h" +#include "driver/kernel/kernel_interrupt_handler.h" +#include "driver/kernel/kernel_registers.h" +#include "port/integral_types.h" +#include "port/ptr_util.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +class BeaglePciDriverProvider : public DriverProvider { + public: + std::vector Enumerate() override; + bool CanCreate(const api::Device& device) override; + util::StatusOr> CreateDriver( + const api::Device& device, const api::DriverOptions& options) override; + + protected: + virtual std::unique_ptr + CreateKernelCoherentAllocator(const std::string& device_path, + int alignment_bytes, size_t size_bytes) = 0; + virtual std::unique_ptr CreateKernelRegisters( + const std::string& device_path, + const std::vector& mmap_region, + bool read_only) = 0; + virtual std::unique_ptr CreateKernelInterruptHandler( + const std::string& device_path) = 0; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_BEAGLE_BEAGLE_PCI_DRIVER_PROVIDER_H_ diff --git a/driver/beagle/beagle_pci_driver_provider_linux.cc b/driver/beagle/beagle_pci_driver_provider_linux.cc new file mode 100644 index 0000000..dd7c280 --- /dev/null +++ b/driver/beagle/beagle_pci_driver_provider_linux.cc @@ -0,0 +1,63 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/beagle/beagle_pci_driver_provider.h" +#include "driver/kernel/linux/kernel_coherent_allocator_linux.h" +#include "driver/kernel/linux/kernel_event_handler_linux.h" +#include "driver/kernel/linux/kernel_registers_linux.h" +#include "port/integral_types.h" +#include "port/ptr_util.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +class BeaglePciDriverProviderLinux : public BeaglePciDriverProvider { + public: + static std::unique_ptr CreateDriverProvider() { + return gtl::WrapUnique(new BeaglePciDriverProviderLinux()); + } + + protected: + std::unique_ptr CreateKernelCoherentAllocator( + const std::string& device_path, int alignment_bytes, + size_t size_bytes) override { + return gtl::MakeUnique( + device_path, alignment_bytes, size_bytes); + } + + std::unique_ptr CreateKernelRegisters( + const std::string& device_path, + const std::vector& mmap_region, + bool read_only) override { + return gtl::MakeUnique(device_path, mmap_region, + read_only); + } + + std::unique_ptr CreateKernelInterruptHandler( + const std::string& device_path) override { + auto event_handler = gtl::MakeUnique( + device_path, DW_INTERRUPT_COUNT); + return gtl::MakeUnique(std::move(event_handler)); + } + + private: + BeaglePciDriverProviderLinux() = default; +}; + +REGISTER_DRIVER_PROVIDER(BeaglePciDriverProviderLinux); + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/beagle/beagle_pci_driver_provider_windows.cc b/driver/beagle/beagle_pci_driver_provider_windows.cc new file mode 100644 index 0000000..50ca82a --- /dev/null +++ b/driver/beagle/beagle_pci_driver_provider_windows.cc @@ -0,0 +1,64 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/beagle/beagle_pci_driver_provider.h" +#include "driver/kernel/windows/kernel_coherent_allocator_windows.h" +#include "driver/kernel/windows/kernel_event_handler_windows.h" +#include "driver/kernel/windows/kernel_registers_windows.h" +#include "port/integral_types.h" +#include "port/ptr_util.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +class BeaglePciDriverProviderWindows : public BeaglePciDriverProvider { + public: + static std::unique_ptr CreateDriverProvider() { + return gtl::WrapUnique( + new BeaglePciDriverProviderWindows()); + } + + protected: + std::unique_ptr CreateKernelCoherentAllocator( + const std::string& device_path, int alignment_bytes, + size_t size_bytes) override { + return gtl::MakeUnique( + device_path, alignment_bytes, size_bytes); + } + + std::unique_ptr CreateKernelRegisters( + const std::string& device_path, + const std::vector& mmap_region, + bool read_only) override { + return gtl::MakeUnique(device_path, mmap_region, + read_only); + } + + std::unique_ptr CreateKernelInterruptHandler( + const std::string& device_path) override { + auto event_handler = gtl::MakeUnique( + device_path, DW_INTERRUPT_COUNT); + return gtl::MakeUnique(std::move(event_handler)); + } + + private: + BeaglePciDriverProviderWindows() = default; +}; + +REGISTER_DRIVER_PROVIDER(BeaglePciDriverProviderWindows); + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/beagle/beagle_usb_driver_provider.cc b/driver/beagle/beagle_usb_driver_provider.cc index 3e8ad1a..e85a46f 100644 --- a/driver/beagle/beagle_usb_driver_provider.cc +++ b/driver/beagle/beagle_usb_driver_provider.cc @@ -40,12 +40,12 @@ #include "driver/package_verifier.h" #include "driver/run_controller.h" #include "driver/scalar_core_controller.h" -#include "driver/time_stamper/driver_time_stamper.h" #include "driver/usb/local_usb_device.h" #include "driver/usb/usb_device_interface.h" #include "driver/usb/usb_driver.h" #include "driver/usb/usb_ml_commands.h" #include "driver/usb/usb_registers.h" +#include "driver_shared/time_stamper/driver_time_stamper.h" #include "port/gflags.h" #include "port/ptr_util.h" #include "port/tracing.h" @@ -350,7 +350,7 @@ BeagleUsbDriverProvider::CreateDriver( auto executable_registry = gtl::MakeUnique( device.chip, std::move(verifier), dram_allocator.get()); - auto time_stamper = gtl::MakeUnique(); + auto time_stamper = gtl::MakeUnique(); // Note that although driver_options is passed into constructor of UsbDriver, // it's USB portion is not used by the driver directly, due to historical diff --git a/driver/config/BUILD b/driver/config/BUILD index c2e79a6..c66677a 100644 --- a/driver/config/BUILD +++ b/driver/config/BUILD @@ -20,6 +20,8 @@ # When compiled in non-google3 environment, pregenerated headers in # ${PROJECT}/... is used. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + DEFAULT_VISIBILITY = [ "//:internal", ] @@ -31,7 +33,7 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # Configuration structures. -cc_library( +libedgetpu_cc_library( name = "config", hdrs = [ "apex_csr_offsets.h", @@ -47,6 +49,7 @@ cc_library( "hib_kernel_csr_offsets.h", "hib_user_csr_offsets.h", "interrupt_csr_offsets.h", + "lpm_csr_offsets.h", "memory_csr_offsets.h", "misc_csr_offsets.h", "msix_csr_offsets.h", @@ -73,7 +76,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "register_constants", hdrs = ["register_constants.h"], deps = [ @@ -81,7 +84,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "scalar_core_csr_offsets_helper", hdrs = ["scalar_core_csr_offsets_helper.h"], deps = [ diff --git a/driver/config/beagle/BUILD b/driver/config/beagle/BUILD index 73f1c37..6ab211c 100644 --- a/driver/config/beagle/BUILD +++ b/driver/config/beagle/BUILD @@ -15,13 +15,15 @@ # Description: # Beagle-specific configuration and CSR Layouts. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) exports_files(glob(["**/*"])) -cc_library( +libedgetpu_cc_library( name = "beagle_config", hdrs = [ "beagle_chip_structures.h", @@ -33,7 +35,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "beagle_chip_config", hdrs = ["beagle_chip_config.h"], deps = [ diff --git a/driver/config/beagle/beagle_chip_structures.h b/driver/config/beagle/beagle_chip_structures.h index 51fb901..7a44025 100644 --- a/driver/config/beagle/beagle_chip_structures.h +++ b/driver/config/beagle/beagle_chip_structures.h @@ -48,6 +48,7 @@ const ChipStructures kBeagleChipStructures = { 0ULL, // NOLINT: support_trace_arch_registers 0ULL, // NOLINT: base_and_bound_unit_size_bytes 1ULL, // NOLINT: number_of_scalar_core_contexts + 0ULL, // NOLINT: support_tile_thread_gcsr_node }; } // namespace config diff --git a/driver/config/beagle/beagle_csr_offsets.h b/driver/config/beagle/beagle_csr_offsets.h index a482450..8984e60 100644 --- a/driver/config/beagle/beagle_csr_offsets.h +++ b/driver/config/beagle/beagle_csr_offsets.h @@ -1480,6 +1480,8 @@ const ScalarCoreCsrOffsets kBeagleScalarCoreCsrOffsets = { kCsrRegisterSpaceInvalidOffset, // UNUSED, outfeed1RunControl kCsrRegisterSpaceInvalidOffset, // UNUSED, contextControl kCsrRegisterSpaceInvalidOffset, // UNUSED, contextStatus + kCsrRegisterSpaceInvalidOffset, // UNUSED, contextSwitchCount + kCsrRegisterSpaceInvalidOffset, // UNUSED, contextSwitchTimeoutCount kCsrRegisterSpaceInvalidOffset, // UNUSED, infeed_0_0RunControl kCsrRegisterSpaceInvalidOffset, // UNUSED, outfeed_0_0RunControl kCsrRegisterSpaceInvalidOffset, // UNUSED, infeed_0_1RunControl @@ -1541,6 +1543,30 @@ const ScalarCoreCsrOffsets kBeagleScalarCoreCsrOffsets = { kCsrRegisterSpaceInvalidOffset, // UNUSED, packageTdpAction kCsrRegisterSpaceInvalidOffset, // UNUSED, ThrottleStallCounter kCsrRegisterSpaceInvalidOffset, // UNUSED, cycleCount + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_ScalarCoreDatapath_0 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Mask_ScalarCoreDatapath_0 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Force_ScalarCoreDatapath_0 + kCsrRegisterSpaceInvalidOffset, // UNUSED, + // Error_Timestamp_ScalarCoreDatapath_0 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Info_ScalarCoreDatapath_0 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_ScalarCoreDatapath_1 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Mask_ScalarCoreDatapath_1 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Force_ScalarCoreDatapath_1 + kCsrRegisterSpaceInvalidOffset, // UNUSED, + // Error_Timestamp_ScalarCoreDatapath_1 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Info_ScalarCoreDatapath_1 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_ScalarCoreDatapath_2 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Mask_ScalarCoreDatapath_2 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Force_ScalarCoreDatapath_2 + kCsrRegisterSpaceInvalidOffset, // UNUSED, + // Error_Timestamp_ScalarCoreDatapath_2 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Info_ScalarCoreDatapath_2 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_ScalarCoreDatapath_3 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Mask_ScalarCoreDatapath_3 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Force_ScalarCoreDatapath_3 + kCsrRegisterSpaceInvalidOffset, // UNUSED, + // Error_Timestamp_ScalarCoreDatapath_3 + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_Info_ScalarCoreDatapath_3 }; const MemoryCsrOffsets kBeagleScmemoryMemoryCsrOffsets = { @@ -1604,6 +1630,12 @@ const TileCsrOffsets kBeagleTileCsrOffsets = { kCsrRegisterSpaceInvalidOffset, // UNUSED, narrowMemoryContext_3 kCsrRegisterSpaceInvalidOffset, // UNUSED, TileRingBusCreditSenderReset kCsrRegisterSpaceInvalidOffset, // UNUSED, TileRingBusCreditReceiverReset + 0x40370, // NOLINT: Error_Tile + 0x40378, // NOLINT: Error_Mask_Tile + 0x40380, // NOLINT: Error_Force_Tile + 0x40388, // NOLINT: Error_Timestamp_Tile + kCsrRegisterSpaceInvalidOffset, // UNUSED, Error_ContextId_Tile + 0x40390, // NOLINT: Error_Info_Tile }; const WireCsrOffsets kBeagleWireCsrOffsets = { diff --git a/driver/config/chip_config.h b/driver/config/chip_config.h index 76f6a49..31ee186 100644 --- a/driver/config/chip_config.h +++ b/driver/config/chip_config.h @@ -26,6 +26,7 @@ #include "driver/config/hib_kernel_csr_offsets.h" #include "driver/config/hib_user_csr_offsets.h" #include "driver/config/interrupt_csr_offsets.h" +#include "driver/config/lpm_csr_offsets.h" #include "driver/config/memory_csr_offsets.h" #include "driver/config/misc_csr_offsets.h" #include "driver/config/msix_csr_offsets.h" diff --git a/driver/config/chip_structures.h b/driver/config/chip_structures.h index 5f2a58b..4a3ad1c 100644 --- a/driver/config/chip_structures.h +++ b/driver/config/chip_structures.h @@ -92,6 +92,8 @@ struct ChipStructures { // Number of scalar core contexts supported. Default 1 is legacy behavior with // no context switching. uint64 number_of_scalar_core_contexts; + + uint64 support_tile_thread_gcsr_node; }; } // namespace config diff --git a/driver/config/common_csr_helper.h b/driver/config/common_csr_helper.h index f070b33..583a28a 100644 --- a/driver/config/common_csr_helper.h +++ b/driver/config/common_csr_helper.h @@ -499,6 +499,7 @@ class ClockEnableReg { void set_clock_enable(uint64 value) { reg_.clock_enable_ = value; } uint64 clock_enable() const { return reg_.clock_enable_(); } void set_idle_override(uint64 value) { reg_.idle_override_ = value; } + uint64 idle_override() const { return reg_.idle_override_(); } private: union { diff --git a/driver/config/lpm_csr_offsets.h b/driver/config/lpm_csr_offsets.h new file mode 100644 index 0000000..13a6988 --- /dev/null +++ b/driver/config/lpm_csr_offsets.h @@ -0,0 +1,41 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_CONFIG_LPM_CSR_OFFSETS_H_ +#define DARWINN_DRIVER_CONFIG_LPM_CSR_OFFSETS_H_ + +#include "port/integral_types.h" + +namespace platforms { +namespace darwinn { +namespace driver { +namespace config { + +// This struct holds various CSR offsets for lpm programming. Members are +// intentionally named to match the GCSR register names. +struct LpmCsrOffsets { + uint64 psm_0_dmem_cfg; + uint64 psm_0_dmem_start; + uint64 psm_0_dmem_status; + uint64 psm_1_dmem_cfg; + uint64 psm_1_dmem_start; + uint64 psm_1_dmem_status; +}; + +} // namespace config +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_CONFIG_LPM_CSR_OFFSETS_H_ diff --git a/driver/config/scalar_core_csr_offsets.h b/driver/config/scalar_core_csr_offsets.h index 902a3da..85fede4 100644 --- a/driver/config/scalar_core_csr_offsets.h +++ b/driver/config/scalar_core_csr_offsets.h @@ -59,9 +59,13 @@ struct ScalarCoreCsrOffsets { uint64 infeed1RunControl; uint64 outfeed1RunControl; + // TODO: Separate out context switching related stuff to + // scalar_core_context_csr_offsets.h // Context Switching uint64 contextControl; uint64 contextStatus; + uint64 contextSwitchCount; + uint64 contextSwitchTimeoutCount; // Context 0 uint64 infeed_0_0RunControl; @@ -131,6 +135,31 @@ struct ScalarCoreCsrOffsets { // Scalar core cycle count. This could be used to synchronize timestamp // between host and the TPU uint64 cycleCount; + + // Error Registers + uint64 Error_ScalarCoreDatapath_0; + uint64 Error_Mask_ScalarCoreDatapath_0; + uint64 Error_Force_ScalarCoreDatapath_0; + uint64 Error_Timestamp_ScalarCoreDatapath_0; + uint64 Error_Info_ScalarCoreDatapath_0; + + uint64 Error_ScalarCoreDatapath_1; + uint64 Error_Mask_ScalarCoreDatapath_1; + uint64 Error_Force_ScalarCoreDatapath_1; + uint64 Error_Timestamp_ScalarCoreDatapath_1; + uint64 Error_Info_ScalarCoreDatapath_1; + + uint64 Error_ScalarCoreDatapath_2; + uint64 Error_Mask_ScalarCoreDatapath_2; + uint64 Error_Force_ScalarCoreDatapath_2; + uint64 Error_Timestamp_ScalarCoreDatapath_2; + uint64 Error_Info_ScalarCoreDatapath_2; + + uint64 Error_ScalarCoreDatapath_3; + uint64 Error_Mask_ScalarCoreDatapath_3; + uint64 Error_Force_ScalarCoreDatapath_3; + uint64 Error_Timestamp_ScalarCoreDatapath_3; + uint64 Error_Info_ScalarCoreDatapath_3; }; } // namespace config diff --git a/driver/config/tile_csr_offsets.h b/driver/config/tile_csr_offsets.h index 823c274..09cd2eb 100644 --- a/driver/config/tile_csr_offsets.h +++ b/driver/config/tile_csr_offsets.h @@ -92,6 +92,14 @@ struct TileCsrOffsets { // Ring bus credit control uint64 TileRingBusCreditSenderReset; uint64 TileRingBusCreditReceiverReset; + + // Error Tile + uint64 Error_Tile; + uint64 Error_Mask_Tile; + uint64 Error_Force_Tile; + uint64 Error_Timestamp_Tile; + uint64 Error_ContextId_Tile; + uint64 Error_Info_Tile; }; } // namespace config diff --git a/driver/driver.cc b/driver/driver.cc index 8da3eb0..3346856 100644 --- a/driver/driver.cc +++ b/driver/driver.cc @@ -24,6 +24,7 @@ #include "driver/package_registry.h" #include "driver/request.h" #include "driver/tpu_request.h" +#include "driver_shared/time_stamper/time_stamper.h" #include "executable/executable_generated.h" #include "port/blocking_counter.h" #include "port/errors.h" @@ -52,7 +53,7 @@ using api::ExecutionContextInterface; Driver::Driver(api::Chip chip, std::unique_ptr executable_registry, const api::DriverOptions& driver_options, - std::unique_ptr time_stamper) + std::unique_ptr time_stamper) : chip_(chip), executable_registry_(std::move(executable_registry)), time_stamper_(std::move(time_stamper)), diff --git a/driver/driver.h b/driver/driver.h index 8dccb15..3b96961 100644 --- a/driver/driver.h +++ b/driver/driver.h @@ -34,8 +34,8 @@ #include "driver/memory/dma_direction.h" #include "driver/package_registry.h" #include "driver/request.h" -#include "driver/time_stamper/time_stamper.h" #include "driver/tpu_request.h" +#include "driver_shared/time_stamper/time_stamper.h" #include "executable/executable_generated.h" #include "port/integral_types.h" #include "port/shared_mutex.h" @@ -129,7 +129,7 @@ class Driver : public api::Driver { protected: Driver(api::Chip chip, std::unique_ptr executable_registry, const api::DriverOptions& driver_options, - std::unique_ptr timestamper); + std::unique_ptr timestamper); // The base driver implementation does the necessary state checks and // validations before issuing the following calls that are implemented by the @@ -209,7 +209,7 @@ class Driver : public api::Driver { GetOldestActiveRequest() const = 0; private: - // Driver state. Transitions : + // Driver state. Transitions: // kClosed -> kOpen -> kClosing -> kClosed. enum State { kOpen, // Driver is Open. @@ -311,7 +311,7 @@ class Driver : public api::Driver { std::unique_ptr executable_registry_; // Driver clock for timestamp reporting - std::unique_ptr time_stamper_; + std::unique_ptr time_stamper_; // Registered fatal Error Callback. FatalErrorCallback fatal_error_callback_; diff --git a/driver/driver_factory_windows.cc b/driver/driver_factory_windows.cc index 9f63088..c8fce9c 100644 --- a/driver/driver_factory_windows.cc +++ b/driver/driver_factory_windows.cc @@ -13,7 +13,8 @@ // limitations under the License. #include "driver/driver_factory.h" - +#include "driver/kernel/gasket_ioctl.h" +#include "port/fileio.h" #include "port/logging.h" namespace platforms { @@ -24,7 +25,58 @@ std::vector DriverProvider::EnumerateByClass( const std::string& class_name, const std::string& device_name, api::Chip chip, api::Device::Type type) { std::vector device_list; - LOG(FATAL) << "EnumerateByClass is not supported on Windows at this time."; + + VLOG(1) << "DriverFactoryWin::EnumerateByClass()... "; + + switch (chip) { + case api::Chip::kBeagle: + for (int j = 0; j < APEX_MAX_DEVICES; j++) { + std::string device_path; + device_path = + "\\\\?\\" + std::string(APEX_DEVICE_NAME_BASE) + std::to_string(j); + + FileDescriptor fd = CreateFileW( + std::wstring(device_path.begin(), device_path.end()).c_str(), + GENERIC_READ, + 0, // No sharing - expect gle==ERROR_ACCESS_DENIED if already + // opened + NULL, // default security attributes + OPEN_EXISTING, // disposition + 0, // file attributes + NULL); // do not copy file attributes + + if (fd != INVALID_FD_VALUE) { + VLOG(4) << "devpath=" << device_path; + device_list.push_back( + {api::Chip::kBeagle, api::Device::Type::PCI, device_path}); + CloseHandle(fd); + } else { + DWORD error = GetLastError(); + if (error == ERROR_FILE_NOT_FOUND) break; + if (error == ERROR_ACCESS_DENIED) { + // Device is used by another process. + VLOG(4) << "devpath=" << device_path << " (in use)"; + device_list.push_back( + {api::Chip::kBeagle, api::Device::Type::PCI, device_path}); + } else { + VLOG(4) << "devpath=" << device_path << " open failed with " + << error; + } + } + } + break; + + default: + LOG(FATAL) + << "EnumerateByClass is not supported on Windows for this device."; + } + + if (device_list.empty()) + VLOG(5) << "DriverFactoryWin::EnumerateByClass returns empty list."; + else + VLOG(5) << "DriverFactoryWin::EnumerateByClass returns listsize(%d)=" + << device_list.size(); + return device_list; } diff --git a/driver/interrupt/BUILD b/driver/interrupt/BUILD index ff6a461..febdb47 100644 --- a/driver/interrupt/BUILD +++ b/driver/interrupt/BUILD @@ -15,17 +15,19 @@ # Description: # Interrupt functionality. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "interrupt_handler", hdrs = ["interrupt_handler.h"], deps = ["//port"], ) -cc_library( +libedgetpu_cc_library( name = "wire_interrupt_handler", srcs = ["wire_interrupt_handler.cc"], hdrs = ["wire_interrupt_handler.h"], @@ -40,7 +42,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "interrupt_controller_interface", hdrs = ["interrupt_controller_interface.h"], deps = [ @@ -49,7 +51,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "dummy_interrupt_controller", hdrs = ["dummy_interrupt_controller.h"], deps = [ @@ -58,7 +60,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "interrupt_controller", srcs = ["interrupt_controller.cc"], hdrs = ["interrupt_controller.h"], @@ -71,7 +73,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "grouped_interrupt_controller", srcs = ["grouped_interrupt_controller.cc"], hdrs = ["grouped_interrupt_controller.h"], @@ -82,7 +84,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "top_level_interrupt_manager", srcs = ["top_level_interrupt_manager.cc"], hdrs = ["top_level_interrupt_manager.h"], diff --git a/driver/interrupt/wire_interrupt_handler.cc b/driver/interrupt/wire_interrupt_handler.cc index f2d0d3e..273e671 100644 --- a/driver/interrupt/wire_interrupt_handler.cc +++ b/driver/interrupt/wire_interrupt_handler.cc @@ -14,7 +14,6 @@ #include "driver/interrupt/wire_interrupt_handler.h" -#include #include // NOLINT #include #include // NOLINT @@ -117,8 +116,6 @@ void WireInterruptHandler::MaskInterrupt(int interrupt_id, bool mask) { void WireInterruptHandler::InvokeInterruptWithMask(int interrupt_id) { StdMutexLock lock(&mutex_); if (interrupts_[interrupt_id]) { - // Mask and unmask interrupt due to b/111367622. - // This is Noronha-specific, but shouldn't hurt for other architectures. MaskInterrupt(interrupt_id, true); interrupts_[interrupt_id](); MaskInterrupt(interrupt_id, false); diff --git a/driver/kernel/BUILD b/driver/kernel/BUILD index d4f6680..d4b1aad 100644 --- a/driver/kernel/BUILD +++ b/driver/kernel/BUILD @@ -15,53 +15,57 @@ # Description: # Kernel driver specific functionality. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "kernel_mmu_mapper", srcs = ["kernel_mmu_mapper.cc"], hdrs = ["kernel_mmu_mapper.h"], deps = [ - ":linux_gasket_ioctl", + ":gasket_ioctl", "//driver:hardware_structures", "//driver/memory:dma_direction", "//driver/memory:mmu_mapper", "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", "//port:tracing", ], ) -cc_library( +libedgetpu_cc_library( name = "kernel_event", - srcs = ["kernel_event.cc"], hdrs = ["kernel_event.h"], deps = [ "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", "//port:tracing", ], ) -cc_library( +libedgetpu_cc_library( name = "kernel_event_handler", srcs = ["kernel_event_handler.cc"], hdrs = ["kernel_event_handler.h"], deps = [ - ":kernel_event", - ":linux_gasket_ioctl", + ":gasket_ioctl", + "//driver/kernel:kernel_event", "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", "//port:tracing", ], ) -cc_library( +libedgetpu_cc_library( name = "kernel_interrupt_handler", srcs = ["kernel_interrupt_handler.cc"], hdrs = ["kernel_interrupt_handler.h"], @@ -69,53 +73,62 @@ cc_library( "//driver/interrupt:interrupt_handler", "//driver/kernel:kernel_event_handler", "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", ], ) -cc_library( +libedgetpu_cc_library( name = "kernel_wire_interrupt_handler", srcs = ["kernel_wire_interrupt_handler.cc"], hdrs = ["kernel_wire_interrupt_handler.h"], deps = [ - ":kernel_event_handler", "//driver/config", "//driver/interrupt:interrupt_handler", "//driver/interrupt:wire_interrupt_handler", + "//driver/kernel:kernel_event_handler", "//driver/registers", "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", ], ) -cc_library( +libedgetpu_cc_library( name = "kernel_coherent_allocator", srcs = ["kernel_coherent_allocator.cc"], hdrs = ["kernel_coherent_allocator.h"], deps = [ - ":linux_gasket_ioctl", + ":gasket_ioctl", "//driver:hardware_structures", "//driver/mmio:coherent_allocator", "//port", + "//port:fileio", ], ) -cc_library( +libedgetpu_cc_library( name = "kernel_registers", srcs = ["kernel_registers.cc"], hdrs = ["kernel_registers.h"], deps = [ + ":gasket_ioctl", "@com_google_absl//absl/strings:str_format", "//driver/registers", "//port", + "//port:fileio", "//port:std_mutex_lock", "//port:thread_annotations", ], ) -cc_library( - name = "linux_gasket_ioctl", - hdrs = ["linux_gasket_ioctl.h"], +libedgetpu_cc_library( + name = "gasket_ioctl", + hdrs = [ + "common_gasket_ioctl.inc", + "gasket_ioctl.h", + ], + deps = ["//driver/kernel/windows:gasket_ioctl_windows"], ) diff --git a/driver/kernel/linux_gasket_ioctl.h b/driver/kernel/common_gasket_ioctl.inc similarity index 86% rename from driver/kernel/linux_gasket_ioctl.h rename to driver/kernel/common_gasket_ioctl.inc index 2ce6f19..bda40ea 100644 --- a/driver/kernel/linux_gasket_ioctl.h +++ b/driver/kernel/common_gasket_ioctl.inc @@ -1,26 +1,7 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - /* Common Gasket device kernel and user space declarations. */ -#ifndef __LINUX_GASKET_IOCTL_H__ -#define __LINUX_GASKET_IOCTL_H__ - -#include -#include -#ifndef __KERNEL__ -#include +#ifndef __GASKET_IOCTL_H__ +#error This file should not be included directly include gasket_ioctl.h instead #endif /* ioctl structure declarations */ @@ -112,8 +93,6 @@ struct gasket_page_table_ioctl_dmabuf { uint32_t flags; }; -/* Base number for all Gasket-common IOCTLs */ -#define GASKET_IOCTL_BASE 0xDC /* Reset the device. */ // NOLINTNEXTLINE @@ -199,4 +178,3 @@ struct gasket_page_table_ioctl_dmabuf { #define GASKET_IOCTL_MAP_DMABUF \ _IOW(GASKET_IOCTL_BASE, 13, struct gasket_page_table_ioctl_dmabuf) -#endif /* __LINUX_GASKET_IOCTL_H__ */ diff --git a/driver/kernel/gasket_ioctl.h b/driver/kernel/gasket_ioctl.h new file mode 100644 index 0000000..c9fe09a --- /dev/null +++ b/driver/kernel/gasket_ioctl.h @@ -0,0 +1,35 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* Common Gasket device kernel and user space declarations. */ +#ifndef __GASKET_IOCTL_H__ +#define __GASKET_IOCTL_H__ + +/* Base number for all Gasket-common IOCTLs */ +#define GASKET_IOCTL_BASE 0xDC + +#ifdef _WIN32 +#include "driver/kernel/windows/windows_gasket_ioctl.inc" +#else // #ifdef _WIN32 +#include +#include +#include +#ifndef __KERNEL__ +#include +#endif +#endif // #ifdef _WIN32 else + +#include "driver/kernel/common_gasket_ioctl.inc" + +#endif /* __GASKET_IOCTL_H__ */ diff --git a/driver/kernel/kernel_coherent_allocator.cc b/driver/kernel/kernel_coherent_allocator.cc index 21e42e1..2bbed09 100644 --- a/driver/kernel/kernel_coherent_allocator.cc +++ b/driver/kernel/kernel_coherent_allocator.cc @@ -12,22 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "driver/kernel/kernel_coherent_allocator.h" + #include #include -#include -#include #include + #include // for PRI*64 #include #include "driver/hardware_structures.h" -#include "driver/kernel/kernel_coherent_allocator.h" -#include "driver/kernel/linux_gasket_ioctl.h" +#include "driver/kernel/gasket_ioctl.h" #include "port/cleanup.h" #include "port/integral_types.h" #include "port/logging.h" #include "port/math_util.h" -#include "port/statusor.h" +#include "port/status_macros.h" namespace platforms { namespace darwinn { @@ -40,19 +40,19 @@ KernelCoherentAllocator::KernelCoherentAllocator(const std::string &device_path, device_path_(device_path) {} util::StatusOr KernelCoherentAllocator::DoOpen(size_t size_bytes) { - if (fd_ != -1) { + if (fd_ != INVALID_FD_VALUE) { return util::FailedPreconditionError("Device already open."); } fd_ = open(device_path_.c_str(), O_RDWR); - if (fd_ < 0) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError( StringPrintf("Device open failed : %d (%s)", fd_, strerror(errno))); } auto fd_closer = MakeCleanup([this] { close(fd_); - fd_ = -1; + fd_ = INVALID_FD_VALUE; }); // Enable the allocator and request the memory region @@ -72,10 +72,9 @@ util::StatusOr KernelCoherentAllocator::DoOpen(size_t size_bytes) { dma_address_ = ioctl_buffer.dma_address; // Map the memory range so as it can be accessed by user. - char *mem_base = - static_cast(mmap(nullptr, size_bytes, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_LOCKED, fd_, dma_address_)); - if (mem_base == MAP_FAILED) { + char *mem_base; + auto statusor = Map(fd_, size_bytes, dma_address_); + if (!statusor.ok()) { // Release the memory block. ioctl_buffer.page_table_index = 0; ioctl_buffer.enable = 0; @@ -85,24 +84,20 @@ util::StatusOr KernelCoherentAllocator::DoOpen(size_t size_bytes) { VLOG(1) << StringPrintf("mmap_failed and couldn't free memory : %s.\n", strerror(errno)); } - return util::FailedPreconditionError( - StringPrintf("CoherentAllocator Could not mmap size %zu.", size_bytes)); + return statusor.status(); } + mem_base = std::move(statusor.ValueOrDie()); + fd_closer.release(); return mem_base; } util::Status KernelCoherentAllocator::DoClose(char *mem_base, size_t size_bytes) { - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } - util::Status status; - - if (munmap(mem_base, size_bytes)) { - status.Update(util::FailedPreconditionError( - StringPrintf("Error unmapping coherent memory. %s", strerror(errno)))); - } + util::Status status = Unmap(fd_, mem_base, size_bytes); // Release the memory block. gasket_coherent_alloc_config_ioctl ioctl_buffer; @@ -119,7 +114,7 @@ util::Status KernelCoherentAllocator::DoClose(char *mem_base, } close(fd_); - fd_ = -1; + fd_ = INVALID_FD_VALUE; dma_address_ = 0; return util::Status(); // OK } diff --git a/driver/kernel/kernel_coherent_allocator.h b/driver/kernel/kernel_coherent_allocator.h index e29b5e6..1afd4b6 100644 --- a/driver/kernel/kernel_coherent_allocator.h +++ b/driver/kernel/kernel_coherent_allocator.h @@ -12,17 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef DARWINN_DRIVER_KERNEL_KERNEL_COHERENT_ALLOCATOR_H_ -#define DARWINN_DRIVER_KERNEL_KERNEL_COHERENT_ALLOCATOR_H_ +#ifndef DARWINN_DRIVER_KERNEL_KERNEL_COHERENT_ALLOCATOR_INTERNAL_H_ +#define DARWINN_DRIVER_KERNEL_KERNEL_COHERENT_ALLOCATOR_INTERNAL_H_ #include #include #include -#include + #include #include "driver/mmio/coherent_allocator.h" #include "port/errors.h" +#include "port/fileio.h" #include "port/integral_types.h" #include "port/logging.h" #include "port/statusor.h" @@ -37,7 +38,14 @@ class KernelCoherentAllocator : public CoherentAllocator { public: KernelCoherentAllocator(const std::string &device_path, int alignment_bytes, size_t size_bytes); - ~KernelCoherentAllocator() = default; + virtual ~KernelCoherentAllocator() = default; + + protected: + // Maps and unmaps kernel allocated memory block to user space. + virtual util::StatusOr Map(FileDescriptor fd, size_t size_bytes, + uint64 dma_address) = 0; + virtual util::Status Unmap(FileDescriptor fd, char *mem_base, + size_t size_bytes) = 0; private: // Implements Open. @@ -47,7 +55,7 @@ class KernelCoherentAllocator : public CoherentAllocator { util::Status DoClose(char *mem_base, size_t size_bytes) override; // File descriptor of the opened device. - int fd_{-1}; + FileDescriptor fd_{INVALID_FD_VALUE}; // Device specific DMA address of the coherent memory block. uint64 dma_address_{0}; @@ -60,4 +68,4 @@ class KernelCoherentAllocator : public CoherentAllocator { } // namespace darwinn } // namespace platforms -#endif // DARWINN_DRIVER_KERNEL_KERNEL_COHERENT_ALLOCATOR_H_ +#endif // DARWINN_DRIVER_KERNEL_KERNEL_COHERENT_ALLOCATOR_INTERNAL_H_ diff --git a/driver/kernel/kernel_event.h b/driver/kernel/kernel_event.h index 3e96274..48123d9 100644 --- a/driver/kernel/kernel_event.h +++ b/driver/kernel/kernel_event.h @@ -15,11 +15,9 @@ #ifndef DARWINN_DRIVER_KERNEL_KERNEL_EVENT_H_ #define DARWINN_DRIVER_KERNEL_KERNEL_EVENT_H_ -#include // NOLINT -#include // NOLINT +#include -#include "port/status.h" -#include "port/thread_annotations.h" +#include "port/fileio.h" namespace platforms { namespace darwinn { @@ -33,31 +31,12 @@ class KernelEvent { public: using Handler = std::function; - KernelEvent(int event_fd, Handler handler); - ~KernelEvent(); + KernelEvent(FileDescriptor event_fd, Handler handler) {} + virtual ~KernelEvent() = default; // This class is neither copyable nor movable. KernelEvent(const KernelEvent&) = delete; KernelEvent& operator=(const KernelEvent&) = delete; - - private: - // Monitors eventfd_. Runs on thread_. - void Monitor(int event_fd, const Handler& handler); - - // Convenience function to read |enabled_| with locks held. - bool IsEnabled() const LOCKS_EXCLUDED(mutex_); - - // Event fd. - const int event_fd_; - - // Mutex that guards enabled_; - mutable std::mutex mutex_; - - // Enabled if true. - bool enabled_ GUARDED_BY(mutex_){true}; - - // Thread for monitoring interrupts. - std::thread thread_; }; } // namespace driver diff --git a/driver/kernel/kernel_event_handler.cc b/driver/kernel/kernel_event_handler.cc index 602b51e..2674273 100644 --- a/driver/kernel/kernel_event_handler.cc +++ b/driver/kernel/kernel_event_handler.cc @@ -17,18 +17,13 @@ #include #include #include -#include -#include -#include #include #include // NOLINT #include #include "driver/kernel/kernel_event.h" -#include "driver/kernel/linux_gasket_ioctl.h" #include "port/errors.h" #include "port/integral_types.h" -#include "port/ptr_util.h" #include "port/status.h" #include "port/status_macros.h" #include "port/std_mutex_lock.h" @@ -42,13 +37,13 @@ namespace driver { KernelEventHandler::KernelEventHandler(const std::string& device_path, int num_events) : device_path_(device_path), num_events_(num_events) { - event_fds_.resize(num_events_, -1); + event_fds_.resize(num_events_, INVALID_FD_VALUE); events_.resize(num_events_); } util::Status KernelEventHandler::Open() { StdMutexLock lock(&mutex_); - if (fd_ != -1) { + if (fd_ != INVALID_FD_VALUE) { return util::FailedPreconditionError("Device already open."); } @@ -59,7 +54,7 @@ util::Status KernelEventHandler::Open() { } for (int i = 0; i < num_events_; ++i) { - event_fds_[i] = eventfd(0, EFD_CLOEXEC); + event_fds_[i] = InitializeEventFd(i); events_[i].reset(); } @@ -68,48 +63,34 @@ util::Status KernelEventHandler::Open() { util::Status KernelEventHandler::Close() { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } + util::Status status; for (int i = 0; i < num_events_; ++i) { events_[i].reset(); - close(event_fds_[i]); + status.Update(ReleaseEventFd(fd_, event_fds_[i], i)); } close(fd_); - fd_ = -1; + fd_ = INVALID_FD_VALUE; - return util::Status(); // OK -} - -util::Status KernelEventHandler::SetEventFd(int event_fd, int event_id) const { - gasket_interrupt_eventfd interrupt; - interrupt.interrupt = event_id; - interrupt.event_fd = event_fd; - if (ioctl(fd_, GASKET_IOCTL_SET_EVENTFD, &interrupt) != 0) { - return util::FailedPreconditionError(StringPrintf( - "Setting Event Fd Failed : %d (%s)", fd_, strerror(errno))); - } - - VLOG(5) << StringPrintf("Set event fd : event_id:%d -> event_fd:%d, ", - event_id, event_fd); - - return util::Status(); // OK + return status; } util::Status KernelEventHandler::RegisterEvent(int event_id, KernelEvent::Handler handler) { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } - RETURN_IF_ERROR(SetEventFd(event_fds_[event_id], event_id)); + RETURN_IF_ERROR(SetEventFd(fd_, event_fds_[event_id], event_id)); // Enable events. events_[event_id] = - gtl::MakeUnique(event_fds_[event_id], std::move(handler)); + CreateKernelEvent(event_fds_[event_id], std::move(handler)); return util::Status(); // OK; } diff --git a/driver/kernel/kernel_event_handler.h b/driver/kernel/kernel_event_handler.h index 86e1479..d0bc217 100644 --- a/driver/kernel/kernel_event_handler.h +++ b/driver/kernel/kernel_event_handler.h @@ -17,12 +17,13 @@ #include #include -#include // NOLINT +#include // NOLINT #include #include // NOLINT #include #include "driver/kernel/kernel_event.h" +#include "port/fileio.h" #include "port/status.h" #include "port/thread_annotations.h" @@ -34,7 +35,7 @@ namespace driver { class KernelEventHandler { public: KernelEventHandler(const std::string& device_path, int num_events); - ~KernelEventHandler() = default; + virtual ~KernelEventHandler() = default; util::Status Open() LOCKS_EXCLUDED(mutex_); util::Status Close() LOCKS_EXCLUDED(mutex_); @@ -43,11 +44,29 @@ class KernelEventHandler { util::Status RegisterEvent(int event_id, KernelEvent::Handler handler) LOCKS_EXCLUDED(mutex_); - private: + protected: // Maps the specified event number with the specified id. - util::Status SetEventFd(int event_fd, int event_id) const - SHARED_LOCKS_REQUIRED(mutex_); + virtual util::Status SetEventFd(FileDescriptor fd, FileDescriptor event_fd, + int event_id) const + SHARED_LOCKS_REQUIRED(mutex_) = 0; + + // Performs platform specific event object handle initialization + virtual FileDescriptor InitializeEventFd(int event_id) const + SHARED_LOCKS_REQUIRED(mutex_) = 0; + + // Releases platform specific resources associated with event object handle + virtual util::Status ReleaseEventFd(FileDescriptor fd, + FileDescriptor event_fd, + int event_id) const + SHARED_LOCKS_REQUIRED(mutex_) = 0; + // Creates platform specific KernelEvent backing object + virtual std::unique_ptr CreateKernelEvent( + FileDescriptor event_fd, KernelEvent::Handler handler) = 0; + + const std::string& GetDevicePath() const { return device_path_; } + + private: // Device path. const std::string device_path_; @@ -58,10 +77,10 @@ class KernelEventHandler { mutable std::mutex mutex_; // File descriptor of the opened device. - int fd_ GUARDED_BY(mutex_){-1}; + FileDescriptor fd_ GUARDED_BY(mutex_){INVALID_FD_VALUE}; // Event FD list. - std::vector event_fds_ GUARDED_BY(mutex_); + std::vector event_fds_ GUARDED_BY(mutex_); // Registered events. std::vector> events_ GUARDED_BY(mutex_); diff --git a/driver/kernel/kernel_interrupt_handler.cc b/driver/kernel/kernel_interrupt_handler.cc index 70843a2..5bcbc56 100644 --- a/driver/kernel/kernel_interrupt_handler.cc +++ b/driver/kernel/kernel_interrupt_handler.cc @@ -30,18 +30,19 @@ namespace platforms { namespace darwinn { namespace driver { -KernelInterruptHandler::KernelInterruptHandler(const std::string& device_path) - : event_handler_(device_path, DW_INTERRUPT_COUNT) {} +KernelInterruptHandler::KernelInterruptHandler( + std::unique_ptr event_handler) + : event_handler_(std::move(event_handler)) {} -util::Status KernelInterruptHandler::Open() { return event_handler_.Open(); } +util::Status KernelInterruptHandler::Open() { return event_handler_->Open(); } util::Status KernelInterruptHandler::Close(bool in_error) { - return event_handler_.Close(); + return event_handler_->Close(); } util::Status KernelInterruptHandler::Register(Interrupt interrupt, Handler handler) { - return event_handler_.RegisterEvent(interrupt, std::move(handler)); + return event_handler_->RegisterEvent(interrupt, std::move(handler)); } } // namespace driver diff --git a/driver/kernel/kernel_interrupt_handler.h b/driver/kernel/kernel_interrupt_handler.h index 0a09dc5..d8f4e02 100644 --- a/driver/kernel/kernel_interrupt_handler.h +++ b/driver/kernel/kernel_interrupt_handler.h @@ -32,7 +32,8 @@ class KernelInterruptHandler : public InterruptHandler { // Default close to avoid name hiding. using InterruptHandler::Close; - explicit KernelInterruptHandler(const std::string& device_path); + explicit KernelInterruptHandler( + std::unique_ptr event_handler); ~KernelInterruptHandler() override = default; util::Status Open() override; @@ -41,7 +42,7 @@ class KernelInterruptHandler : public InterruptHandler { private: // Backing event handler. - KernelEventHandler event_handler_; + std::unique_ptr event_handler_; }; } // namespace driver diff --git a/driver/kernel/kernel_mmu_mapper.cc b/driver/kernel/kernel_mmu_mapper.cc index 9a8e4cb..a59840c 100644 --- a/driver/kernel/kernel_mmu_mapper.cc +++ b/driver/kernel/kernel_mmu_mapper.cc @@ -15,17 +15,15 @@ #include "driver/kernel/kernel_mmu_mapper.h" #include -#include -#include + #include #include -#include #include // for PRI*64 #include #include "driver/hardware_structures.h" -#include "driver/kernel/linux_gasket_ioctl.h" +#include "driver/kernel/gasket_ioctl.h" #include "port/errors.h" #include "port/integral_types.h" #include "port/status.h" @@ -44,7 +42,7 @@ KernelMmuMapper::KernelMmuMapper(const std::string &device_path) util::Status KernelMmuMapper::Open( int num_simple_page_table_entries_requested) { StdMutexLock lock(&mutex_); - if (fd_ != -1) { + if (fd_ != INVALID_FD_VALUE) { return util::FailedPreconditionError("Device already open."); } @@ -58,6 +56,7 @@ util::Status KernelMmuMapper::Open( memset(&ioctl_buffer, 0, sizeof(ioctl_buffer)); ioctl_buffer.page_table_index = 0; ioctl_buffer.size = num_simple_page_table_entries_requested; + if (ioctl(fd_, GASKET_IOCTL_PARTITION_PAGE_TABLE, &ioctl_buffer) != 0) { return util::FailedPreconditionError(StringPrintf( "Could not partition page table. : %d (%s)", fd_, strerror(errno))); @@ -68,12 +67,13 @@ util::Status KernelMmuMapper::Open( util::Status KernelMmuMapper::Close() { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } close(fd_); - fd_ = -1; + fd_ = INVALID_FD_VALUE; return util::Status(); // OK } @@ -96,7 +96,8 @@ util::Status KernelMmuMapper::DoMap(const void *buffer, int num_pages, DmaDirection direction) { TRACE_SCOPE("KernelMmuMapper::DoMap"); StdMutexLock lock(&mutex_); - if (fd_ == -1) { + + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } @@ -109,7 +110,7 @@ util::Status KernelMmuMapper::DoMap(const void *buffer, int num_pages, buffer_to_map.flags = DirectionFlag(direction) << GASKET_PT_FLAGS_DMA_DIRECTION_SHIFT; - int ioctl_retval; + int ioctl_retval = 0; if (map_flags_supported_) { ioctl_retval = ioctl(fd_, GASKET_IOCTL_MAP_BUFFER_FLAGS, &buffer_to_map); if (ioctl_retval == -EPERM || ioctl_retval == -ENOTTY || @@ -151,7 +152,8 @@ util::Status KernelMmuMapper::DoUnmap(const void *buffer, int num_pages, uint64 device_virtual_address) { TRACE_SCOPE("KernelMmuMapper::DoUnmap"); StdMutexLock lock(&mutex_); - if (fd_ == -1) { + + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } @@ -161,6 +163,7 @@ util::Status KernelMmuMapper::DoUnmap(const void *buffer, int num_pages, buffer_to_map.host_address = reinterpret_cast(buffer); buffer_to_map.size = num_pages * kHostPageSize; buffer_to_map.device_address = device_virtual_address; + if (ioctl(fd_, GASKET_IOCTL_UNMAP_BUFFER, &buffer_to_map) != 0) { return util::FailedPreconditionError( StringPrintf("Could not unmap pages : %d (%s)", fd_, strerror(errno))); @@ -178,7 +181,7 @@ util::Status KernelMmuMapper::DoMap(int fd, int num_pages, DmaDirection direction) { TRACE_SCOPE("KernelMmuMapper::DoMap"); StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } @@ -209,7 +212,7 @@ util::Status KernelMmuMapper::DoUnmap(int fd, int num_pages, uint64 device_virtual_address) { TRACE_SCOPE("KernelMmuMapper::DoUnmap"); StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } diff --git a/driver/kernel/kernel_mmu_mapper.h b/driver/kernel/kernel_mmu_mapper.h index 02d790a..b55b553 100644 --- a/driver/kernel/kernel_mmu_mapper.h +++ b/driver/kernel/kernel_mmu_mapper.h @@ -15,11 +15,15 @@ #ifndef DARWINN_DRIVER_KERNEL_KERNEL_MMU_MAPPER_H_ #define DARWINN_DRIVER_KERNEL_KERNEL_MMU_MAPPER_H_ -#include #include // NOLINT +#ifndef _WIN32 +#include +#endif + #include "driver/memory/dma_direction.h" #include "driver/memory/mmu_mapper.h" +#include "port/fileio.h" #include "port/integral_types.h" #include "port/status.h" #include "port/statusor.h" @@ -64,11 +68,11 @@ class KernelMmuMapper : public MmuMapper { // then may want to make appropriate locking the caller's responsibility. StdMutexLock lock(&mutex_); - if (fd_ != -1) { + if (fd_ != INVALID_FD_VALUE) { return ioctl(fd_, std::forward(params)...); } else { VLOG(4) << "Invalid file descriptor."; - return -1; + return INVALID_FD_VALUE; } } @@ -77,7 +81,7 @@ class KernelMmuMapper : public MmuMapper { const std::string device_path_; // File descriptor of the opened device. - int fd_ GUARDED_BY(mutex_){-1}; + FileDescriptor fd_ GUARDED_BY(mutex_){INVALID_FD_VALUE}; // Mutex that guards fd_; mutable std::mutex mutex_; diff --git a/driver/kernel/kernel_registers.cc b/driver/kernel/kernel_registers.cc index 7baf9e0..7bbadbd 100644 --- a/driver/kernel/kernel_registers.cc +++ b/driver/kernel/kernel_registers.cc @@ -16,10 +16,8 @@ #include #include -#include #include #include -#include #include @@ -52,29 +50,33 @@ KernelRegisters::KernelRegisters(const std::string& device_path, : KernelRegisters(device_path, {{mmap_offset, mmap_size}}, read_only) {} KernelRegisters::~KernelRegisters() { - for (auto& region : mmap_region_) { - if (region.registers != nullptr) { - const int ret = munmap(region.registers, region.size); - if (ret != 0) { - LOG(ERROR) << "Error unmapping registers: " << strerror(errno); - } - region.registers = nullptr; - } - } - - if (fd_ != -1) { + util::Status status; + if (fd_ != INVALID_FD_VALUE) { LOG(WARNING) << "Destroying KernelRegisters - Close() had not yet been called!"; - util::Status status = Close(); + status = Close(); if (!status.ok()) { LOG(ERROR) << status; } } } +void KernelRegisters::UnmapAllRegions() { + util::Status status; + for (auto& region : mmap_region_) { + if (region.registers != nullptr) { + status = UnmapRegion(fd_, region); + if (status.ok()) { + LOG(ERROR) << status; + } + region.registers = nullptr; + } + } +} + util::Status KernelRegisters::Open() { StdMutexLock lock(&mutex_); - if (fd_ != -1) { + if (fd_ != INVALID_FD_VALUE) { return util::FailedPreconditionError("Device already open."); } @@ -86,32 +88,23 @@ util::Status KernelRegisters::Open() { } fd_ = open(device_path_.c_str(), mode); - if (fd_ < 0) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError( StringPrintf("Device open failed : %d (%s)", fd_, strerror(errno))); } - int protections = PROT_READ | PROT_WRITE; - if (read_only_) { - protections = PROT_READ; - } - for (auto& region : mmap_region_) { VLOG(1) << StringPrintf("mmap_offset=0x%016llx, mmap_size=%lld", static_cast(region.offset), static_cast(region.size)); - region.registers = - static_cast(mmap(nullptr, region.size, protections, - MAP_SHARED | MAP_LOCKED, fd_, region.offset)); - if (region.registers == MAP_FAILED) { + auto statusor = MapRegion(fd_, region, read_only_); + if (!statusor.ok()) { close(fd_); - fd_ = -1; - region.registers = nullptr; - - return util::FailedPreconditionError( - StringPrintf("Could not mmap: %s.", strerror(errno))); + fd_ = INVALID_FD_VALUE; + return statusor.status(); } + region.registers = std::move(statusor.ValueOrDie()); VLOG(3) << "Got map addr at 0x" << std::hex << region.registers; } @@ -120,7 +113,7 @@ util::Status KernelRegisters::Open() { util::Status KernelRegisters::Close() { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } @@ -132,16 +125,16 @@ util::Status KernelRegisters::Close() { static_cast(region.offset), // NOLINT(runtime/int) static_cast(region.size), // NOLINT(runtime/int) read_only_); - const int ret = munmap(region.registers, region.size); - if (ret != 0) { - LOG(ERROR) << "Error unmapping registers: " << strerror(errno); + util::Status status = UnmapRegion(fd_, region); + if (!status.ok()) { + LOG(ERROR) << status; } region.registers = nullptr; } } close(fd_); - fd_ = -1; + fd_ = INVALID_FD_VALUE; return util::Status(); // OK } @@ -180,7 +173,7 @@ inline util::StatusOr KernelRegisters::GetMappedOffset( util::Status KernelRegisters::Write(uint64 offset, uint64 value) { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } if (read_only_) { @@ -204,7 +197,7 @@ util::Status KernelRegisters::Write(uint64 offset, uint64 value) { util::StatusOr KernelRegisters::Read(uint64 offset) { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } if (offset % sizeof(uint64) != 0) { @@ -225,7 +218,7 @@ util::StatusOr KernelRegisters::Read(uint64 offset) { util::Status KernelRegisters::Write32(uint64 offset, uint32 value) { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } if (read_only_) { @@ -233,7 +226,7 @@ util::Status KernelRegisters::Write32(uint64 offset, uint32 value) { } if (offset % sizeof(uint32) != 0) { return util::FailedPreconditionError(StringPrintf( - "Offset (0x%016llx) not aligned to 8B", + "Offset (0x%016llx) not aligned to 4B", static_cast(offset))); // NOLINT(runtime/int) } @@ -249,7 +242,7 @@ util::Status KernelRegisters::Write32(uint64 offset, uint32 value) { util::StatusOr KernelRegisters::Read32(uint64 offset) { StdMutexLock lock(&mutex_); - if (fd_ == -1) { + if (fd_ == INVALID_FD_VALUE) { return util::FailedPreconditionError("Device not open."); } if (offset % sizeof(uint32) != 0) { diff --git a/driver/kernel/kernel_registers.h b/driver/kernel/kernel_registers.h index 4e95ea7..5ade4fc 100644 --- a/driver/kernel/kernel_registers.h +++ b/driver/kernel/kernel_registers.h @@ -20,6 +20,7 @@ #include #include "driver/registers/registers.h" +#include "port/fileio.h" #include "port/integral_types.h" #include "port/status.h" #include "port/statusor.h" @@ -70,6 +71,20 @@ class KernelRegisters : public Registers { // Returns the reference to the mapped regions. std::vector& GetMmapRegion() { return mmap_region_; } + // Unmaps all device BARs ranges previously mapped to user mode space VAs. + void UnmapAllRegions() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Maps and returns user mode space VA for device BARs range described by + // region.offset and region.size. + virtual util::StatusOr MapRegion(FileDescriptor fd, + const MappedRegisterRegion& region, + bool read_only) = 0; + + // Unmaps device BARs range described by region.offset and region.size + // which was previously mapped to region.registers user mode space VA. + virtual util::Status UnmapRegion(FileDescriptor fd, + const MappedRegisterRegion& region) = 0; + private: // Maps CSR offset to virtual address without acquiring the lock. util::StatusOr GetMappedOffset(uint64 offset, int alignment) const @@ -85,7 +100,7 @@ class KernelRegisters : public Registers { const bool read_only_; // File descriptor of the opened device. - int fd_ GUARDED_BY(mutex_){-1}; + FileDescriptor fd_ GUARDED_BY(mutex_){INVALID_FD_VALUE}; // Mutex that guards fd_; mutable std::mutex mutex_; diff --git a/driver/kernel/kernel_wire_interrupt_handler.cc b/driver/kernel/kernel_wire_interrupt_handler.cc index cf40370..9a7c67a 100644 --- a/driver/kernel/kernel_wire_interrupt_handler.cc +++ b/driver/kernel/kernel_wire_interrupt_handler.cc @@ -34,9 +34,9 @@ namespace driver { KernelWireInterruptHandler::KernelWireInterruptHandler( Registers* registers, const config::WireCsrOffsets& wire_csr_offsets, - const std::string& device_path, int num_wires) + std::unique_ptr event_handler, int num_wires) : wire_handler_(registers, wire_csr_offsets, num_wires), - event_handler_(device_path, num_wires), + event_handler_(std::move(event_handler)), num_wires_(num_wires) {} util::Status KernelWireInterruptHandler::Open() { @@ -44,12 +44,13 @@ util::Status KernelWireInterruptHandler::Open() { auto wire_handler_closer = MakeCleanup( [this]() NO_THREAD_SAFETY_ANALYSIS { CHECK_OK(wire_handler_.Close()); }); - RETURN_IF_ERROR(event_handler_.Open()); - auto event_handler_closer = MakeCleanup( - [this]() NO_THREAD_SAFETY_ANALYSIS { CHECK_OK(event_handler_.Close()); }); + RETURN_IF_ERROR(event_handler_->Open()); + auto event_handler_closer = MakeCleanup([this]() NO_THREAD_SAFETY_ANALYSIS { + CHECK_OK(event_handler_->Close()); + }); for (int wire = 0; wire < num_wires_; ++wire) { - RETURN_IF_ERROR(event_handler_.RegisterEvent(wire, [this, wire]() { + RETURN_IF_ERROR(event_handler_->RegisterEvent(wire, [this, wire]() { wire_handler_.InvokeAllPendingInterrupts(wire); })); } @@ -63,7 +64,7 @@ util::Status KernelWireInterruptHandler::Open() { util::Status KernelWireInterruptHandler::Close(bool in_error) { util::Status status; - status.Update(event_handler_.Close()); + status.Update(event_handler_->Close()); status.Update(wire_handler_.Close()); return status; } diff --git a/driver/kernel/kernel_wire_interrupt_handler.h b/driver/kernel/kernel_wire_interrupt_handler.h index 718efc1..4a5efdd 100644 --- a/driver/kernel/kernel_wire_interrupt_handler.h +++ b/driver/kernel/kernel_wire_interrupt_handler.h @@ -39,7 +39,8 @@ class KernelWireInterruptHandler : public InterruptHandler { KernelWireInterruptHandler(Registers* registers, const config::WireCsrOffsets& wire_csr_offsets, - const std::string& device_path, int num_wires); + std::unique_ptr event_handler, + int num_wires); ~KernelWireInterruptHandler() override = default; // This class is neither copyable nor movable. @@ -56,7 +57,7 @@ class KernelWireInterruptHandler : public InterruptHandler { WireInterruptHandler wire_handler_; // KernelEventHandler - KernelEventHandler event_handler_; + std::unique_ptr event_handler_; // Number of wires. const int num_wires_; diff --git a/driver/kernel/linux/BUILD b/driver/kernel/linux/BUILD new file mode 100644 index 0000000..d135f63 --- /dev/null +++ b/driver/kernel/linux/BUILD @@ -0,0 +1,83 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: +# Kernel driver specific functionality. + +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +libedgetpu_cc_library( + name = "kernel_event_linux", + srcs = ["kernel_event_linux.cc"], + hdrs = ["kernel_event_linux.h"], + deps = [ + "//driver/kernel:kernel_event", + "//port", + "//port:fileio", + "//port:std_mutex_lock", + "//port:thread_annotations", + "//port:tracing", + ], +) + +libedgetpu_cc_library( + name = "kernel_event_handler_linux", + srcs = ["kernel_event_handler_linux.cc"], + hdrs = ["kernel_event_handler_linux.h"], + deps = [ + "//driver/kernel:gasket_ioctl", + "//driver/kernel:kernel_event", + "//driver/kernel:kernel_event_handler", + "//driver/kernel/linux:kernel_event_linux", + "//port", + "//port:fileio", + "//port:std_mutex_lock", + "//port:thread_annotations", + "//port:tracing", + ], +) + +libedgetpu_cc_library( + name = "kernel_coherent_allocator_linux", + srcs = ["kernel_coherent_allocator_linux.cc"], + hdrs = ["kernel_coherent_allocator_linux.h"], + deps = [ + "//driver:hardware_structures", + "//driver/kernel:gasket_ioctl", + "//driver/kernel:kernel_coherent_allocator", + "//driver/mmio:coherent_allocator", + "//port", + "//port:fileio", + ], +) + +libedgetpu_cc_library( + name = "kernel_registers_linux", + srcs = ["kernel_registers_linux.cc"], + hdrs = ["kernel_registers_linux.h"], + deps = [ + "@com_google_absl//absl/strings:str_format", + "//driver/kernel:gasket_ioctl", + "//driver/kernel:kernel_registers", + "//driver/registers", + "//port", + "//port:fileio", + "//port:std_mutex_lock", + "//port:thread_annotations", + ], +) diff --git a/driver/kernel/linux/kernel_coherent_allocator_linux.cc b/driver/kernel/linux/kernel_coherent_allocator_linux.cc new file mode 100644 index 0000000..5c072fd --- /dev/null +++ b/driver/kernel/linux/kernel_coherent_allocator_linux.cc @@ -0,0 +1,62 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/kernel/linux/kernel_coherent_allocator_linux.h" + +#include +#include +#include + +#include "driver/kernel/gasket_ioctl.h" +#include "port/errors.h" +#include "port/logging.h" +#include "port/statusor.h" +#include "port/stringprintf.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +KernelCoherentAllocatorLinux::KernelCoherentAllocatorLinux( + const std::string &device_path, int alignment_bytes, size_t size_bytes) + : KernelCoherentAllocator(device_path, alignment_bytes, size_bytes) {} + +util::StatusOr KernelCoherentAllocatorLinux::Map(FileDescriptor fd, + size_t size_bytes, + uint64 dma_address) { + // Map the memory range so as it can be accessed by user. + char *mem_base = + static_cast(mmap(nullptr, size_bytes, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED, fd, dma_address)); + if (mem_base == MAP_FAILED) { + return util::FailedPreconditionError( + StringPrintf("CoherentAllocator Could not mmap size %zu.", size_bytes)); + } + + return mem_base; +} + +util::Status KernelCoherentAllocatorLinux::Unmap(FileDescriptor fd, + char *mem_base, + size_t size_bytes) { + if (munmap(mem_base, size_bytes)) { + return util::FailedPreconditionError( + StringPrintf("Error unmapping coherent memory. %s", strerror(errno))); + } + return util::Status(); // OK +} + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/kernel/linux/kernel_coherent_allocator_linux.h b/driver/kernel/linux/kernel_coherent_allocator_linux.h new file mode 100644 index 0000000..f630993 --- /dev/null +++ b/driver/kernel/linux/kernel_coherent_allocator_linux.h @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_KERNEL_LINUX_KERNEL_COHERENT_ALLOCATOR_LINUX_H_ +#define DARWINN_DRIVER_KERNEL_LINUX_KERNEL_COHERENT_ALLOCATOR_LINUX_H_ + +#include "driver/kernel/kernel_coherent_allocator.h" +#include "port/fileio.h" +#include "port/statusor.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +// Functions to allocate coherent memory that is DMA-able by a Darwinn device. +class KernelCoherentAllocatorLinux : public KernelCoherentAllocator { + public: + KernelCoherentAllocatorLinux(const std::string &device_path, + int alignment_bytes, size_t size_bytes); + + private: + // Maps and unmaps kernel allocated memory block to user space. + util::StatusOr Map(FileDescriptor fd, size_t size_bytes, + uint64 dma_address) override; + util::Status Unmap(FileDescriptor fd, char *mem_base, + size_t size_bytes) override; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_KERNEL_LINUX_KERNEL_COHERENT_ALLOCATOR_LINUX_H_ diff --git a/driver/kernel/linux/kernel_event_handler_linux.cc b/driver/kernel/linux/kernel_event_handler_linux.cc new file mode 100644 index 0000000..972ae40 --- /dev/null +++ b/driver/kernel/linux/kernel_event_handler_linux.cc @@ -0,0 +1,69 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/kernel/linux/kernel_event_handler_linux.h" + +#include + +#include "driver/kernel/gasket_ioctl.h" +#include "driver/kernel/linux/kernel_event_linux.h" +#include "port/errors.h" +#include "port/ptr_util.h" +#include "port/status.h" +#include "port/stringprintf.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +KernelEventHandlerLinux::KernelEventHandlerLinux(const std::string& device_path, + int num_events) + : KernelEventHandler(device_path, num_events) {} + +util::Status KernelEventHandlerLinux::SetEventFd(FileDescriptor fd, + FileDescriptor event_fd, + int event_id) const { + gasket_interrupt_eventfd interrupt; + interrupt.interrupt = event_id; + interrupt.event_fd = event_fd; + if (ioctl(fd, GASKET_IOCTL_SET_EVENTFD, &interrupt) != 0) { + return util::FailedPreconditionError( + StringPrintf("Setting Event Fd Failed : %d (%s)", fd, strerror(errno))); + } + + VLOG(5) << StringPrintf("Set event fd : event_id:%d -> event_fd:%d, ", + event_id, event_fd); + + return util::OkStatus(); +} + +FileDescriptor KernelEventHandlerLinux::InitializeEventFd(int event_id) const { + return eventfd(0, EFD_CLOEXEC); +} + +util::Status KernelEventHandlerLinux::ReleaseEventFd(FileDescriptor fd, + FileDescriptor event_fd, + int event_id) const { + close(event_fd); + return util::OkStatus(); +} + +std::unique_ptr KernelEventHandlerLinux::CreateKernelEvent( + FileDescriptor event_fd, KernelEvent::Handler handler) { + return gtl::MakeUnique(event_fd, std::move(handler)); +} + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/kernel/linux/kernel_event_handler_linux.h b/driver/kernel/linux/kernel_event_handler_linux.h new file mode 100644 index 0000000..784bd5b --- /dev/null +++ b/driver/kernel/linux/kernel_event_handler_linux.h @@ -0,0 +1,52 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_KERNEL_KERNEL_EVENT_HANDLER_LINUX_H_ +#define DARWINN_DRIVER_KERNEL_KERNEL_EVENT_HANDLER_LINUX_H_ + +#include "driver/kernel/kernel_event_handler.h" +#include "port/fileio.h" +#include "port/status.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +// Implements a mechanism for processing kernel events. +class KernelEventHandlerLinux : public KernelEventHandler { + public: + KernelEventHandlerLinux(const std::string& device_path, int num_events); + + private: + // Maps the specified event number with the specified id. + util::Status SetEventFd(FileDescriptor fd, FileDescriptor event_fd, + int event_id) const override; + + // Performs platform specific event object handle initialization + FileDescriptor InitializeEventFd(int event_id) const override; + + // Releases platform specific resources associated with event object handle + util::Status ReleaseEventFd(FileDescriptor fd, FileDescriptor event_fd, + int event_id) const override; + + // Creates platform specific KernelEvent backing object + std::unique_ptr CreateKernelEvent( + FileDescriptor event_fd, KernelEvent::Handler handler) override; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_KERNEL_KERNEL_EVENT_HANDLER_LINUX_H_ diff --git a/driver/kernel/kernel_event.cc b/driver/kernel/linux/kernel_event_linux.cc similarity index 80% rename from driver/kernel/kernel_event.cc rename to driver/kernel/linux/kernel_event_linux.cc index 87bc442..826c773 100644 --- a/driver/kernel/kernel_event.cc +++ b/driver/kernel/linux/kernel_event_linux.cc @@ -12,20 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "driver/kernel/kernel_event.h" +#include "driver/kernel/linux/kernel_event_linux.h" #include -#include #include -#include -#include +#include #include -#include // NOLINT - #include "port/errors.h" #include "port/integral_types.h" -#include "port/ptr_util.h" #include "port/status.h" #include "port/status_macros.h" #include "port/std_mutex_lock.h" @@ -36,13 +31,15 @@ namespace platforms { namespace darwinn { namespace driver { -KernelEvent::KernelEvent(int event_fd, Handler handler) : event_fd_(event_fd) { - std::thread event_thread(&KernelEvent::Monitor, this, event_fd, +KernelEventLinux::KernelEventLinux(FileDescriptor event_fd, Handler handler) + : KernelEvent(event_fd, handler), + event_fd_(event_fd) { + std::thread event_thread(&KernelEventLinux::Monitor, this, std::move(handler)); thread_ = std::move(event_thread); } -KernelEvent::~KernelEvent() { +KernelEventLinux::~KernelEventLinux() { // Mark as disabled. { StdMutexLock lock(&mutex_); @@ -61,27 +58,22 @@ KernelEvent::~KernelEvent() { thread_.join(); } -bool KernelEvent::IsEnabled() const { - StdMutexLock lock(&mutex_); - return enabled_; -} - -void KernelEvent::Monitor(int event_fd, const Handler& handler) { - VLOG(5) << StringPrintf("event_fd=%d. Monitor thread begin.", event_fd); +void KernelEventLinux::Monitor(const Handler& handler) { + VLOG(5) << StringPrintf("event_fd=%d. Monitor thread begin.", event_fd_); TRACE_START_THREAD("KernelEventHandlerMonitor"); while (IsEnabled()) { // Wait for events (blocking). uint64_t num_events = 0; - int result = read(event_fd, &num_events, sizeof(num_events)); + int result = read(event_fd_, &num_events, sizeof(num_events)); if (result != sizeof(num_events)) { - LOG(WARNING) << StringPrintf("event_fd=%d. Read failed (%d).", event_fd, + LOG(WARNING) << StringPrintf("event_fd=%d. Read failed (%d).", event_fd_, result); break; } VLOG(5) << StringPrintf( - "event_fd=%d. Monitor thread got num_events=%" PRId64 ".", event_fd, + "event_fd=%d. Monitor thread got num_events=%" PRId64 ".", event_fd_, num_events); if (IsEnabled()) { for (int i = 0; i < num_events; ++i) { @@ -90,7 +82,12 @@ void KernelEvent::Monitor(int event_fd, const Handler& handler) { } } - VLOG(5) << StringPrintf("event_fd=%d. Monitor thread exit.", event_fd); + VLOG(5) << StringPrintf("event_fd=%d. Monitor thread exit.", event_fd_); +} + +bool KernelEventLinux::IsEnabled() const { + StdMutexLock lock(&mutex_); + return enabled_; } } // namespace driver diff --git a/driver/kernel/linux/kernel_event_linux.h b/driver/kernel/linux/kernel_event_linux.h new file mode 100644 index 0000000..def436f --- /dev/null +++ b/driver/kernel/linux/kernel_event_linux.h @@ -0,0 +1,66 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_KERNEL_LINUX_KERNEL_EVENT_LINUX_H_ +#define DARWINN_DRIVER_KERNEL_LINUX_KERNEL_EVENT_LINUX_H_ + +#include // NOLINT +#include // NOLINT + +#include "driver/kernel/kernel_event.h" +#include "port/fileio.h" +#include "port/thread_annotations.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +// Monitors events generated through eventfd. The eventfd file +// descriptor passed through the constructor must already be open +// and associated with an event source. Monitoring starts +// on instance creation and stops on destroy. +class KernelEventLinux : public KernelEvent { + public: + KernelEventLinux(FileDescriptor event_fd, Handler handler); + ~KernelEventLinux() override; + + // This class is neither copyable nor movable. + KernelEventLinux(const KernelEvent&) = delete; + KernelEventLinux& operator=(const KernelEvent&) = delete; + + private: + // Monitors eventfd_. Runs on thread_. + void Monitor(const Handler& handler); + + // Convenience function to read |enabled_| with locks held. + bool IsEnabled() const LOCKS_EXCLUDED(mutex_); + + // Event fd. + const FileDescriptor event_fd_; + + // Mutex that guards enabled_; + mutable std::mutex mutex_; + + // Enabled if true. + bool enabled_ GUARDED_BY(mutex_){true}; + + // Thread for monitoring interrupts. + std::thread thread_; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_KERNEL_LINUX_KERNEL_EVENT_LINUX_H_ diff --git a/driver/kernel/linux/kernel_registers_linux.cc b/driver/kernel/linux/kernel_registers_linux.cc new file mode 100644 index 0000000..5402854 --- /dev/null +++ b/driver/kernel/linux/kernel_registers_linux.cc @@ -0,0 +1,71 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/kernel/linux/kernel_registers_linux.h" + +#include +#include + +#include "port/errors.h" +#include "port/integral_types.h" +#include "port/status.h" +#include "port/statusor.h" +#include "port/stringprintf.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +KernelRegistersLinux::KernelRegistersLinux( + const std::string& device_path, const std::vector& mmap_region, + bool read_only) + : KernelRegisters(device_path, mmap_region, read_only) {} + +KernelRegistersLinux::KernelRegistersLinux(const std::string& device_path, + uint64 mmap_offset, uint64 mmap_size, + bool read_only) + : KernelRegisters(device_path, mmap_offset, mmap_size, read_only) {} + +KernelRegistersLinux::~KernelRegistersLinux() { UnmapAllRegions(); } + +util::StatusOr KernelRegistersLinux::MapRegion( + FileDescriptor fd, const MappedRegisterRegion& region, bool read_only) { + int protections = PROT_READ | PROT_WRITE; + if (read_only) { + protections = PROT_READ; + } + + void* result = mmap(nullptr, region.size, protections, + MAP_SHARED, fd, region.offset); + if (result == MAP_FAILED) { + return util::InternalError( + StringPrintf("Could not mmap: %s", strerror(errno))); + } + + return static_cast(result); +} + +util::Status KernelRegistersLinux::UnmapRegion( + FileDescriptor fd, const MappedRegisterRegion& region) { + const int ret = munmap(region.registers, region.size); + if (ret != 0) { + return util::InternalError( + StringPrintf("Error unmapping registers: %s", strerror(errno))); + } + return util::OkStatus(); +} + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/kernel/linux/kernel_registers_linux.h b/driver/kernel/linux/kernel_registers_linux.h new file mode 100644 index 0000000..2c8257e --- /dev/null +++ b/driver/kernel/linux/kernel_registers_linux.h @@ -0,0 +1,51 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_KERNEL_LINUX_KERNEL_REGISTERS_LINUX_H_ +#define DARWINN_DRIVER_KERNEL_LINUX_KERNEL_REGISTERS_LINUX_H_ + +#include // NOLINT +#include +#include + +#include "driver/kernel/kernel_registers.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +// Kernel implementation of the register interface. +class KernelRegistersLinux : public KernelRegisters { + public: + KernelRegistersLinux(const std::string& device_path, + const std::vector& mmap_region, + bool read_only); + + KernelRegistersLinux(const std::string& device_path, uint64 mmap_offset, + uint64 mmap_size, bool read_only); + ~KernelRegistersLinux() override; + + protected: + util::StatusOr MapRegion(FileDescriptor fd, + const MappedRegisterRegion& region, + bool read_only) override; + util::Status UnmapRegion(FileDescriptor fd, + const MappedRegisterRegion& region) override; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_KERNEL_LINUX_KERNEL_REGISTERS_LINUX_H_ diff --git a/driver/kernel/windows/BUILD b/driver/kernel/windows/BUILD new file mode 100644 index 0000000..ed9023a --- /dev/null +++ b/driver/kernel/windows/BUILD @@ -0,0 +1,113 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: +# Kernel driver specific functionality. + +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +libedgetpu_cc_library( + name = "kernel_event_windows", + srcs = ["kernel_event_windows.cc"], + hdrs = ["kernel_event_windows.h"], + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [ + ":gasket_ioctl_windows", + "//driver/kernel:gasket_ioctl", + "//driver/kernel:kernel_event", + "//port", + "//port:fileio", + "//port:std_mutex_lock", + "//port:thread_annotations", + "//port:tracing", + ], +) + +libedgetpu_cc_library( + name = "kernel_event_handler_windows", + srcs = ["kernel_event_handler_windows.cc"], + hdrs = ["kernel_event_handler_windows.h"], + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [ + ":gasket_ioctl_windows", + "//driver/kernel:gasket_ioctl", + "//driver/kernel:kernel_event", + "//driver/kernel:kernel_event_handler", + "//driver/kernel/windows:kernel_event_windows", + "//port", + "//port:fileio", + "//port:std_mutex_lock", + "//port:thread_annotations", + "//port:tracing", + ], +) + +libedgetpu_cc_library( + name = "kernel_coherent_allocator_windows", + srcs = ["kernel_coherent_allocator_windows.cc"], + hdrs = ["kernel_coherent_allocator_windows.h"], + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [ + ":gasket_ioctl_windows", + "//driver:hardware_structures", + "//driver/kernel:gasket_ioctl", + "//driver/kernel:kernel_coherent_allocator", + "//driver/mmio:coherent_allocator", + "//port", + "//port:fileio", + ], +) + +libedgetpu_cc_library( + name = "kernel_registers_windows", + srcs = ["kernel_registers_windows.cc"], + hdrs = ["kernel_registers_windows.h"], + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [ + ":gasket_ioctl_windows", + "@com_google_absl//absl/strings:str_format", + "//driver/kernel:gasket_ioctl", + "//driver/kernel:kernel_registers", + "//driver/registers", + "//port", + "//port:fileio", + "//port:std_mutex_lock", + "//port:thread_annotations", + ], +) + +libedgetpu_cc_library( + name = "gasket_ioctl_windows", + hdrs = ["windows_gasket_ioctl.inc"], +) diff --git a/driver/kernel/windows/kernel_coherent_allocator_windows.cc b/driver/kernel/windows/kernel_coherent_allocator_windows.cc new file mode 100644 index 0000000..8b4b2fb --- /dev/null +++ b/driver/kernel/windows/kernel_coherent_allocator_windows.cc @@ -0,0 +1,62 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/kernel/windows/kernel_coherent_allocator_windows.h" + +#include "driver/kernel/gasket_ioctl.h" +#include "port/errors.h" +#include "port/logging.h" +#include "port/statusor.h" +#include "port/stringprintf.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +KernelCoherentAllocatorWindows::KernelCoherentAllocatorWindows( + const std::string &device_path, int alignment_bytes, size_t size_bytes) + : KernelCoherentAllocator(device_path, alignment_bytes, size_bytes) {} + +util::StatusOr KernelCoherentAllocatorWindows::Map(FileDescriptor fd, + size_t size_bytes, + uint64 dma_address) { + gasket_address_map_ioctl apex_memmap_ioctl; + memset(&apex_memmap_ioctl, 0x00, sizeof(apex_memmap_ioctl)); + apex_memmap_ioctl.dev_dma_addr = dma_address; + apex_memmap_ioctl.size = size_bytes; + int rc = ioctl(fd, GASKET_IOCTL_MAP_UMDMA_VIEW, &apex_memmap_ioctl); + if (rc != 0) { + return util::FailedPreconditionError( + StringPrintf("CoherentAllocator Could not map size %zu.", size_bytes)); + } + return (char *)apex_memmap_ioctl.virtaddr; +} + +util::Status KernelCoherentAllocatorWindows::Unmap(FileDescriptor fd, + char *mem_base, + size_t size_bytes) { + gasket_address_map_ioctl apex_memmap_ioctl; + memset(&apex_memmap_ioctl, 0x00, sizeof(apex_memmap_ioctl)); + apex_memmap_ioctl.virtaddr = reinterpret_cast(mem_base); + int rc = ioctl(fd, GASKET_IOCTL_UNMAP_UMDMA_VIEW, &apex_memmap_ioctl); + if (rc != 0) { + return util::FailedPreconditionError(StringPrintf( + "CoherentAllocator Could not unmap coherent %p.", mem_base)); + } + return util::Status(); // OK +} + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/kernel/windows/kernel_coherent_allocator_windows.h b/driver/kernel/windows/kernel_coherent_allocator_windows.h new file mode 100644 index 0000000..d07b785 --- /dev/null +++ b/driver/kernel/windows/kernel_coherent_allocator_windows.h @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_COHERENT_ALLOCATOR_WINDOWS_H_ +#define DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_COHERENT_ALLOCATOR_WINDOWS_H_ + +#include "driver/kernel/kernel_coherent_allocator.h" +#include "port/fileio.h" +#include "port/statusor.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +// Functions to allocate coherent memory that is DMA-able by a Darwinn device. +class KernelCoherentAllocatorWindows : public KernelCoherentAllocator { + public: + KernelCoherentAllocatorWindows(const std::string &device_path, + int alignment_bytes, size_t size_bytes); + + private: + // Maps and unmaps kernel allocated memory block to user space. + util::StatusOr Map(FileDescriptor fd, size_t size_bytes, + uint64 dma_address) override; + util::Status Unmap(FileDescriptor fd, char *mem_base, + size_t size_bytes) override; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_COHERENT_ALLOCATOR_WINDOWS_H_ diff --git a/driver/kernel/windows/kernel_event_handler_windows.cc b/driver/kernel/windows/kernel_event_handler_windows.cc new file mode 100644 index 0000000..1ddcf06 --- /dev/null +++ b/driver/kernel/windows/kernel_event_handler_windows.cc @@ -0,0 +1,119 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/kernel/windows/kernel_event_handler_windows.h" + +#include + +#include + +#include "driver/kernel/gasket_ioctl.h" +#include "driver/kernel/windows/kernel_event_windows.h" +#include "port/errors.h" +#include "port/ptr_util.h" +#include "port/status.h" +#include "port/status_macros.h" +#include "port/stringprintf.h" + +namespace platforms { +namespace darwinn { +namespace driver { +namespace { + +std::wstring EventName(const std::string& device_path, int event_id) { + size_t index = device_path.find(APEX_DEVICE_NAME_BASE); + if (index == std::string::npos) { + LOG(ERROR) << "Unexpected device name " << device_path; + return std::wstring(); + } + std::string name = + StringPrintf("%sEvent%d", &device_path.c_str()[index], event_id); + return std::wstring(name.begin(), name.end()); +} + +} // namespace + +KernelEventHandlerWindows::KernelEventHandlerWindows( + const std::string& device_path, int num_events) + : KernelEventHandler(device_path, num_events) {} + +util::Status KernelEventHandlerWindows::SetEventFd(FileDescriptor fd, + FileDescriptor event_fd, + int event_id) const { + gasket_set_event_ioctl gasket_set_event; + gasket_set_event.int_num = event_id; + std::wstring event_name = EventName(GetDevicePath(), event_id); + wcscpy(gasket_set_event.event_name, event_name.c_str()); + + if (!DeviceIoControl(fd, GASKET_IOCTL_SET_EVENTFD, &gasket_set_event, + sizeof(gasket_set_event), NULL, 0, NULL, NULL)) { + return util::InternalError( + StringPrintf("Setting Interrupt event failed: event_id:%d gle=%d", + event_id, GetLastError())); + } + + VLOG(5) << StringPrintf("Set event fd : event_id:%d -> event_fd:%p, ", + event_id, event_fd); + + return util::OkStatus(); +} + +FileDescriptor KernelEventHandlerWindows::InitializeEventFd( + int event_id) const { + std::wstring wc_event_name = EventName(GetDevicePath(), event_id); + FileDescriptor event_fd = + CreateEventW(NULL, TRUE, FALSE, wc_event_name.c_str()); + if (event_fd == INVALID_FD_VALUE) { + LOG(ERROR) << "Create event failed: gle=" << GetLastError(); + } + return event_fd; +} + +util::Status KernelEventHandlerWindows::ReleaseEventFd(FileDescriptor fd, + FileDescriptor event_fd, + int event_id) const { + if (fd == INVALID_FD_VALUE) { + return util::FailedPreconditionError("Device not open."); + } + + gasket_set_event_ioctl gasket_set_event; + gasket_set_event.int_num = event_id; + std::wstring event_name = EventName(GetDevicePath(), event_id); + wcscpy(gasket_set_event.event_name, event_name.c_str()); + + if (!DeviceIoControl(fd, GASKET_IOCTL_CLEAR_EVENTFD, &gasket_set_event, + sizeof(gasket_set_event), NULL, 0, NULL, NULL)) { + return util::InternalError( + StringPrintf("Clearing Interrupt event failed: event_id:%d gle=%d", + event_id, GetLastError())); + } + + BOOL result = CloseHandle(event_fd); + if (!result) { + return util::InternalError( + StringPrintf("Close Int Event failed: event_id:%d gle=%d", event_id, + GetLastError())); + } + + return util::OkStatus(); +} + +std::unique_ptr KernelEventHandlerWindows::CreateKernelEvent( + FileDescriptor event_fd, KernelEvent::Handler handler) { + return gtl::MakeUnique(event_fd, std::move(handler)); +} + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/kernel/windows/kernel_event_handler_windows.h b/driver/kernel/windows/kernel_event_handler_windows.h new file mode 100644 index 0000000..211a9b8 --- /dev/null +++ b/driver/kernel/windows/kernel_event_handler_windows.h @@ -0,0 +1,52 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_EVENT_HANDLER_WINDOWS_H_ +#define DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_EVENT_HANDLER_WINDOWS_H_ + +#include "driver/kernel/kernel_event_handler.h" +#include "port/fileio.h" +#include "port/status.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +// Implements a mechanism for processing kernel events. +class KernelEventHandlerWindows : public KernelEventHandler { + public: + KernelEventHandlerWindows(const std::string& device_path, int num_events); + + private: + // Maps the specified event number with the specified id. + util::Status SetEventFd(FileDescriptor fd, FileDescriptor event_fd, + int event_id) const override; + + // Performs platform specific event object handle initialization + FileDescriptor InitializeEventFd(int event_id) const override; + + // Releases platform specific resources associated with event object handle + util::Status ReleaseEventFd(FileDescriptor fd, FileDescriptor event_fd, + int event_id) const override; + + // Creates platform specific KernelEvent backing object + std::unique_ptr CreateKernelEvent( + FileDescriptor event_fd, KernelEvent::Handler handler) override; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_EVENT_HANDLER_WINDOWS_H_ diff --git a/driver/kernel/windows/kernel_event_windows.cc b/driver/kernel/windows/kernel_event_windows.cc new file mode 100644 index 0000000..4c20dc1 --- /dev/null +++ b/driver/kernel/windows/kernel_event_windows.cc @@ -0,0 +1,98 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/kernel/windows/kernel_event_windows.h" + +#include +#include + +#include "port/errors.h" +#include "port/integral_types.h" +#include "port/status.h" +#include "port/status_macros.h" +#include "port/std_mutex_lock.h" +#include "port/stringprintf.h" +#include "port/tracing.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +KernelEventWindows::KernelEventWindows(FileDescriptor event_fd, Handler handler) + : KernelEvent(event_fd, handler), + event_fd_(event_fd) { + std::thread event_thread(&KernelEventWindows::Monitor, this, + std::move(handler)); + thread_ = std::move(event_thread); +} + +KernelEventWindows::~KernelEventWindows() { + // Mark as disabled. + { + StdMutexLock lock(&mutex_); + enabled_ = false; + } + + // Signal a fake event to force WaitForSingleObject() to return. + BOOL result = SetEvent(event_fd_); + if (!result) { + DWORD gle = GetLastError(); + LOG(WARNING) << StringPrintf("SetEvent failed! event_fd=%p gle=%d", + event_fd_, gle); + } + + // Wait for thread to exit. + thread_.join(); +} + +void KernelEventWindows::Monitor(const Handler& handler) { + VLOG(5) << StringPrintf("event_fd=%p. Monitor thread begin.", event_fd_); + TRACE_START_THREAD("KernelEventHandlerMonitor"); + + while (IsEnabled()) { + // Wait for events (blocking). + DWORD result = WaitForSingleObject(event_fd_, INFINITE); + if (result != WAIT_OBJECT_0) { + VLOG(5) << StringPrintf( + "WaitForSingleObject failed " + "evend_fd=%p result=%d gle=%d", + event_fd_, result, GetLastError()); + break; + } + + BOOL reset = ResetEvent(event_fd_); + if (!reset) { + VLOG(5) << StringPrintf("ResetEvent failed evend_fd=%p gle=%d", event_fd_, + GetLastError()); + break; + } + + VLOG(5) << StringPrintf("event_fd=%p. Monitor thread got event.", + event_fd_); + if (IsEnabled()) { + handler(); + } + } + + VLOG(5) << StringPrintf("event_fd=%p. Monitor thread exit.", event_fd_); +} + +bool KernelEventWindows::IsEnabled() const { + StdMutexLock lock(&mutex_); + return enabled_; +} + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/kernel/windows/kernel_event_windows.h b/driver/kernel/windows/kernel_event_windows.h new file mode 100644 index 0000000..c2b2d1f --- /dev/null +++ b/driver/kernel/windows/kernel_event_windows.h @@ -0,0 +1,66 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_EVENT_WINDOWS_H_ +#define DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_EVENT_WINDOWS_H_ + +#include // NOLINT +#include // NOLINT + +#include "driver/kernel/kernel_event.h" +#include "port/fileio.h" +#include "port/thread_annotations.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +// Monitors events generated through eventfd. The eventfd file +// descriptor passed through the constructor must already be open +// and associated with an event source. Monitoring starts +// on instance creation and stops on destroy. +class KernelEventWindows : public KernelEvent { + public: + KernelEventWindows(FileDescriptor event_fd, Handler handler); + ~KernelEventWindows() override; + + // This class is neither copyable nor movable. + KernelEventWindows(const KernelEvent&) = delete; + KernelEventWindows& operator=(const KernelEvent&) = delete; + + private: + // Monitors eventfd_. Runs on thread_. + void Monitor(const Handler& handler); + + // Convenience function to read |enabled_| with locks held. + bool IsEnabled() const LOCKS_EXCLUDED(mutex_); + + // Event fd. + const FileDescriptor event_fd_; + + // Mutex that guards enabled_; + mutable std::mutex mutex_; + + // Enabled if true. + bool enabled_ GUARDED_BY(mutex_){true}; + + // Thread for monitoring interrupts. + std::thread thread_; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_EVENT_WINDOWS_H_ diff --git a/driver/kernel/windows/kernel_registers_windows.cc b/driver/kernel/windows/kernel_registers_windows.cc new file mode 100644 index 0000000..fb448fc --- /dev/null +++ b/driver/kernel/windows/kernel_registers_windows.cc @@ -0,0 +1,83 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "driver/kernel/windows/kernel_registers_windows.h" + +#include +#include +#include +#include +#include + +#include "driver/kernel/gasket_ioctl.h" +#include "driver/registers/registers.h" +#include "port/errors.h" +#include "port/integral_types.h" +#include "port/status.h" +#include "port/status_macros.h" +#include "port/statusor.h" +#include "port/std_mutex_lock.h" +#include "port/stringprintf.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +KernelRegistersWindows::KernelRegistersWindows( + const std::string& device_path, const std::vector& mmap_region, + bool read_only) + : KernelRegisters(device_path, mmap_region, read_only) {} + +KernelRegistersWindows::KernelRegistersWindows(const std::string& device_path, + uint64 mmap_offset, + uint64 mmap_size, bool read_only) + : KernelRegisters(device_path, mmap_offset, mmap_size, read_only) {} + +KernelRegistersWindows::~KernelRegistersWindows() { UnmapAllRegions(); } + +util::StatusOr KernelRegistersWindows::MapRegion( + FileDescriptor fd, const MappedRegisterRegion& region, bool read_only) { + gasket_address_map_ioctl ioctl = {0, region.offset, region.size, 0, + 0, nullptr}; + bool res; + + res = DeviceIoControl(fd, GASKET_IOCTL_MAP_HDW_VIEW, &ioctl, sizeof(ioctl), + &ioctl, sizeof(ioctl), NULL, NULL); + if (!res) { + return util::InternalError(StringPrintf( + "KernelRegisters::MapRegion failed! gle=%d", GetLastError())); + } + + return static_cast(ioctl.virtaddr); +} + +util::Status KernelRegistersWindows::UnmapRegion( + FileDescriptor fd, const MappedRegisterRegion& region) { + gasket_address_map_ioctl ioctl = {0, region.offset, region.size, 0, + 0, region.registers}; + bool res; + + res = DeviceIoControl(fd, GASKET_IOCTL_UNMAP_HDW_VIEW, &ioctl, sizeof(ioctl), + &ioctl, sizeof(ioctl), NULL, NULL); + if (!res) { + util::FailedPreconditionError(StringPrintf( + "KernelRegisters::UnmapRegion failed! gle=%d", GetLastError())); + } + + return util::OkStatus(); +} + +} // namespace driver +} // namespace darwinn +} // namespace platforms diff --git a/driver/kernel/windows/kernel_registers_windows.h b/driver/kernel/windows/kernel_registers_windows.h new file mode 100644 index 0000000..60d1fac --- /dev/null +++ b/driver/kernel/windows/kernel_registers_windows.h @@ -0,0 +1,51 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_REGISTERS_WINDOWS_H_ +#define DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_REGISTERS_WINDOWS_H_ + +#include // NOLINT +#include +#include + +#include "driver/kernel/kernel_registers.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +// Kernel implementation of the register interface. +class KernelRegistersWindows : public KernelRegisters { + public: + KernelRegistersWindows(const std::string& device_path, + const std::vector& mmap_region, + bool read_only); + + KernelRegistersWindows(const std::string& device_path, uint64 mmap_offset, + uint64 mmap_size, bool read_only); + ~KernelRegistersWindows() override; + + protected: + util::StatusOr MapRegion(FileDescriptor fd, + const MappedRegisterRegion& region, + bool read_only) override; + util::Status UnmapRegion(FileDescriptor fd, + const MappedRegisterRegion& region) override; +}; + +} // namespace driver +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_KERNEL_WINDOWS_KERNEL_REGISTERS_WINDOWS_H_ diff --git a/driver/kernel/windows/windows_gasket_ioctl.inc b/driver/kernel/windows/windows_gasket_ioctl.inc new file mode 100644 index 0000000..8e6d54e --- /dev/null +++ b/driver/kernel/windows/windows_gasket_ioctl.inc @@ -0,0 +1,168 @@ +/* Common Gasket device kernel and user space declarations. */ +#ifndef __WINDOWS_GASKET_IOCTL_H__ +#define __WINDOWS_GASKET_IOCTL_H__ + +#ifndef __GASKET_IOCTL_H__ +#error This file should not be included directly include gasket_ioctl.h instead +#endif + +#ifndef __KERNEL__ +#include "port/fileio.h" +#include +#endif + +#ifndef MAX_PATH +#define MAX_PATH 260 +#endif + +#define CORAL_DOS_DRIVER_NAME L"\\\\?\\ApexDriver" +#define APEX_DEVICE_NAME_BASE "ApexDevice" +#define APEX_MAX_DEVICES 256 // arbitrary + +#define APEX_DRIVER_TYPE 43000 +#define APEX_DRIVER_IOCTL_ENUM_BASE 2833 + +/* + * Common structure for ioctls associating an eventfd with a device interrupt, + * when using the Gasket interrupt module. + */ +struct gasket_set_event_ioctl { + uint64_t int_num; + WCHAR event_name[MAX_PATH]; +}; + +/* + * Common structure for ioctls mapping or unmapping kernel address space into + * user space. + */ +struct gasket_address_map_ioctl { + uint64_t base_addr; + uint64_t offset; + uint64_t size; + uint64_t dev_dma_addr; + uint32_t flags; + uint64_t* virtaddr; +}; + +// Device type -- in the "User Defined" range." +#define GASKET_TYPE 43000 +#define WINDOWS_GASKET_IOCTL_BASE 2833 + +#define _CTL_CODE(base, nr, buffering, access) \ + CTL_CODE(GASKET_TYPE, WINDOWS_GASKET_IOCTL_BASE + (base) + (nr), buffering, \ + access) + +#define _IO(type, nr) _CTL_CODE(type, nr, METHOD_NEITHER, FILE_ANY_ACCESS) +#define _IOW(type, nr, size) \ + _CTL_CODE(type, nr, METHOD_BUFFERED, FILE_WRITE_ACCESS) +#define _IOR(type, nr, size) \ + _CTL_CODE(type, nr, METHOD_BUFFERED, FILE_WRITE_ACCESS) +#define _IOWR(type, nr, size) \ + _CTL_CODE(type, nr, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS) + +// Ioctl numbers "nr" should not overlap with ones defined in +// linux_gasket_ioctl.h + + +/* + * Tells the kernel to map "size" bytes at offset "offset" from device BAR2 + * returns user mode virtual address in "virtaddr". + */ +#define GASKET_IOCTL_MAP_HDW_VIEW \ + _IOWR(GASKET_IOCTL_BASE, 14, struct gasket_address_map_ioctl) + +/* + * Unmaps device BAR2 region previously mapped to user mode virtual address in + * "virtaddr". + */ +#define GASKET_IOCTL_UNMAP_HDW_VIEW \ + _IOWR(GASKET_IOCTL_BASE, 15, struct gasket_address_map_ioctl) + +/* + * Tells the kernel to map "size" bytes of kernel allocated common buffer which + * was previously mapped to device's "dev_dma_addr" address. + * Returns user mode virtual address in "virtaddr". + */ +#define GASKET_IOCTL_MAP_UMDMA_VIEW \ + _IOWR(GASKET_IOCTL_BASE, 16, struct gasket_address_map_ioctl) + +/* + * Unmaps kernel common buffer mapped to "virtaddr" user mode virtual address. + */ +#define GASKET_IOCTL_UNMAP_UMDMA_VIEW \ + _IOWR(GASKET_IOCTL_BASE, 17, struct gasket_address_map_ioctl) + + +#ifndef __KERNEL__ + +#include "port/logging.h" + +namespace platforms { +namespace darwinn { +namespace driver { + +template +int ioctl(FileDescriptor fd, ULONG ctlcode, IPT* ipt) { + ULONG access = (ctlcode >> 14) & 0x00000003; + ULONG transfer_type = ctlcode & 0x00000003; + // + BOOL rc = 0; + + if (transfer_type != METHOD_BUFFERED) { + VLOG(1) << StringPrintf( + "Parameter error: Windows ioctl(x%X): invalid transfer-type", ctlcode); + return -EINVAL; + } + + if (access == FILE_ANY_ACCESS) { + // No data transfer: we should not have a buffer + if ((ipt != NULL) || (sizeof(*ipt) != 0)) { + VLOG(1) << StringPrintf( + "Parameter error: Windows ioctl(x%X): no file access with a buffer", + ctlcode); + return -EINVAL; + } + } else { + // We should have a buffer + if ((ipt == NULL) || (sizeof(*ipt) == 0)) { + VLOG(1) << StringPrintf( + "Parameter error: Windows ioctl(x%X): file access without a buffer", + ctlcode); + return -EINVAL; + } + } + + switch (access) { + case FILE_ANY_ACCESS: + rc = DeviceIoControl(fd, ctlcode, NULL, 0, NULL, 0, NULL, NULL); + break; + + case FILE_WRITE_DATA: + rc = DeviceIoControl(fd, ctlcode, ipt, sizeof(*ipt), NULL, 0, NULL, NULL); + break; + + case FILE_READ_DATA: + rc = DeviceIoControl(fd, ctlcode, NULL, 0, ipt, sizeof(*ipt), NULL, NULL); + break; + + case (FILE_WRITE_DATA | FILE_READ_DATA): + rc = DeviceIoControl(fd, ctlcode, ipt, sizeof(*ipt), ipt, sizeof(*ipt), + NULL, NULL); + } + + if (rc == 0) { + DWORD gle = GetLastError(); + VLOG(1) << StringPrintf( + "Execute error: Windows ioctl(x%X): gle=%d strerror=%s", ctlcode, gle, + strerror(errno)); + } + + return (rc != 0 /*good result*/) ? 0 /*linux-style no error*/ : errno; +} + +} // namespace driver +} // namespace darwinn +} // namespace platforms +#endif // ifndef __KERNEL__ + +#endif /* __WINDOWS_GASKET_IOCTL_H__ */ diff --git a/driver/memory/BUILD b/driver/memory/BUILD index 603852d..9b310af 100644 --- a/driver/memory/BUILD +++ b/driver/memory/BUILD @@ -15,11 +15,13 @@ # Description: # Memory management related functionality. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "address_utilities", hdrs = ["address_utilities.h"], deps = [ @@ -28,7 +30,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "mmu_mapper", srcs = ["mmu_mapper.cc"], hdrs = ["mmu_mapper.h"], @@ -42,7 +44,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "address_space", hdrs = ["address_space.h"], deps = [ @@ -53,7 +55,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "mmio_address_space", srcs = ["mmio_address_space.cc"], hdrs = ["mmio_address_space.h"], @@ -70,7 +72,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "buddy_address_space", srcs = ["buddy_address_space.cc"], hdrs = ["buddy_address_space.h"], @@ -88,13 +90,13 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "address_space_allocator", hdrs = ["address_space_allocator.h"], deps = ["//port"], ) -cc_library( +libedgetpu_cc_library( name = "buddy_allocator", srcs = ["buddy_allocator.cc"], hdrs = ["buddy_allocator.h"], @@ -108,7 +110,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "fake_mmu_mapper", srcs = ["fake_mmu_mapper.cc"], hdrs = ["fake_mmu_mapper.h"], @@ -123,14 +125,14 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "dma_direction", hdrs = ["dma_direction.h"], deps = [ ], ) -cc_library( +libedgetpu_cc_library( name = "nop_address_space", srcs = ["nop_address_space.cc"], hdrs = ["nop_address_space.h"], @@ -143,7 +145,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "dual_address_space", srcs = ["dual_address_space.cc"], hdrs = ["dual_address_space.h"], @@ -157,7 +159,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "dram_allocator", hdrs = ["dram_allocator.h"], deps = [ @@ -166,7 +168,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "null_dram_allocator", hdrs = ["null_dram_allocator.h"], deps = [ @@ -176,7 +178,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "fake_dram_allocator", srcs = ["fake_dram_allocator.cc"], hdrs = ["fake_dram_allocator.h"], diff --git a/driver/mmio/BUILD b/driver/mmio/BUILD index 8a986b1..d3f6794 100644 --- a/driver/mmio/BUILD +++ b/driver/mmio/BUILD @@ -15,11 +15,13 @@ # Description: # Memory Mapped IO (ex: PCIe.) +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "coherent_allocator", srcs = ["coherent_allocator.cc"], hdrs = ["coherent_allocator.h"], @@ -31,7 +33,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "host_queue", hdrs = ["host_queue.h"], deps = [ diff --git a/driver/mmio/host_queue.h b/driver/mmio/host_queue.h index 94df964..d54ea2e 100644 --- a/driver/mmio/host_queue.h +++ b/driver/mmio/host_queue.h @@ -92,6 +92,13 @@ class HostQueue { // element, invoke registered callback. void ProcessStatusBlock() LOCKS_EXCLUDED(queue_mutex_); + // Process status block only if the host queue is open. This is only needed to + // work around an interrupt race condition in the Darwinn 1.0 stack. + // See: https://b.corp.google.com/issues/159997870#comment44 + // TODO: Remove this work-around once the DV team has fully + // transitioned to the 2.0 stack for testing. + void ProcessStatusBlockIfOpen() LOCKS_EXCLUDED(open_mutex_, queue_mutex_); + // Return available space in the queue. virtual int GetAvailableSpace() const LOCKS_EXCLUDED(queue_mutex_) { StdMutexLock lock(&queue_mutex_); @@ -427,6 +434,47 @@ void HostQueue::ProcessStatusBlock() { } } +template +void HostQueue::ProcessStatusBlockIfOpen() { + StdMutexLock lock(&open_mutex_); + if (!open_) { + return; + } + + // Note: The logic from ProcessStatusBlock must be duplicated here as this + // does not require the instruction queue interrupt status to be cleared, + // whereas ProcessStatusBlock does. + // + // Other than clearing the instruction queue status, this should be identical + // to ProcessStatusBlock. + StdMutexLock callback_lock(&callback_mutex_); + int completed = 0; + + StatusBlock status_block = *status_block_; + const int completed_until = status_block.completed_head_pointer; + const uint32 error_status = status_block.fatal_error; + + std::vector> dones; + { + StdMutexLock lock(&queue_mutex_); + while (completed_head_ != completed_until) { + ++completed; + + if (callbacks_[completed_head_]) { + dones.push_back(std::move(callbacks_[completed_head_])); + } + ++completed_head_; + completed_head_ &= (size_ - 1); + } + VLOG(3) << "Completed " << completed << " elements."; + } + + // Perform callbacks. + for (const auto& done : dones) { + done(error_status); + } +} + } // namespace driver } // namespace darwinn } // namespace platforms diff --git a/driver/mmio_driver.cc b/driver/mmio_driver.cc index 1213538..d2768ed 100644 --- a/driver/mmio_driver.cc +++ b/driver/mmio_driver.cc @@ -37,9 +37,10 @@ #include "driver/mmio/host_queue.h" #include "driver/package_registry.h" #include "driver/single_tpu_request.h" -#include "driver/time_stamper/driver_time_stamper.h" #include "driver/top_level_handler.h" #include "driver/tpu_request.h" +#include "driver_shared/time_stamper/driver_time_stamper.h" +#include "driver_shared/time_stamper/time_stamper.h" #include "executable/executable_generated.h" #include "port/cleanup.h" #include "port/errors.h" @@ -82,7 +83,7 @@ MmioDriver::MmioDriver( std::unique_ptr run_controller, std::unique_ptr top_level_handler, std::unique_ptr executable_registry, - std::unique_ptr time_stamper) + std::unique_ptr time_stamper) : Driver( [](config::ChipConfig* chip_config) { CHECK(chip_config != nullptr); @@ -111,7 +112,7 @@ MmioDriver::MmioDriver( dma_scheduler_(api::Watchdog::MakeWatchdog( driver_options.watchdog_timeout_ns(), [this](int64) { HandleWatchdogTimeout(); }), - gtl::MakeUnique()), + gtl::MakeUnique()), chip_config_(std::move(chip_config)) {} MmioDriver::~MmioDriver() { @@ -318,10 +319,14 @@ util::Status MmioDriver::DoOpen(bool debug_mode) { RETURN_IF_ERROR(run_controller_->DoRunControl(RunControl::kMoveToRun)); } - // Disable periodic status block updates. // TODO: refactor for Darwinn 1.0 vs 2.0 driver. - RETURN_IF_ERROR( - registers_->Write(hib_user_csr_offsets_.status_block_update, 0)); + + if (hib_user_csr_offsets_.status_block_update != + kCsrRegisterSpaceInvalidOffset) { + // Disable periodic status block updates. + RETURN_IF_ERROR( + registers_->Write(hib_user_csr_offsets_.status_block_update, 0)); + } // Register and enable all interrupts. RETURN_IF_ERROR(RegisterAndEnableAllInterrupts()); diff --git a/driver/mmio_driver.h b/driver/mmio_driver.h index f80501e..5deb04c 100644 --- a/driver/mmio_driver.h +++ b/driver/mmio_driver.h @@ -45,9 +45,9 @@ #include "driver/registers/registers.h" #include "driver/run_controller.h" #include "driver/scalar_core_controller.h" -#include "driver/time_stamper/time_stamper.h" #include "driver/top_level_handler.h" #include "driver/tpu_request.h" +#include "driver_shared/time_stamper/time_stamper.h" #include "executable/executable_generated.h" #include "port/integral_types.h" #include "port/statusor.h" @@ -79,7 +79,7 @@ class MmioDriver : public Driver { std::unique_ptr run_controller, std::unique_ptr top_level_handler, std::unique_ptr executable_registry, - std::unique_ptr time_stamper); + std::unique_ptr time_stamper); // This class is neither copyable nor movable. MmioDriver(const MmioDriver&) = delete; diff --git a/driver/real_time_dma_scheduler.h b/driver/real_time_dma_scheduler.h index 7d0ece2..6c0e99f 100644 --- a/driver/real_time_dma_scheduler.h +++ b/driver/real_time_dma_scheduler.h @@ -31,8 +31,8 @@ #include "driver/dma_scheduler.h" #include "driver/package_registry.h" #include "driver/single_queue_dma_scheduler.h" -#include "driver/time_stamper/time_stamper.h" #include "driver/tpu_request.h" +#include "driver_shared/time_stamper/time_stamper.h" #include "port/errors.h" #include "port/ptr_util.h" #include "port/status.h" @@ -49,7 +49,7 @@ class RealTimeDmaScheduler : public DmaScheduler { public: RealTimeDmaScheduler() = delete; RealTimeDmaScheduler(std::unique_ptr watchdog, - std::unique_ptr time_stamper) + std::unique_ptr time_stamper) : backing_scheduler_( gtl::MakeUnique(std::move(watchdog))), time_stamper_(std::move(time_stamper)) {} @@ -177,7 +177,7 @@ class RealTimeDmaScheduler : public DmaScheduler { std::unique_ptr backing_scheduler_; // Time-stamper for tracking submission and completion time for requests. - std::unique_ptr time_stamper_; + std::unique_ptr time_stamper_; // Tracks registered executables. std::unordered_map diff --git a/driver/registers/BUILD b/driver/registers/BUILD index e2d6031..0fb6840 100644 --- a/driver/registers/BUILD +++ b/driver/registers/BUILD @@ -15,18 +15,23 @@ # Description: # Register access related functionality. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "registers", srcs = ["registers.cc"], hdrs = ["registers.h"], - deps = ["//port"], + deps = [ + "//driver_shared:registers", + "//port", + ], ) -cc_library( +libedgetpu_cc_library( name = "socket_registers", srcs = ["socket_registers.cc"], hdrs = ["socket_registers.h"], diff --git a/driver/registers/registers.h b/driver/registers/registers.h index 6fbb2e6..1704414 100644 --- a/driver/registers/registers.h +++ b/driver/registers/registers.h @@ -15,6 +15,7 @@ #ifndef DARWINN_DRIVER_REGISTERS_REGISTERS_H_ #define DARWINN_DRIVER_REGISTERS_REGISTERS_H_ +#include "driver_shared/registers.h" #include "port/errors.h" #include "port/integral_types.h" #include "port/status.h" @@ -27,22 +28,13 @@ namespace darwinn { namespace driver { // Interface for CSR access. -class Registers { +class Registers : public driver_shared::Registers { public: // To indicate the polling functions should poll forever. static constexpr int64 kInfiniteTimeout = -1; virtual ~Registers() = default; - // Open / Close the register interface. - virtual util::Status Open() = 0; - virtual util::Status Close() = 0; - - // Write / Read from a register at the given 64 bit aligned offset. - // Offset may be implementation dependent. - virtual util::Status Write(uint64 offset, uint64 value) = 0; - virtual util::StatusOr Read(uint64 offset) = 0; - // Polls the specified register until it has the given value. util::Status Poll(uint64 offset, uint64 expected_value) { return Poll(offset, expected_value, kInfiniteTimeout); @@ -56,8 +48,6 @@ class Registers { int64 timeout_us); // 32-bit version of above. Usually, it is same as running 64 bit version. - virtual util::Status Write32(uint64 offset, uint32 value) = 0; - virtual util::StatusOr Read32(uint64 offset) = 0; util::Status Poll32(uint64 offset, uint32 expected_value) { return Poll32(offset, expected_value, kInfiniteTimeout); } diff --git a/driver/request.cc b/driver/request.cc index 2c41b0b..8b27826 100644 --- a/driver/request.cc +++ b/driver/request.cc @@ -15,6 +15,7 @@ #include "driver/request.h" #include "api/request.h" +#include "driver_shared/time_stamper/time_stamper.h" #include "port/math_util.h" #include "port/status.h" #include "port/status_macros.h" @@ -27,16 +28,16 @@ namespace platforms { namespace darwinn { namespace driver { Request::Request(int id, const PackageReference& package_ref, - const TimeStamper& timestamper) + const driver_shared::TimeStamper& timestamper) : id_(id), package_ref_(package_ref), main_executable_ref_(*package_ref.MainExecutableReference()), hardware_batch_size_(package_ref.MainExecutableReference()->BatchSize()), current_time_(timestamper) { - timing_.created_ns = timestamper.GetTimeNanoSeconds(); - timing_.submitted_ns = -1; - timing_.completed_ns = -1; - } + timing_.created_ns = timestamper.GetTimeNanoSeconds(); + timing_.submitted_ns = -1; + timing_.completed_ns = -1; +} util::Status Request::AddInput(const std::string& name, const Buffer& input) { StdMutexLock lock(&mutex_); diff --git a/driver/request.h b/driver/request.h index ad6dadc..2d74253 100644 --- a/driver/request.h +++ b/driver/request.h @@ -16,8 +16,8 @@ #define DARWINN_DRIVER_REQUEST_H_ #include "api/request.h" -#include "driver/time_stamper/time_stamper.h" #include "driver/tpu_request.h" +#include "driver_shared/time_stamper/time_stamper.h" #include "port/statusor.h" namespace platforms { @@ -44,7 +44,7 @@ class Request : public api::Request { // Constructs a request provided a unique ID and a reference to the package, // and an interface to get current timestamps in nanoseconds. Request(int id, const PackageReference& package_ref, - const TimeStamper& timestamper); + const driver_shared::TimeStamper& timestamper); // This class is not copyable nor movable. Request(const Request&) = delete; @@ -191,7 +191,7 @@ class Request : public api::Request { util::Status done_status_ GUARDED_BY(mutex_); // Gets the current time in nanoseconds. - const TimeStamper& current_time_; + const driver_shared::TimeStamper& current_time_; // Timing information of this request. Timing timing_; diff --git a/driver/usb/BUILD b/driver/usb/BUILD index 9c266eb..e321753 100644 --- a/driver/usb/BUILD +++ b/driver/usb/BUILD @@ -15,11 +15,13 @@ # Description: # USB driver specific functionality. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "usb_device_interface", hdrs = ["usb_device_interface.h"], deps = ["//port"], @@ -30,7 +32,7 @@ LIBUSB_OPTIONS_SRC = select({ "//conditions:default": ["libusb_options_default.cc"], }) -cc_library( +libedgetpu_cc_library( name = "libusb_options", srcs = LIBUSB_OPTIONS_SRC, hdrs = ["libusb_options.h"], @@ -40,7 +42,7 @@ cc_library( }), ) -cc_library( +libedgetpu_cc_library( name = "libusb_options_no_external_release", srcs = LIBUSB_OPTIONS_SRC, hdrs = ["libusb_options.h"], @@ -51,7 +53,7 @@ cc_library( ) # libUSB is dynamically linked in this version. -cc_library( +libedgetpu_cc_library( name = "local_usb_device", srcs = ["local_usb_device.cc"], hdrs = ["local_usb_device.h"], @@ -69,7 +71,7 @@ cc_library( ) # libUSB is statically linked in this version. -cc_library( +libedgetpu_cc_library( name = "local_usb_device_no_external_release", srcs = ["local_usb_device.cc"], hdrs = ["local_usb_device.h"], @@ -84,7 +86,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "usb_standard_commands", srcs = ["usb_standard_commands.cc"], hdrs = ["usb_standard_commands.h"], @@ -95,7 +97,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "usb_dfu_commands", srcs = ["usb_dfu_commands.cc"], hdrs = ["usb_dfu_commands.h"], @@ -132,7 +134,7 @@ genrule( """, ) -cc_library( +libedgetpu_cc_library( name = "usb_ml_commands", srcs = ["usb_ml_commands.cc"], hdrs = ["usb_ml_commands.h"], @@ -144,7 +146,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "usb_registers", srcs = ["usb_registers.cc"], hdrs = ["usb_registers.h"], @@ -155,7 +157,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "usb_dfu_util", srcs = ["usb_dfu_util.cc"], hdrs = ["usb_dfu_util.h"], @@ -167,7 +169,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "usb_io_request", srcs = ["usb_io_request.cc"], hdrs = ["usb_io_request.h"], @@ -180,7 +182,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "usb_driver", srcs = [ "usb_driver.cc", @@ -219,7 +221,7 @@ cc_library( "//driver/memory:dram_allocator", "//driver/memory:nop_address_space", "//driver/registers", - "//driver/time_stamper", + "//driver_shared/time_stamper", "//port", "//port:std_mutex_lock", "//port:thread_annotations", diff --git a/driver/usb/apex_latest_multi_ep.bin b/driver/usb/apex_latest_multi_ep.bin old mode 100755 new mode 100644 diff --git a/driver/usb/apex_latest_single_ep.bin b/driver/usb/apex_latest_single_ep.bin old mode 100755 new mode 100644 diff --git a/driver/usb/usb_driver.cc b/driver/usb/usb_driver.cc index 645ee5f..cc3ac6f 100644 --- a/driver/usb/usb_driver.cc +++ b/driver/usb/usb_driver.cc @@ -31,12 +31,12 @@ #include "driver/memory/dram_allocator.h" #include "driver/package_registry.h" #include "driver/single_tpu_request.h" -#include "driver/time_stamper/time_stamper.h" #include "driver/top_level_handler.h" #include "driver/tpu_request.h" #include "driver/usb/usb_dfu_util.h" #include "driver/usb/usb_latest_firmware.h" #include "driver/usb/usb_ml_commands.h" +#include "driver_shared/time_stamper/time_stamper.h" #include "port/cleanup.h" #include "port/errors.h" #include "port/integral_types.h" @@ -135,7 +135,8 @@ UsbDriver::UsbDriver( std::unique_ptr top_level_handler, std::unique_ptr dram_allocator, std::unique_ptr executable_registry, - const UsbDriverOptions& options, std::unique_ptr time_stamper) + const UsbDriverOptions& options, + std::unique_ptr time_stamper) : Driver( [](config::ChipConfig* chip_config) { CHECK(chip_config != nullptr); @@ -189,7 +190,8 @@ UsbDriver::UsbDriver( std::unique_ptr top_level_handler, std::unique_ptr dram_allocator, std::unique_ptr executable_registry, - const UsbDriverOptions& options, std::unique_ptr time_stamper) + const UsbDriverOptions& options, + std::unique_ptr time_stamper) : UsbDriver(driver_options, std::move(chip_config), std::move(registers), std::move(top_level_interrupt_manager), std::move(fatal_error_interrupt_controller), @@ -211,7 +213,8 @@ UsbDriver::UsbDriver( std::unique_ptr top_level_handler, std::unique_ptr dram_allocator, std::unique_ptr executable_registry, - const UsbDriverOptions& options, std::unique_ptr time_stamper) + const UsbDriverOptions& options, + std::unique_ptr time_stamper) : UsbDriver(driver_options, std::move(chip_config), std::move(registers), std::move(top_level_interrupt_manager), std::move(fatal_error_interrupt_controller), diff --git a/driver/usb/usb_driver.h b/driver/usb/usb_driver.h index bfcf0be..7f5761e 100644 --- a/driver/usb/usb_driver.h +++ b/driver/usb/usb_driver.h @@ -188,7 +188,7 @@ class UsbDriver : public Driver { std::unique_ptr dram_allocator, std::unique_ptr executable_registry, const UsbDriverOptions& options, - std::unique_ptr time_stamper); + std::unique_ptr time_stamper); // Constructs a driver instance around a supplied device object. // Note that since no factory is provided, this driver cannot be re-opened @@ -205,7 +205,7 @@ class UsbDriver : public Driver { std::unique_ptr dram_allocator, std::unique_ptr executable_registry, const UsbDriverOptions& options, - std::unique_ptr time_stamper); + std::unique_ptr time_stamper); // This class is neither copyable nor movable. UsbDriver(const UsbDriver&) = delete; @@ -290,7 +290,7 @@ class UsbDriver : public Driver { std::unique_ptr dram_allocator, std::unique_ptr executable_registry, const UsbDriverOptions& options, - std::unique_ptr time_stamper); + std::unique_ptr time_stamper); // Prepares USB device with resets and DFU according to options_. util::Status PrepareUsbDevice(); diff --git a/driver_shared/BUILD b/driver_shared/BUILD new file mode 100644 index 0000000..8f03f1b --- /dev/null +++ b/driver_shared/BUILD @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + +# Darwinn ddriver libraries shared across runtime generations. +package( + default_visibility = ["//visibility:public"], +) + +# Google Owned Code +licenses(["unencumbered"]) + +libedgetpu_cc_library( + name = "registers", + hdrs = ["registers.h"], + deps = ["//port"], +) diff --git a/driver_shared/registers.h b/driver_shared/registers.h new file mode 100644 index 0000000..7abcc3d --- /dev/null +++ b/driver_shared/registers.h @@ -0,0 +1,65 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_DRIVER_SHARED_REGISTERS_H_ +#define DARWINN_DRIVER_SHARED_REGISTERS_H_ + +#include "port/integral_types.h" +#include "port/status.h" +#include "port/statusor.h" + +namespace platforms { +namespace darwinn { + +namespace driver_shared { + +// An interface to device registers. +// TODO Move this to a shared location with 1.0. +class Registers { + public: + Registers() = default; + virtual ~Registers() = default; + + // This class is neither copyable nor movable. + Registers(const Registers&) = delete; + Registers& operator=(const Registers&) = delete; + + // Open / Close the register interface. Any register access can only happen + // when the interface is open. + virtual util::Status Open() = 0; + virtual util::Status Close() = 0; + + // Write / Read from a register at the given 64 bit aligned offset. + // Offset may be implementation dependent. + virtual util::Status Write(uint64 offset, uint64 value) = 0; + virtual util::StatusOr Read(uint64 offset) = 0; + + // Polls the sepcified register until it has the given value or until it + // takes longer than the provided timeout in nanosecondss. + // Polls forever if timeout is zero or negative. + virtual util::Status Poll(uint64 offset, uint64 expected_value, + int64 timeout_ns = 0) = 0; + + // 32-bit version of above. Usually, it is same as running 64 bit version. + virtual util::Status Write32(uint64 offset, uint32 value) = 0; + virtual util::StatusOr Read32(uint64 offset) = 0; + virtual util::Status Poll32(uint64 offset, uint32 expected_value, + int64 timeout_ns = 0) = 0; +}; + +} // namespace driver_shared +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_DRIVER_SHARED_REGISTERS_H_ diff --git a/driver/time_stamper/BUILD b/driver_shared/time_stamper/BUILD similarity index 60% rename from driver/time_stamper/BUILD rename to driver_shared/time_stamper/BUILD index 86229a5..7b8bdfe 100644 --- a/driver/time_stamper/BUILD +++ b/driver_shared/time_stamper/BUILD @@ -14,13 +14,15 @@ # Description: time related libraries used by DarwiNN drivers. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package( default_visibility = ["//visibility:public"], ) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "driver_time_stamper", srcs = ["driver_time_stamper.cc"], hdrs = ["driver_time_stamper.h"], @@ -30,13 +32,47 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( + name = "mock_time_stamper", + testonly = 1, + hdrs = ["mock_time_stamper.h"], + deps = [ + ":time_stamper", + "//port:lock_guard", + "//port:port_test_helper", + ], +) + +libedgetpu_cc_library( + name = "mock_time_stamper_no_config", + testonly = 1, + hdrs = ["mock_time_stamper_no_config.h"], + deps = [ + ":time_stamper", + "//testing/base/public:gunit_main", + ], +) + +libedgetpu_cc_library( name = "time_stamper", hdrs = ["time_stamper.h"], deps = ["//port"], ) -cc_library( +libedgetpu_cc_library( + name = "mock_time_stamper_factory", + testonly = 1, + hdrs = ["mock_time_stamper_factory.h"], + deps = [ + ":mock_time_stamper", + ":time_stamper", + ":time_stamper_factory", + "//port", + "//port:port_test_helper", + ], +) + +libedgetpu_cc_library( name = "driver_time_stamper_factory", hdrs = ["driver_time_stamper_factory.h"], deps = [ @@ -47,7 +83,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "time_stamper_factory", hdrs = ["time_stamper_factory.h"], deps = [ diff --git a/driver/time_stamper/driver_time_stamper.cc b/driver_shared/time_stamper/driver_time_stamper.cc similarity index 87% rename from driver/time_stamper/driver_time_stamper.cc rename to driver_shared/time_stamper/driver_time_stamper.cc index a634d69..9ec0c1b 100644 --- a/driver/time_stamper/driver_time_stamper.cc +++ b/driver_shared/time_stamper/driver_time_stamper.cc @@ -12,18 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "driver/time_stamper/driver_time_stamper.h" +#include "driver_shared/time_stamper/driver_time_stamper.h" #include "port/time.h" namespace platforms { namespace darwinn { -namespace driver { + +namespace driver_shared { int64 DriverTimeStamper::GetTimeNanoSeconds() const { return GetCurrentTimeNanos(); } -} // namespace driver +} // namespace driver_shared } // namespace darwinn } // namespace platforms diff --git a/driver/time_stamper/driver_time_stamper.h b/driver_shared/time_stamper/driver_time_stamper.h similarity index 74% rename from driver/time_stamper/driver_time_stamper.h rename to driver_shared/time_stamper/driver_time_stamper.h index 9a437f3..608799c 100644 --- a/driver/time_stamper/driver_time_stamper.h +++ b/driver_shared/time_stamper/driver_time_stamper.h @@ -12,14 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef DARWINN_DRIVER_TIME_STAMPER_DRIVER_TIME_STAMPER_H_ -#define DARWINN_DRIVER_TIME_STAMPER_DRIVER_TIME_STAMPER_H_ +#ifndef DARWINN_DRIVER_SHARED_TIME_STAMPER_DRIVER_TIME_STAMPER_H_ +#define DARWINN_DRIVER_SHARED_TIME_STAMPER_DRIVER_TIME_STAMPER_H_ -#include "driver/time_stamper/time_stamper.h" +#include "driver_shared/time_stamper/time_stamper.h" namespace platforms { namespace darwinn { -namespace driver { + +namespace driver_shared { // Microsec-resolution monotonic clock. class DriverTimeStamper : public TimeStamper { @@ -30,8 +31,8 @@ class DriverTimeStamper : public TimeStamper { int64 GetTimeNanoSeconds() const override; }; -} // namespace driver +} // namespace driver_shared } // namespace darwinn } // namespace platforms -#endif // DARWINN_DRIVER_TIME_STAMPER_DRIVER_TIME_STAMPER_H_ +#endif // DARWINN_DRIVER_SHARED_TIME_STAMPER_DRIVER_TIME_STAMPER_H_ diff --git a/driver/time_stamper/driver_time_stamper_factory.h b/driver_shared/time_stamper/driver_time_stamper_factory.h similarity index 69% rename from driver/time_stamper/driver_time_stamper_factory.h rename to driver_shared/time_stamper/driver_time_stamper_factory.h index 166160f..85637d7 100644 --- a/driver/time_stamper/driver_time_stamper_factory.h +++ b/driver_shared/time_stamper/driver_time_stamper_factory.h @@ -12,16 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef DARWINN_DRIVER_TIME_STAMPER_DRIVER_TIME_STAMPER_FACTORY_H_ -#define DARWINN_DRIVER_TIME_STAMPER_DRIVER_TIME_STAMPER_FACTORY_H_ +#ifndef DARWINN_DRIVER_SHARED_TIME_STAMPER_DRIVER_TIME_STAMPER_FACTORY_H_ +#define DARWINN_DRIVER_SHARED_TIME_STAMPER_DRIVER_TIME_STAMPER_FACTORY_H_ -#include "driver/time_stamper/driver_time_stamper.h" -#include "driver/time_stamper/time_stamper_factory.h" +#include "driver_shared/time_stamper/driver_time_stamper.h" +#include "driver_shared/time_stamper/time_stamper.h" +#include "driver_shared/time_stamper/time_stamper_factory.h" #include "port/ptr_util.h" namespace platforms { namespace darwinn { -namespace driver { + +namespace driver_shared { // Factory class for creating DriverTimeStamper objects. class DriverTimeStamperFactory : public TimeStamperFactory { @@ -34,8 +36,8 @@ class DriverTimeStamperFactory : public TimeStamperFactory { } }; -} // namespace driver +} // namespace driver_shared } // namespace darwinn } // namespace platforms -#endif // DARWINN_DRIVER_TIME_STAMPER_DRIVER_TIME_STAMPER_FACTORY_H_ +#endif // DARWINN_DRIVER_SHARED_TIME_STAMPER_DRIVER_TIME_STAMPER_FACTORY_H_ diff --git a/driver/time_stamper/time_stamper.h b/driver_shared/time_stamper/time_stamper.h similarity index 91% rename from driver/time_stamper/time_stamper.h rename to driver_shared/time_stamper/time_stamper.h index 5d83418..0952bf0 100644 --- a/driver/time_stamper/time_stamper.h +++ b/driver_shared/time_stamper/time_stamper.h @@ -12,14 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef DARWINN_DRIVER_TIME_STAMPER_TIME_STAMPER_H_ -#define DARWINN_DRIVER_TIME_STAMPER_TIME_STAMPER_H_ +#ifndef DARWINN_DRIVER_SHARED_TIME_STAMPER_TIME_STAMPER_H_ +#define DARWINN_DRIVER_SHARED_TIME_STAMPER_TIME_STAMPER_H_ #include "port/integral_types.h" namespace platforms { namespace darwinn { -namespace driver { + +namespace driver_shared { // Abstract class for timestamping. Make it a class so a stateful mock can be // used in tests. @@ -66,8 +67,8 @@ class TimeStamper { } }; -} // namespace driver +} // namespace driver_shared } // namespace darwinn } // namespace platforms -#endif // DARWINN_DRIVER_TIME_STAMPER_TIME_STAMPER_H_ +#endif // DARWINN_DRIVER_SHARED_TIME_STAMPER_TIME_STAMPER_H_ diff --git a/driver/time_stamper/time_stamper_factory.h b/driver_shared/time_stamper/time_stamper_factory.h similarity index 78% rename from driver/time_stamper/time_stamper_factory.h rename to driver_shared/time_stamper/time_stamper_factory.h index e6a4fed..c91ab64 100644 --- a/driver/time_stamper/time_stamper_factory.h +++ b/driver_shared/time_stamper/time_stamper_factory.h @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef DARWINN_DRIVER_TIME_STAMPER_TIME_STAMPER_FACTORY_H_ -#define DARWINN_DRIVER_TIME_STAMPER_TIME_STAMPER_FACTORY_H_ - -#include "driver/time_stamper/time_stamper.h" +#ifndef DARWINN_DRIVER_SHARED_TIME_STAMPER_TIME_STAMPER_FACTORY_H_ +#define DARWINN_DRIVER_SHARED_TIME_STAMPER_TIME_STAMPER_FACTORY_H_ +#include "driver_shared/time_stamper/time_stamper.h" #include "port/ptr_util.h" namespace platforms { namespace darwinn { -namespace driver { + +namespace driver_shared { // Factory class for allocating TimeStamper objects. class TimeStamperFactory { @@ -37,8 +37,8 @@ class TimeStamperFactory { virtual std::unique_ptr CreateTimeStamper() = 0; }; -} // namespace driver +} // namespace driver_shared } // namespace darwinn } // namespace platforms -#endif // DARWINN_DRIVER_TIME_STAMPER_TIME_STAMPER_FACTORY_H_ +#endif // DARWINN_DRIVER_SHARED_TIME_STAMPER_TIME_STAMPER_FACTORY_H_ diff --git a/executable/BUILD b/executable/BUILD index 68f279e..7783dae 100644 --- a/executable/BUILD +++ b/executable/BUILD @@ -28,13 +28,6 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) -cc_library( - name = "export_graph_fbs", - data = [ - "graph.fbs", - ], -) - flatbuffer_cc_library( name = "executable_fbs", srcs = ["executable.fbs"], @@ -44,66 +37,3 @@ flatbuffer_cc_library( "--gen-mutable", # No size increase to the generated code. ], ) - -# The extended version has a larger code size and is meant for tools/tests. -# Make sure it does not overlap with executable_fbs. -flatbuffer_cc_library( - name = "executable_fbs_extended", - srcs = ["executable.fbs"], - flatc_args = [ - "--gen-object-api", - "--force-empty", - "--reflect-names", - ], - out_prefix = "extended/", -) - -# Flatbuffer library for graph that would be interpreted by the runtime. -flatbuffer_cc_library( - name = "graph_fbs", - srcs = ["graph.fbs"], - flatc_args = [ - "--gen-object-api", - "--force-empty", - "--gen-mutable", - ], - include_paths = [ - ".", - "third_party/darwinn/executable", - ], -) - -# Flatbuffer library for graph that would be interpreted by the runtime. -# The extended version has a larger code size and is meant for tools/tests. -# Make sure it does not overlap with graph_fbs. -flatbuffer_cc_library( - name = "graph_fbs_extended", - srcs = ["graph.fbs"], - flatc_args = [ - "--gen-object-api", - "--force-empty", - "--reflect-names", - ], - include_paths = [ - ".", - "third_party/darwinn/executable", - ], - out_prefix = "extended/", -) - -filegroup( - name = "graph_schema", - srcs = [ - "graph.fbs", - ], -) - -flatbuffer_cc_library( - name = "vii_commands_fbs", - srcs = ["vii_commands.fbs"], - flatc_args = [ - "--gen-object-api", - "--force-empty", - "--gen-mutable", - ], -) diff --git a/executable/executable.fbs b/executable/executable.fbs index 3ffe28b..e064d53 100644 --- a/executable/executable.fbs +++ b/executable/executable.fbs @@ -414,10 +414,10 @@ table Executable { parameter_caching_token:uint64; // If set, parameters in this model will be loaded in the TPU DRAM for higher - // performance. TPU DRAM is available on some architectures (e.g. Noronha). - // TPU DRAM is a scarce resource, therefore only selected models can have this - // option enabled (e.g. RNN-T for Noronha). If this option is enabled and - // enough TPU DRAM is not available an error is returned at run time. + // performance. TPU DRAM is available on some architectures. TPU DRAM is a + // scarce resource, therefore only selected models can have this option + // enabled. If this option is enabled and enough TPU DRAM is not available an + // error is returned at run time. use_tpu_dram_for_parameters:bool = false; // Estimated runtime in cycles for this model. diff --git a/libedgetpu_cc_rules.bzl b/libedgetpu_cc_rules.bzl new file mode 100644 index 0000000..adc1059 --- /dev/null +++ b/libedgetpu_cc_rules.bzl @@ -0,0 +1,114 @@ +""" +This module defines rules to set copts and linkopts for +libraries and modules in the libedgetpu project. +""" + +# This allows the dependencies to work correctly +# if we are included as a submodule. +def clean_dep(dep): + return str(Label(dep)) + +WINDOWS_COPTS = [ + "/DSTRIP_LOG=1", + "/DABSL_FLAGS_STRIP_NAMES", + "/D_HAS_DEPRECATED_RESULT_OF", + "/D_HAS_DEPRECATED_ADAPTOR_TYPEDEFS", + "/GR-", + "/DWIN32_LEAN_AND_MEAN", + "/D_WINSOCKAPI_", +] +WINDOWS_LINKOPTS = [] +WINDOWS_OPT_LINKOPTS = [] + +DARWIN_COPTS = [ + "-fvisibility=hidden", +] +DARWIN_LINKOPTS = [ + "-L/opt/local/lib", + "-lusb-1.0", +] +DARWIN_OPT_LINKOPTS = [] + +LINUX_COPTS = [] +LINUX_LINKOPTS = [ + "-l:libusb-1.0.so", +] +LINUX_OPT_LINKOPTS = [ + "-Wl,--strip-all", +] + +COMMON_COPTS = [ + "-DSTRIP_LOG=1", + "-fno-rtti", + "-fno-exceptions", + '-D__FILE__=\\"\\"', +] +COMMON_LINKOPTS = [] +COMMON_OPT_LINKOPTS = [] + +def libedgetpu_copts(): + return select({ + clean_dep("//:windows"): WINDOWS_COPTS, + clean_dep("//:darwin"): DARWIN_COPTS + COMMON_COPTS, + "//conditions:default": LINUX_COPTS + COMMON_COPTS, + }) + +def libedgetpu_linkopts(): + return select({ + clean_dep("//:windows"): WINDOWS_LINKOPTS, + clean_dep("//:darwin"): DARWIN_LINKOPTS + COMMON_LINKOPTS, + "//conditions:default": LINUX_LINKOPTS + COMMON_LINKOPTS, + }) + +def libedgetpu_opt_linkopts(): + return select({ + clean_dep("//:windows"): WINDOWS_OPT_LINKOPTS, + clean_dep("//:darwin"): DARWIN_OPT_LINKOPTS + COMMON_OPT_LINKOPTS, + "//conditions:default": LINUX_OPT_LINKOPTS + COMMON_OPT_LINKOPTS, + }) + +def libedgetpu_cc_library(name, copts = [], **attrs): + native.cc_library( + name = name + "_opt", + copts = copts + libedgetpu_copts(), + **attrs + ) + native.cc_library( + name = name + "_default", + copts = copts + libedgetpu_copts(), + **attrs + ) + native.cc_library( + name = name, + deps = select({ + clean_dep("//:opt"): [name + "_opt"], + "//conditions:default": [name + "_default"], + }), + ) + +def libedgetpu_cc_binary(name, copts = [], linkopts = [], deps = [], additional_linker_inputs = [], linkshared = 0, **attrs): + native.cc_library( + name = name + "_opt", + copts = copts + libedgetpu_copts(), + linkopts = libedgetpu_linkopts() + libedgetpu_opt_linkopts(), + deps = deps, + **attrs + ) + native.cc_library( + name = name + "_default", + copts = copts + libedgetpu_copts(), + linkopts = libedgetpu_linkopts(), + deps = deps, + **attrs + ) + native.cc_binary( + name = name, + deps = select({ + clean_dep("//:opt"): [name + "_opt"], + "//conditions:default": [name + "_default"], + }) + deps, + linkopts = linkopts, + additional_linker_inputs = additional_linker_inputs, + linkshared = linkshared, + **attrs + ) diff --git a/port/BUILD b/port/BUILD index 1bceb41..72db276 100644 --- a/port/BUILD +++ b/port/BUILD @@ -25,6 +25,7 @@ # For compiling as part of Android system however, DARWINN_PORT_ANDROID # must be defined as part of the Android build system. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") load("//:build_defs.bzl", "darwinn_port_defines") package(default_visibility = ["//visibility:public"]) @@ -33,7 +34,7 @@ package(default_visibility = ["//visibility:public"]) # - certain files in port/default/ that are under Apache 2.0 license. licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "port", hdrs = [ "aligned_malloc.h", @@ -64,12 +65,13 @@ cc_library( ], "//conditions:default": [ "//base", + "//base:port", "//util/gtl:ptr_util", "//util/math:mathutil", "//util/task:status", - "//util/task:statusor", "@com_google_absl//absl/time", "@com_google_absl//absl/types:span", + "@com_google_absl//absl/status:statusor", ], }) + [ "@com_google_absl//absl/flags:parse", @@ -79,7 +81,42 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( + name = "aligned_malloc", + hdrs = [ + "aligned_malloc.h", + "defs.h", + ], + defines = darwinn_port_defines(), + deps = select({ + "//:darwinn_portable": [ + "//port/default:port", + ], + "//conditions:default": [ + "//base", + "//base:port", + ], + }), +) + +libedgetpu_cc_library( + name = "math_util", + hdrs = [ + "defs.h", + "math_util.h", + ], + defines = darwinn_port_defines(), + deps = select({ + "//:darwinn_portable": [ + "//port/default:port", + ], + "//conditions:default": [ + "//util/math:mathutil", + ], + }), +) + +libedgetpu_cc_library( name = "unreachable", hdrs = [ "unreachable.h", @@ -90,7 +127,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "macros", hdrs = [ "defs.h", @@ -110,7 +147,7 @@ cc_library( }), ) -cc_library( +libedgetpu_cc_library( name = "logging", hdrs = [ "defs.h", @@ -131,7 +168,7 @@ cc_library( ) # This library is used for Darwinn 1.0 runtime only (Android/Google3). -cc_library( +libedgetpu_cc_library( name = "thread_annotations", hdrs = [ "defs.h", @@ -143,7 +180,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "integral_types", hdrs = [ "defs.h", @@ -163,7 +200,7 @@ cc_library( }), ) -cc_library( +libedgetpu_cc_library( name = "std_mutex_lock", hdrs = ["std_mutex_lock.h"], deps = [ @@ -172,7 +209,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "shared_mutex", srcs = [ "shared_mutex.cc", @@ -184,7 +221,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "mutex", hdrs = [ "defs.h", @@ -206,7 +243,7 @@ cc_library( alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "dma330", hdrs = [ "defs.h", @@ -224,7 +261,7 @@ cc_library( alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "semaphore", hdrs = [ "defs.h", @@ -236,13 +273,13 @@ cc_library( "//third_party/safertos_addons", ], "//conditions:default": [ - "//port/default:port", + "//port/default:semaphore", ], }), alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "condition_variable", hdrs = [ "condition_variable.h", @@ -261,7 +298,7 @@ cc_library( alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "restartable_thread", hdrs = ["restartable_thread.h"], defines = darwinn_port_defines(), @@ -272,7 +309,7 @@ cc_library( alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "blocking_queue", hdrs = ["blocking_queue.h"], defines = darwinn_port_defines(), @@ -283,7 +320,7 @@ cc_library( alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "lock_guard", srcs = ["lock_guard.cc"], hdrs = ["lock_guard.h"], @@ -295,7 +332,7 @@ cc_library( alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "blocking_counter", srcs = ["blocking_counter.cc"], hdrs = ["blocking_counter.h"], @@ -348,13 +385,13 @@ config_setting( }, ) -cc_library( +libedgetpu_cc_library( name = "tracing", hdrs = ["tracing.h"], defines = darwinn_port_defines(), ) -cc_library( +libedgetpu_cc_library( name = "string_util", hdrs = ["string_util.h"], deps = [ @@ -362,7 +399,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "timer", srcs = select({ "//:windows": ["timer_windows.cc"], @@ -379,7 +416,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "posix_time", srcs = ["posix_time.cc"], hdrs = ["posix_time.h"], @@ -388,7 +425,22 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( + name = "fileio", + srcs = select({ + "//:windows": ["fileio_windows.cc"], + "//conditions:default": [], + }), + hdrs = ["fileio.h"] + select({ + "//:windows": ["fileio_windows.h"], + "//conditions:default": ["fileio_linux.h"], + }), + deps = [ + ":port", + ], +) + +libedgetpu_cc_library( name = "cpu", hdrs = ["cpu.h"], defines = darwinn_port_defines(), @@ -398,7 +450,7 @@ cc_library( }), ) -cc_library( +libedgetpu_cc_library( name = "demangle", hdrs = [ "defs.h", @@ -416,7 +468,7 @@ cc_library( }), ) -cc_library( +libedgetpu_cc_library( name = "bsp", hdrs = [ "bsp.h", @@ -432,3 +484,38 @@ cc_library( }), alwayslink = 1, ) + +libedgetpu_cc_library( + name = "status_macros", + hdrs = [ + "defs.h", + "status_macros.h", + ], + defines = darwinn_port_defines(), + deps = select({ + "//:darwinn_portable": [ + "//port/default:port", + ], + "//conditions:default": [ + "//util/task:status", + ], + }), +) + +libedgetpu_cc_library( + name = "dma_manager", + hdrs = [ + "defs.h", + "dma_manager.h", + ], + defines = darwinn_port_defines(), + deps = select({ + "//:darwinn_firmware": [ + "//firmware/driver/dma:dma_manager", + ], + "//conditions:default": [ + "//port/default:dma_manager", + ], + }), + alwayslink = 1, +) diff --git a/port/aligned_malloc.h b/port/aligned_malloc.h index 4d7e4c0..0d22e9e 100644 --- a/port/aligned_malloc.h +++ b/port/aligned_malloc.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "base/port.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/aligned_malloc.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_ALIGNED_MALLOC_H_ diff --git a/port/array_slice.h b/port/array_slice.h index 39c2547..7b5e232 100644 --- a/port/array_slice.h +++ b/port/array_slice.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "absl/types/span.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/array_slice.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_ARRAY_SLICE_H_ diff --git a/port/builddata.h b/port/builddata.h index d8a57b2..91629ff 100644 --- a/port/builddata.h +++ b/port/builddata.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "absl/base/builddata.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/builddata.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_BUILDDATA_H_ diff --git a/port/casts.h b/port/casts.h index 3beadba..25c7252 100644 --- a/port/casts.h +++ b/port/casts.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "base/casts.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/casts.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_CASTS_H_ diff --git a/port/default/BUILD b/port/default/BUILD index 2987cec..895cea8 100644 --- a/port/default/BUILD +++ b/port/default/BUILD @@ -15,14 +15,23 @@ # Description: # Port of various google3 libraries and utilities. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) # All Google Owned Code except : # - certain files in port/default/ that are under Apache 2.0 license. licenses(["notice"]) +config_setting( + name = "windows", + values = { + "cpu": "x64_windows", + }, +) + # Independent headers that can be included by port_from_tf. -cc_library( +libedgetpu_cc_library( name = "port_base", hdrs = [ "error_codes.h", @@ -35,7 +44,7 @@ cc_library( ) # Port of various google3 libraries and utilities. -cc_library( +libedgetpu_cc_library( name = "port", srcs = [ "status_macros.cc", @@ -54,7 +63,6 @@ cc_library( "mutex.h", "math_util.h", "ptr_util.h", - "semaphore.h", "status.h", "status_macros.h", "statusor.h", @@ -64,13 +72,17 @@ cc_library( "//:windows": ["aligned_malloc_windows.h"], "//conditions:default": ["aligned_malloc_default.h"], }), + linkstamp = select({ + ":windows": None, # Linkstamp doesn't work on Windows, https://github.com/bazelbuild/bazel/issues/6997 + "//conditions:default": "builddata.cc", + }), deps = [ ":thread_annotations", "//port/default/port_from_tf", ], ) -cc_library( +libedgetpu_cc_library( name = "thread_annotations", hdrs = [ "thread_annotations.h", @@ -80,14 +92,27 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( + name = "semaphore", + hdrs = [ + "semaphore.h", + ], + deps = [ + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "//thread/fiber", + "//thread/fiber/semaphore:fifo_semaphore", + ], +) + +libedgetpu_cc_library( name = "strcat", hdrs = [ "strcat.h", ], ) -cc_library( +libedgetpu_cc_library( name = "unreachable", hdrs = [ "unreachable.h", @@ -97,7 +122,7 @@ cc_library( }), ) -cc_library( +libedgetpu_cc_library( name = "cleanup", hdrs = ["cleanup.h"], deps = [ @@ -105,37 +130,40 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "cpu", hdrs = ["cpu.h"], ) -cc_library( +libedgetpu_cc_library( name = "restartable_thread", hdrs = ["restartable_thread.h"], deps = [ "//firmware/common:log", "//firmware/common/callback", "//firmware/os:restartable_thread_interface", + "//thread/fiber", ], ) -cc_library( +libedgetpu_cc_library( name = "blocking_queue", hdrs = ["blocking_queue.h"], deps = [ + "@com_google_absl//absl/memory", "//firmware/common:log", - "//firmware/datastruct:circular_queue", + "//firmware/common:status", "//firmware/os:queue_interface", + "//thread/fiber", ], ) -cc_library( +libedgetpu_cc_library( name = "memory_barriers", hdrs = ["memory_barriers.h"], ) -cc_library( +libedgetpu_cc_library( name = "dma330", hdrs = ["dma330.h"], deps = [ @@ -146,3 +174,16 @@ cc_library( "//firmware/driver/dma:dma330_interface", ], ) + +libedgetpu_cc_library( + name = "dma_manager", + hdrs = ["dma_manager.h"], + deps = [ + ":port", + "//firmware/common:iomem", + "//firmware/common:status", + "//firmware/common/callback", + "//firmware/driver/dma:dma330_interface", + "//firmware/driver/dma:dma_manager_interface", + ], +) diff --git a/port/default/builddata.cc b/port/default/builddata.cc new file mode 100644 index 0000000..afe1ef6 --- /dev/null +++ b/port/default/builddata.cc @@ -0,0 +1,35 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extern "C" const char kDarwinnRuntimeBuildEmbedLabel[]; +const char kDarwinnRuntimeBuildEmbedLabel[] = BUILD_EMBED_LABEL; + +extern "C" const char kDarwinnRuntimeBaseChangeList[]; +const char kDarwinnRuntimeBaseChangeList[] = "CL_NUMBER=339933739"; + +namespace { +// Build a type whose constructor will contain references to all the build data +// variables, preventing them from being GC'ed by the linker. +struct KeepBuildDataVariables { + KeepBuildDataVariables() { + volatile int opaque_flag = 0; + if (!opaque_flag) return; + + const void* volatile capture; + capture = &kDarwinnRuntimeBuildEmbedLabel; + capture = &kDarwinnRuntimeBaseChangeList; + static_cast(capture); + } +} dummy; +} // namespace diff --git a/port/default/builddata.h b/port/default/builddata.h index 8a10819..f014aee 100644 --- a/port/default/builddata.h +++ b/port/default/builddata.h @@ -15,9 +15,10 @@ #ifndef DARWINN_PORT_DEFAULT_BUILDDATA_H_ #define DARWINN_PORT_DEFAULT_BUILDDATA_H_ -#if defined(_WIN32) #define STRINGIFY(x) #x #define TOSTRING(x) STRINGIFY(x) + +#if defined(_WIN32) #define COMPILER_VERSION "MSVC " TOSTRING(_MSC_FULL_VER) #elif defined(__GNUC__) #define COMPILER_VERSION __VERSION__ @@ -27,7 +28,7 @@ struct BuildData { static const char* BuildLabel() { - return "COMPILER=" COMPILER_VERSION ",DATE=" __DATE__ ",TIME=" __TIME__ ",CL_NUMBER=317268237"; + return "COMPILER=" COMPILER_VERSION ",DATE=" __DATE__ ",TIME=" __TIME__; } }; diff --git a/port/default/dma_manager.h b/port/default/dma_manager.h new file mode 100644 index 0000000..00f5a1a --- /dev/null +++ b/port/default/dma_manager.h @@ -0,0 +1,70 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_PORT_DEFAULT_DMA_MANAGER_H_ +#define DARWINN_PORT_DEFAULT_DMA_MANAGER_H_ + +#include "firmware/common/errors.h" +#include "firmware/common/iomem.h" +#include "firmware/common/status.h" +#include "firmware/driver/dma/dma330_interface.h" +#include "firmware/driver/dma/dma_manager_interface.h" + +namespace platforms { +namespace darwinn { + +class DmaManager : public firmware::driver::dma::DmaManagerInterface { + public: + DmaManager(firmware::common::IomemInterface* iomem) : iomem_(*iomem) {} + + firmware::common::Status IssueTransfer( + const firmware::common::Buffer& source, + const firmware::common::Buffer& destination, + firmware::driver::dma::Dma330Interface::Channel channel, + firmware::driver::dma::DmaDoneHandler handler) override { + if (source.size_bytes() != destination.size_bytes()) { + return firmware::common::InvalidArgumentError(); + } + iomem_.IoMemcpy(reinterpret_cast(destination.address()), + reinterpret_cast(source.address()), + source.size_bytes()); + handler(firmware::common::Status::OK()); + return firmware::common::Status::OK(); + } + + firmware::common::Status CancelTransfer( + firmware::driver::dma::Dma330Interface::Channel channel) override { + return firmware::common::Status::OK(); + } + + firmware::common::StatusOr + AllocateChannel() override { + return firmware::driver::dma::Dma330Interface::Channel::k0; + } + + void FreeChannel( + firmware::driver::dma::Dma330Interface::Channel channel) override{}; + + static bool DmaAccessible(const firmware::common::Buffer& buffer) { + return true; + } + + private: + firmware::common::IomemInterface& iomem_; +}; + +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_PORT_DEFAULT_DMA_MANAGER_H_ diff --git a/port/default/port_from_tf/BUILD b/port/default/port_from_tf/BUILD index 464655e..eb775b7 100644 --- a/port/default/port_from_tf/BUILD +++ b/port/default/port_from_tf/BUILD @@ -15,6 +15,8 @@ # Description: # Port of various google3 libraries and utilities. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) # Files are originally ported from TensorFlow. @@ -23,7 +25,7 @@ licenses(["notice"]) exports_files(["logging.h"]) # Test helpers -cc_library( +libedgetpu_cc_library( name = "status_test_util", testonly = 1, hdrs = [ @@ -36,7 +38,7 @@ cc_library( ) # Port of various google3 libraries and utilities. -cc_library( +libedgetpu_cc_library( name = "port_from_tf", srcs = [ "logging.cc", @@ -61,7 +63,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "thread_annotations", hdrs = [ "thread_annotations.h", diff --git a/port/default/port_from_tf/logging.h b/port/default/port_from_tf/logging.h index d654be7..f9bf74b 100644 --- a/port/default/port_from_tf/logging.h +++ b/port/default/port_from_tf/logging.h @@ -93,6 +93,8 @@ class LogMessageFatal : public LogMessage { #define _LOG_QFATAL _LOG_FATAL +#define _LOG_DFATAL _LOG_FATAL + // CHECK dies with a fatal error if condition is not true. It is *not* // controlled by NDEBUG, so the check will be executed regardless of // compilation mode. Therefore, it is safe to do things like: diff --git a/port/default/port_from_tf/statusor.cc b/port/default/port_from_tf/statusor.cc index 4d979db..4a7efb4 100644 --- a/port/default/port_from_tf/statusor.cc +++ b/port/default/port_from_tf/statusor.cc @@ -41,6 +41,12 @@ void Helper::Crash(const Status& status) { unreachable(); // NOLINT } +void ThrowBadStatusOrAccess(Status status) { + LOG(FATAL) << "Attempting to fetch value instead of handling error " + << status; + unreachable(); // NOLINT +} + } // namespace internal_statusor } // namespace util } // namespace darwinn diff --git a/port/default/port_from_tf/statusor.h b/port/default/port_from_tf/statusor.h index 39be0db..1e12660 100644 --- a/port/default/port_from_tf/statusor.h +++ b/port/default/port_from_tf/statusor.h @@ -28,7 +28,7 @@ limitations under the License. // // StatusOr result = DoBigCalculationThatCouldFail(); // if (result.ok()) { -// float answer = result.ValueOrDie(); +// float answer = *result; // printf("Big calculation yielded: %f", answer); // } else { // LOG(ERROR) << result.status(); @@ -38,7 +38,7 @@ limitations under the License. // // StatusOr result = FooFactory::MakeNewFoo(arg); // if (result.ok()) { -// std::unique_ptr foo(result.ValueOrDie()); +// std::unique_ptr foo(*result); // foo->DoSomethingCool(); // } else { // LOG(ERROR) << result.status(); @@ -48,7 +48,7 @@ limitations under the License. // // StatusOr> result = FooFactory::MakeNewFoo(arg); // if (result.ok()) { -// std::unique_ptr foo = std::move(result.ValueOrDie()); +// std::unique_ptr foo = std::move(*result); // foo->DoSomethingCool(); // } else { // LOG(ERROR) << result.status(); @@ -172,6 +172,37 @@ class StatusOr : private internal_statusor::StatusOrData, const Status& status() const &; Status status() &&; + // Returns a reference to the held value if `this->ok()`. Otherwise, it + // is guaranteed to terminate the process. + // + // If you have already checked the status using `this->ok()`, you probably + // want to use `operator*()` or `operator->()` to access the value instead of + // `value`. + // + // Note: for value types that are cheap to copy, prefer simple code: + // + // T value = statusor.value(); + // + // Otherwise, if the value type is expensive to copy, but can be left + // in the StatusOr, simply assign to a reference: + // + // T& value = statusor.value(); // or `const T&` + // + // Otherwise, if the value type supports an efficient move, it can be + // used as follows: + // + // T value = std::move(statusor).value(); + // + // The `std::move` on statusor instead of on the whole expression enables + // warnings about possible uses of the statusor object after the move. + // C++ style guide waiver for ref-qualified overloads granted in cl/143176389 + // See go/ref-qualifiers for more details on such overloads. + const T& value() const&; + T& value() &; + const T&& value() const&&; + T&& value() &&; + + // DEPRECATED. // Returns a reference to our current value, or CHECK-fails if !this->ok(). // // Note: for value types that are cheap to copy, prefer simple code: @@ -197,6 +228,27 @@ class StatusOr : private internal_statusor::StatusOrData, const T&& ValueOrDie() const &&; T&& ValueOrDie() &&; + // Returns a reference to the current value. + // + // REQUIRES: `this->ok() == true`, otherwise the behavior is undefined. + // + // Use `this->ok()` to verify that there is a current value within the + // `StatusOr`. Alternatively, see the `value()` member function for a + // similar API that guarantees crashing if there is + // no current value. + const T& operator*() const&; + T& operator*() &; + const T&& operator*() const&&; + T&& operator*() &&; + + // Returns a pointer to the current value. + // + // REQUIRES: `this->ok() == true`, otherwise the behavior is undefined. + // + // Use `this->ok()` to verify that there is a current value. + const T* operator->() const; + T* operator->(); + T ConsumeValueOrDie() { return std::move(ValueOrDie()); } // Ignores any errors. This method does nothing except potentially suppress @@ -275,6 +327,34 @@ Status StatusOr::status() && { return ok() ? Status::OK() : std::move(this->status_); } +template +const T& StatusOr::value() const& { + if (!this->ok()) internal_statusor::ThrowBadStatusOrAccess(this->status_); + return this->data_; +} + +template +T& StatusOr::value() & { + if (!this->ok()) internal_statusor::ThrowBadStatusOrAccess(this->status_); + return this->data_; +} + +template +const T&& StatusOr::value() const&& { + if (!this->ok()) { + internal_statusor::ThrowBadStatusOrAccess(std::move(this->status_)); + } + return std::move(this->data_); +} + +template +T&& StatusOr::value() && { + if (!this->ok()) { + internal_statusor::ThrowBadStatusOrAccess(std::move(this->status_)); + } + return std::move(this->data_); +} + template const T& StatusOr::ValueOrDie() const & { this->EnsureOk(); @@ -299,6 +379,42 @@ T&& StatusOr::ValueOrDie() && { return std::move(this->data_); } +template +const T& StatusOr::operator*() const& { + this->EnsureOk(); + return this->data_; +} + +template +T& StatusOr::operator*() & { + this->EnsureOk(); + return this->data_; +} + +template +const T&& StatusOr::operator*() const&& { + this->EnsureOk(); + return std::move(this->data_); +} + +template +T&& StatusOr::operator*() && { + this->EnsureOk(); + return std::move(this->data_); +} + +template +const T* StatusOr::operator->() const { + this->EnsureOk(); + return &this->data_; +} + +template +T* StatusOr::operator->() { + this->EnsureOk(); + return &this->data_; +} + template void StatusOr::IgnoreError() const { // no-op diff --git a/port/default/port_from_tf/statusor_internals.h b/port/default/port_from_tf/statusor_internals.h index eb69ea1..b51f92e 100644 --- a/port/default/port_from_tf/statusor_internals.h +++ b/port/default/port_from_tf/statusor_internals.h @@ -242,6 +242,8 @@ struct TraitsBase { TraitsBase& operator=(TraitsBase&&) = delete; }; +void ThrowBadStatusOrAccess(Status status); + } // namespace internal_statusor } // namespace util } // namespace darwinn diff --git a/port/default/semaphore.h b/port/default/semaphore.h index 9c23574..0308e01 100644 --- a/port/default/semaphore.h +++ b/port/default/semaphore.h @@ -15,11 +15,14 @@ #ifndef DARWINN_PORT_DEFAULT_SEMAPHORE_H_ #define DARWINN_PORT_DEFAULT_SEMAPHORE_H_ -#include +#include -#include //NOLINT -#include //NOLINT -#include //NOLINT +#include "absl/synchronization/mutex.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" +#include "thread/fiber/fiber.h" +#include "thread/fiber/select.h" +#include "thread/fiber/semaphore/fifo_semaphore.h" namespace platforms { namespace darwinn { @@ -29,59 +32,64 @@ namespace internal { // Base class, not for direct use. class Semaphore { public: - virtual ~Semaphore() = default; + virtual ~Semaphore() { + // Give the underlying semaphore its credits back so that it doesn't CHECK. + semaphore_.Release(semaphore_.capacity() - semaphore_.current_value()); + } bool Take() { - std::unique_lock lock(mutex_); - while (count_ == 0) { - cv_.wait(lock); + if (thread::Select({thread::OnCancel(), semaphore_.OnAcquire(1)}) == 0) { + return false; } - count_--; return true; } + // A common::firmware::Status return type would be nicer here, but by keeping + // it as a bool we can avoid the need for a wrapper class in firmware. bool Take(uint32_t timeout) { // Generally we're running one tick per ms. - auto timeout_time = - std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); + auto timeout_time = absl::Now() + absl::Milliseconds(timeout); - std::unique_lock lock(mutex_); - while (count_ == 0) { - if (cv_.wait_until(lock, timeout_time) == std::cv_status::timeout) { - // Timed-out, check the predicate one last time. - if (count_ != 0) { - break; - } + int i = thread::SelectUntil(timeout_time, + {thread::OnCancel(), semaphore_.OnAcquire(1)}); + switch (i) { + case -1: + // timeout + return false; + case 0: + // canceled return false; - } } - count_--; return true; } bool Give() { - mutex_.lock(); - if (count_ < max_count_) { - count_++; + // Grab a mutex to prevent multiple givers from releasing at the same time. + absl::MutexLock lock(&giver_mutex_); + + bool released = false; + // Prevent the semaphore from overflowing. + if (semaphore_.current_value() < semaphore_.capacity()) { + semaphore_.Release(1); + released = true; } - mutex_.unlock(); - cv_.notify_one(); - return true; + return released; } protected: Semaphore(unsigned int max_count, unsigned int initial_count) - : count_(initial_count), max_count_(max_count) {} + : semaphore_(max_count) { + // By default the FifoSemaphore starts fully given. + semaphore_.Acquire(max_count - initial_count); + } private: - std::mutex mutex_; - std::condition_variable cv_; - int count_; - const int max_count_; + thread::FifoSemaphore semaphore_; + absl::Mutex giver_mutex_; }; } // namespace internal diff --git a/port/default/status_macros.h b/port/default/status_macros.h index 0ede878..bf21f07 100644 --- a/port/default/status_macros.h +++ b/port/default/status_macros.h @@ -16,7 +16,7 @@ #define DARWINN_PORT_DEFAULT_STATUS_MACROS_H_ #include -#include // NOLINT +#include // NOLINT #include #include #include @@ -230,14 +230,13 @@ class StatusAdaptorForMacros { if (PREDICT_FALSE(!_status.ok())) return _status; \ } while (0) -#define RETURN_WITH_CONTEXT_IF_ERROR(expr, ...) \ - do { \ - ::platforms::darwinn::util::Status _status = (expr); \ - if (PREDICT_FALSE(!_status.ok())) { \ - ::platforms::darwinn::util::error::AppendToMessage(&_status, \ - __VA_ARGS__); \ - return _status; \ - } \ +#define RETURN_WITH_CONTEXT_IF_ERROR(expr, ...) \ + do { \ + ::platforms::darwinn::util::Status _status = (expr); \ + if (PREDICT_FALSE(!_status.ok())) { \ + ::platforms::darwinn::util::AppendToMessage(&_status, __VA_ARGS__); \ + return _status; \ + } \ } while (0) #endif // DARWINN_PORT_DEFAULT_STATUS_MACROS_H_ diff --git a/port/dma_manager.h b/port/dma_manager.h new file mode 100644 index 0000000..0f9d25a --- /dev/null +++ b/port/dma_manager.h @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_PORT_DMA_MANAGER_H_ +#define DARWINN_PORT_DMA_MANAGER_H_ + +#include "port/defs.h" + +#if DARWINN_PORT_FIRMWARE + +#include "firmware/driver/dma/dma_manager.h" + +namespace platforms { +namespace darwinn { + +using DmaManager = firmware::driver::dma::DmaManager; + +} // namespace darwinn +} // namespace platforms + +#else // !DARWINN_PORT_FIRMWARE + +#include "port/default/dma_manager.h" + +#endif // DARWINN_PORT_FIRMWARE + +#endif // DARWINN_PORT_DMA_MANAGER_H_ diff --git a/port/errors.h b/port/errors.h index 5aab328..3be7c78 100644 --- a/port/errors.h +++ b/port/errors.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "util/task/canonical_errors.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/errors.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_ERRORS_H_ diff --git a/port/fileio.h b/port/fileio.h new file mode 100644 index 0000000..4bc3bc3 --- /dev/null +++ b/port/fileio.h @@ -0,0 +1,24 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_PORT_FILEIO_H_ +#define DARWINN_PORT_FILEIO_H_ + +#if defined(_WIN32) +#include "port/fileio_windows.h" +#else +#include "port/fileio_linux.h" +#endif // defined(_WIN32) + +#endif // DARWINN_PORT_FILEIO_H_ diff --git a/port/fileio_linux.h b/port/fileio_linux.h new file mode 100644 index 0000000..b3cedde --- /dev/null +++ b/port/fileio_linux.h @@ -0,0 +1,31 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_PORT_FILEIO_LINUX_H_ +#define DARWINN_PORT_FILEIO_LINUX_H_ + +#include +#include + +namespace platforms { +namespace darwinn { + +#define INVALID_FD_VALUE (-1) + +typedef int FileDescriptor; + +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_PORT_FILEIO_LINUX_H_ diff --git a/port/fileio_windows.cc b/port/fileio_windows.cc new file mode 100644 index 0000000..8a16621 --- /dev/null +++ b/port/fileio_windows.cc @@ -0,0 +1,75 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "port/errors.h" +#include "port/fileio.h" +#include "port/logging.h" +#include "port/stringprintf.h" + +namespace platforms { +namespace darwinn { + +FileDescriptor open(const char* path, int access) { + std::string c_path(path); + std::wstring wc_path(c_path.begin(), c_path.end()); + + DWORD win_access = (GENERIC_READ | GENERIC_WRITE); + FileDescriptor fd = INVALID_FD_VALUE; + + switch (access) { + case O_RDONLY: + win_access = GENERIC_READ; + break; + + case O_WRONLY: + win_access = GENERIC_WRITE; + break; + + case O_RDWR: + win_access = (GENERIC_READ | GENERIC_WRITE); + break; + + default: + VLOG(1) << StringPrintf("Device open: access unspecified(=%d)", access); + break; + } + + fd = CreateFileW(wc_path.c_str(), win_access, + (FILE_SHARE_READ | FILE_SHARE_WRITE), + NULL, // default security attributes + OPEN_EXISTING, // disposition + 0, // file attributes + NULL); // do not copy file attributes + if (fd == INVALID_FD_VALUE) { + DWORD gle = GetLastError(); + VLOG(1) << StringPrintf("Device open: gle=%d strerror=%s", gle, + strerror(errno)); + } + + return fd; +} + +void close(FileDescriptor fd) { + BOOL rc = CloseHandle(fd); + if (!rc) { + DWORD gle = GetLastError(); + VLOG(1) << StringPrintf("CloseHandle failed: fd(%p) gle=%d strerror=%s", fd, + gle, strerror(errno)); + } +} + +} // namespace darwinn +} // namespace platforms diff --git a/port/fileio_windows.h b/port/fileio_windows.h new file mode 100644 index 0000000..f590a1d --- /dev/null +++ b/port/fileio_windows.h @@ -0,0 +1,49 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DARWINN_PORT_FILEIO_WINDOWS_H_ +#define DARWINN_PORT_FILEIO_WINDOWS_H_ + +#include + +// It's defined in port/default/port_from_tf/macros.h and winnt.h which +// is included through windows.h +// port/macros.h should be included only after windows specific code if +// this define is used. +#ifdef ARRAYSIZE +#undef ARRAYSIZE +#endif +#include +#undef ARRAYSIZE +// Work around a redundancy between and port/logging.h (enum ERROR) +#undef ERROR + +namespace platforms { +namespace darwinn { + +#define INVALID_FD_VALUE INVALID_HANDLE_VALUE + +typedef HANDLE FileDescriptor; + +// Opens existing file on Windows by parsing linux style RD/WR access mode. +// Doesn't support any access options beyond O_RDONLY/O_WRONLY/O_RDWR +FileDescriptor open(const char* path, int access); + +// Closes file handle +void close(FileDescriptor fd); + +} // namespace darwinn +} // namespace platforms + +#endif // DARWINN_PORT_FILEIO_WINDOWS_H_ diff --git a/port/integral_types.h b/port/integral_types.h index f9f9be9..6514d72 100644 --- a/port/integral_types.h +++ b/port/integral_types.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "base/integral_types.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/integral_types.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_INTEGRAL_TYPES_H_ diff --git a/port/logging.h b/port/logging.h index cc8e387..4f0163d 100644 --- a/port/logging.h +++ b/port/logging.h @@ -17,6 +17,7 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_FIRMWARE #include "port/firmware/logging.h" #elif DARWINN_PORT_USE_GOOGLE3 @@ -24,5 +25,6 @@ #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/logging.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_LOGGING_H_ diff --git a/port/macros.h b/port/macros.h index 723c715..886432b 100644 --- a/port/macros.h +++ b/port/macros.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "base/macros.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/macros.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_MACROS_H_ diff --git a/port/math_util.h b/port/math_util.h index 4f095bc..ad5965b 100644 --- a/port/math_util.h +++ b/port/math_util.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "util/math/mathutil.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/math_util.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_MATH_UTIL_H_ diff --git a/port/ptr_util.h b/port/ptr_util.h index 081131c..f61c689 100644 --- a/port/ptr_util.h +++ b/port/ptr_util.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "util/gtl/ptr_util.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/ptr_util.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_PTR_UTIL_H_ diff --git a/port/status.h b/port/status.h index 749ee57..5175b84 100644 --- a/port/status.h +++ b/port/status.h @@ -17,11 +17,13 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "util/task/status.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/status.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports namespace platforms { namespace darwinn { diff --git a/port/status_macros.h b/port/status_macros.h index db64418..8f3506d 100644 --- a/port/status_macros.h +++ b/port/status_macros.h @@ -17,10 +17,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "util/task/status_macros.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/status_macros.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_STATUS_MACROS_H_ diff --git a/port/statusor.h b/port/statusor.h index 763adde..97a3ca4 100644 --- a/port/statusor.h +++ b/port/statusor.h @@ -18,9 +18,16 @@ #include "port/defs.h" #if DARWINN_PORT_USE_GOOGLE3 -#include "util/task/statusor.h" +#include "absl/status/statusor.h" +// Darwinn 1.0 code relies on util:: namespace to be portable across google3 and +// android. For now keep StatusOr at global "util" namespace. +namespace util { +template +using StatusOr = absl::StatusOr; +} // namespace util + #else // !DARWINN_PORT_USE_GOOGLE3 -#include "port/default/statusor.h" +#include "port/default/statusor.h" // IWYU pragma: export #endif // DARWINN_PORT_USE_GOOGLE3 #endif // DARWINN_PORT_STATUSOR_H_ diff --git a/port/stringprintf.h b/port/stringprintf.h index e98f6d4..5b9bebd 100644 --- a/port/stringprintf.h +++ b/port/stringprintf.h @@ -18,10 +18,12 @@ #include "port/defs.h" +// IWYU pragma: begin_exports #if DARWINN_PORT_USE_GOOGLE3 #include "base/stringprintf.h" #else // !DARWINN_PORT_USE_GOOGLE3 #include "port/default/stringprintf.h" #endif // DARWINN_PORT_USE_GOOGLE3 +// IWYU pragma: end_exports #endif // DARWINN_PORT_STRINGPRINTF_H_ diff --git a/port/tracing.h b/port/tracing.h index 88e7cd8..7448b8e 100644 --- a/port/tracing.h +++ b/port/tracing.h @@ -20,122 +20,79 @@ #define DARWINN_SCOPE_PREFIX "DarwiNN::" // For Android binaries built on Android. -// We borrow the NNAPI systrace implementation to profile DarwiNN drivers built -// on Android. #if defined(DARWINN_PORT_ANDROID_SYSTEM) -#include "Tracing.h" - -#define TRACE_INITIALIZE() -// Use this only once per function, ideally at the beginning of each scope. -#define TRACE_SCOPE(name) NNTRACE_NAME_1(DARWINN_SCOPE_PREFIX name) - -// Use this to add trace markers in a scope that already has a TRACE_SCOPE. -#define TRACE_WITHIN_SCOPE(name) NNTRACE_NAME_SWITCH(DARWINN_SCOPE_PREFIX name) - -// Use this when a new thread starts up. -#define TRACE_START_THREAD(name) - -#define TRACE_DUMP(output_file) -#define TRACE_FINALIZE() +#include "port/tracer/darwinn_android_native_trace_macros.h" // For Android binaries built on google3. -// When building on google3, blaze will not be able to link against the atrace -// symbols and we will need to dynamically link to it by ourselves. #elif defined(__ANDROID__) && defined(DARWINN_ANDROID_GOOGLE3_TRACE_ENABLED) -#include "port/tracer/darwinn_android_tracer.h" - -#define TRACE_SCOPE(name) \ - ::platforms::darwinn::DARWINN_ANDROID_TRACE_SCOPE(DARWINN_SCOPE_PREFIX name) - -#define TRACE_WITHIN_SCOPE(name) \ - ::platforms::darwinn::DARWINN_ANDROID_TRACE_SCOPE(DARWINN_SCOPE_PREFIX name) - -#define TRACE_START_THREAD(name) -#define TRACE_DUMP(output_file) -#define TRACE_FINALIZE() +#include "port/tracer/darwinn_android_google3_trace_macros.h" // Perfetto tracing for firmware can be enabled at build time. // Build the firmware with --define darwinn_firmware_trace_enabled=1 -// and build run_graph_executor with --define darwinn_perfetto_trace_enabled=1. +// and build the application with --define darwinn_perfetto_trace_enabled=1. #elif defined(__ANDROID__) && defined(DARWINN_PERFETTO_TRACE_ENABLED) -#include "port/tracer/darwinn_perfetto_scoped_tracer.h" -#include "port/tracer/darwinn_perfetto_tracer.h" - -#define TRACE_INITIALIZE() ::platforms::darwinn::InitializeScopedPerfetto() - -#define TRACE_SCOPE(name) \ - PERFTETTO_TRACK_SCOPE(DARWINN_SCOPE_PREFIX name) - -#define TRACE_WITHIN_SCOPE(name) \ - PERFTETTO_TRACK_SCOPE(DARWINN_SCOPE_PREFIX name) - -#define TRACE_START_THREAD(name) -#define TRACE_DUMP(output_file) - -#define TRACE_FINALIZE() ::platforms::darwinn::FinalizePerfetto() +#include "port/tracer/darwinn_perfetto_trace_macros.h" // Web Tracing Framework can be enabled at build time. // --define=GLOBAL_WTF_ENABLE=1 #elif defined(WTF_ENABLE) -#include "third_party/tracing_framework_bindings_cpp/macros.h" // IWYU pragma: export - -#define TRACE_INITIALIZE() -#define TRACE_SCOPE(name) WTF_SCOPE0(DARWINN_SCOPE_PREFIX name) -#define TRACE_WITHIN_SCOPE(name) WTF_EVENT0(DARWINN_SCOPE_PREFIX name) -#define TRACE_START_THREAD(name) WTF_THREAD_ENABLE(DARWINN_SCOPE_PREFIX name) -#define TRACE_DUMP(output_file) -#define TRACE_FINALIZE() +#include "port/tracer/darwinn_wtf_trace_macros.h" +// If --define darwinn_csv_trace_enabled=1 is specified, the trace events could +// be dumped into a CSV file. #elif defined(DARWINN_CSV_TRACE_ENABLED) -#include "port/tracer/darwinn_csv_tracer.h" - -#define TRACE_INITIALIZE() -#define TRACE_SCOPE(name) \ - ::platforms::darwinn::DARWINN_CSV_TRACE_SCOPE(DARWINN_SCOPE_PREFIX name) -#define TRACE_WITHIN_SCOPE(name) \ - ::platforms::darwinn::DARWINN_CSV_TRACE_SCOPE(DARWINN_SCOPE_PREFIX name) - -#define TRACE_START_THREAD(name) - -#define TRACE_DUMP(output_file) \ - ::platforms::darwinn::DarwinnCSVTracer::DumpTrace(output_file) - -#define TRACE_FINALIZE() +#include "port/tracer/darwinn_csv_trace_macros.h" // If xprof tracing is enabled at build time: --define=darwinn_xprof_enabled=1 -// To capture the trace, use perftools/gputools/profiler/xprof.sh. +// Add --xprof_end_2_end_upload to the test to upload the xprof trace. #elif defined(DARWINN_XPROF_ENABLED) -#include "port/tracer/darwinn_fw_xprof_tracer.h" -#include "tensorflow/core/profiler/lib/traceme.h" - -#define _PASTE(x, y) x##y -#define PASTE(x, y) _PASTE(x, y) - -#define TRACE_INITIALIZE() -#define TRACE_SCOPE(name) \ - tensorflow::profiler::TraceMe PASTE(activity, \ - __LINE__)(DARWINN_SCOPE_PREFIX name) -#define TRACE_WITHIN_SCOPE(name) TRACE_SCOPE(name) -#define TRACE_START_THREAD(name) TRACE_SCOPE(name) - -#define TRACE_FINALIZE() +#include "port/tracer/darwinn_xprof_trace_macros.h" // No tracing for other environments. #else +// Initializes tracing. Only required for Perfetto tracing. #define TRACE_INITIALIZE() -#define TRACE_SCOPE(name) + +// Adds a trace event with the start and end time specified by the life time of +// the created scope object. +// The "CRITICAL" makes it visible even for lab testing. See the PNP_BENCHMARK +// below. +#define TRACE_SCOPE_CRITICAL(name) + +// Starts a trace event. A uint64 unique ID will be returned which could be used +// to match the trace event in TRACE_SCOPE_CRITICAL_END. `device_paths` are +// added as metadata to xprof, and ignored for other tracing backends. +#define TRACE_SCOPE_CRITICAL_BEGIN(name, device_paths) 0 +#define TRACE_SCOPE_CRITICAL_END(id) + +// For adding a trace event inside anoter scoped trace event. The newly added +// one has the same nested layer as the outer scope trace event. +// Only supported by our Android native tracing. #define TRACE_WITHIN_SCOPE(name) + +// To mark the start of a thread. Only supported by WTF trace. #define TRACE_START_THREAD(name) + +// Dumps the trace events into a file. Only supported by CSV trace. #define TRACE_DUMP(output_file) + +// Marks the end of the profiling. Only required by Perfetto. #define TRACE_FINALIZE() #endif +// Don't trace TRACE_SCOPE macros when PNP_BENCHMARKING is on +#ifdef PNP_BENCHMARKING +#define TRACE_SCOPE(name) +#else +#define TRACE_SCOPE(name) TRACE_SCOPE_CRITICAL(name) +#endif + #endif // DARWINN_PORT_DEFAULT_SYSTRACE_H_ diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..9b808c4 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +LIBEDGETPU_DIR="${SCRIPT_DIR}/../out" +RULES_FILE="${SCRIPT_DIR}/../debian/edgetpu-accelerator.rules" + +function info { + echo -e "\033[0;32m${1}\033[0m" # green +} + +function warn { + echo -e "\033[0;33m${1}\033[0m" # yellow +} + +function error { + echo -e "\033[0;31m${1}\033[0m" # red +} + +function install_file { + local name="${1}" + local src="${2}" + local dst="${3}" + + info "Installing ${name} [${dst}]..." + if [[ -f "${dst}" ]]; then + warn "File already exists. Replacing it..." + rm -f "${dst}" + fi + cp -a "${src}" "${dst}" +} + +if [[ "${EUID}" != 0 ]]; then + error "Please use sudo to run as root." + exit 1 +fi + +if [[ -f /etc/mendel_version ]]; then + error "Looks like you're using a Coral Dev Board. You should instead use Debian packages to manage Edge TPU software." + exit 1 +fi + +readonly OS="$(uname -s)" +readonly MACHINE="$(uname -m)" + +if [[ "${OS}" == "Linux" ]]; then + case "${MACHINE}" in + x86_64) + HOST_GNU_TYPE=x86_64-linux-gnu + CPU=k8 + ;; + armv6l) + HOST_GNU_TYPE=arm-linux-gnueabihf + CPU=armv6 + ;; + armv7l) + HOST_GNU_TYPE=arm-linux-gnueabihf + CPU=armv7a + ;; + aarch64) + HOST_GNU_TYPE=aarch64-linux-gnu + CPU=aarch64 + ;; + *) + error "Your Linux platform is not supported." + exit 1 + ;; + esac +elif [[ "${OS}" == "Darwin" ]]; then + CPU=darwin + + MACPORTS_PATH_AUTO="$(command -v port || true)" + MACPORTS_PATH="${MACPORTS_PATH_AUTO:-/opt/local/bin/port}" + + BREW_PATH_AUTO="$(command -v brew || true)" + BREW_PATH="${BREW_PATH_AUTO:-/usr/local/bin/brew}" + + if [[ -x "${MACPORTS_PATH}" ]]; then + DARWIN_INSTALL_COMMAND="${MACPORTS_PATH}" + DARWIN_INSTALL_USER="$(whoami)" + elif [[ -x "${BREW_PATH}" ]]; then + DARWIN_INSTALL_COMMAND="${BREW_PATH}" + DARWIN_INSTALL_USER="${SUDO_USER}" + else + error "You need to install either Homebrew or MacPorts first." + exit 1 + fi +else + error "Your operating system is not supported." + exit 1 +fi + +cat << EOM +Warning: If you're using the Coral USB Accelerator, it may heat up during operation, depending +on the computation workloads and operating frequency. Touching the metal part of the USB +Accelerator after it has been operating for an extended period of time may lead to discomfort +and/or skin burns. As such, if you enable the Edge TPU runtime using the maximum operating +frequency, the USB Accelerator should be operated at an ambient temperature of 25°C or less. +Alternatively, if you enable the Edge TPU runtime using the reduced operating frequency, then +the device is intended to safely operate at an ambient temperature of 35°C or less. + +Google does not accept any responsibility for any loss or damage if the device +is operated outside of the recommended ambient temperature range. + +Note: This question affects only USB-based Coral devices, and is irrelevant for PCIe devices. +................................................................................ +Would you like to enable the maximum operating frequency for your Coral USB device? Y/N +EOM + +read USE_MAX_FREQ +case "${USE_MAX_FREQ}" in + [yY]) + info "Using the maximum operating frequency for Coral USB devices." + FREQ_DIR=direct + ;; + *) + info "Using the reduced operating frequency for Coral USB devices." + FREQ_DIR=throttled + ;; +esac + +if [[ "${CPU}" == "darwin" ]]; then + sudo -u "${DARWIN_INSTALL_USER}" "${DARWIN_INSTALL_COMMAND}" install libusb + + DARWIN_INSTALL_LIB_DIR="$(dirname "$(dirname "${DARWIN_INSTALL_COMMAND}")")/lib" + LIBEDGETPU_LIB_DIR="/usr/local/lib" + mkdir -p "${LIBEDGETPU_LIB_DIR}" + + install_file "Edge TPU runtime library" \ + "${LIBEDGETPU_DIR}/${FREQ_DIR}/darwin/libedgetpu.1.0.dylib" \ + "${LIBEDGETPU_LIB_DIR}" + + install_file "Edge TPU runtime library symlink" \ + "${LIBEDGETPU_DIR}/${FREQ_DIR}/darwin/libedgetpu.1.dylib" \ + "${LIBEDGETPU_LIB_DIR}" + + install_name_tool -id "${LIBEDGETPU_LIB_DIR}/libedgetpu.1.dylib" \ + "${LIBEDGETPU_LIB_DIR}/libedgetpu.1.0.dylib" + + install_name_tool -change "/opt/local/lib/libusb-1.0.0.dylib" \ + "${DARWIN_INSTALL_LIB_DIR}/libusb-1.0.0.dylib" \ + "${LIBEDGETPU_LIB_DIR}/libedgetpu.1.0.dylib" +else + for pkg in libc6 libgcc1 libstdc++6 libusb-1.0-0; do + if ! dpkg -l "${pkg}" > /dev/null; then + PACKAGES+=" ${pkg}" + fi + done + + if [[ -n "${PACKAGES}" ]]; then + info "Installing library dependencies:${PACKAGES}..." + apt-get update && apt-get install -y ${PACKAGES} + info "Done." + fi + + if [[ -x "$(command -v udevadm)" ]]; then + install_file "device rule file" \ + "${RULES_FILE}" \ + "/etc/udev/rules.d/99-edgetpu-accelerator.rules" + udevadm control --reload-rules && udevadm trigger + info "Done." + fi + + install_file "Edge TPU runtime library" \ + "${LIBEDGETPU_DIR}/${FREQ_DIR}/${CPU}/libedgetpu.so.1.0" \ + "/usr/lib/${HOST_GNU_TYPE}/libedgetpu.so.1.0" + ldconfig # Generates libedgetpu.so.1 symlink + info "Done." +fi + diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh new file mode 100755 index 0000000..9ec0126 --- /dev/null +++ b/scripts/uninstall.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +function info { + echo -e "\033[0;32m${1}\033[0m" # green +} + +function warn { + echo -e "\033[0;33m${1}\033[0m" # yellow +} + +function error { + echo -e "\033[0;31m${1}\033[0m" # red +} + +if [[ "${EUID}" != 0 ]]; then + error "Please use sudo to run as root." + exit 1 +fi + +if [[ -f /etc/mendel_version ]]; then + error "Looks like you're using a Coral Dev Board. You should instead use Debian packages to manage Edge TPU software." + exit 1 +fi + +readonly OS="$(uname -s)" +readonly MACHINE="$(uname -m)" + +if [[ "${OS}" == "Linux" ]]; then + case "${MACHINE}" in + x86_64) + HOST_GNU_TYPE=x86_64-linux-gnu + CPU_DIR=k8 + ;; + armv7l) + HOST_GNU_TYPE=arm-linux-gnueabihf + CPU_DIR=armv7a + ;; + aarch64) + HOST_GNU_TYPE=aarch64-linux-gnu + CPU_DIR=aarch64 + ;; + *) + error "Your Linux platform is not supported. There's nothing to uninstall." + exit 1 + ;; + esac +elif [[ "${OS}" == "Darwin" ]]; then + CPU=darwin +else + error "Your operating system is not supported. There's nothing to uninstall." + exit 1 +fi + +if [[ "${CPU}" == "darwin" ]]; then + LIBEDGETPU_LIB_DIR="/usr/local/lib" + + if [[ -f "${LIBEDGETPU_LIB_DIR}/libedgetpu.1.0.dylib" ]]; then + info "Uninstalling Edge TPU runtime library..." + rm -f "${LIBEDGETPU_LIB_DIR}/libedgetpu.1.0.dylib" + info "Done" + fi + + if [[ -L "${LIBEDGETPU_LIB_DIR}/libedgetpu.1.dylib" ]]; then + info "Uninstalling Edge TPU runtime library symlink..." + rm -f "${LIBEDGETPU_LIB_DIR}/libedgetpu.1.dylib" + info "Done" + fi +else + if [[ -x "$(command -v udevadm)" ]]; then + UDEV_RULE_PATH="/etc/udev/rules.d/99-edgetpu-accelerator.rules" + if [[ -f "${UDEV_RULE_PATH}" ]]; then + info "Uninstalling device rule file [${UDEV_RULE_PATH}]..." + rm -f "${UDEV_RULE_PATH}" + udevadm control --reload-rules && udevadm trigger + info "Done." + fi + fi + + LIBEDGETPU_DST="/usr/lib/${HOST_GNU_TYPE}/libedgetpu.so.1.0" + if [[ -f "${LIBEDGETPU_DST}" ]]; then + info "Uninstalling Edge TPU runtime library [${LIBEDGETPU_DST}]..." + rm -f "${LIBEDGETPU_DST}" + ldconfig + info "Done." + fi +fi diff --git a/tflite/BUILD b/tflite/BUILD index d0b54db..983f2c1 100644 --- a/tflite/BUILD +++ b/tflite/BUILD @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_binary", "libedgetpu_cc_library") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -cc_library( +libedgetpu_cc_library( name = "custom_op_data", srcs = ["custom_op_data.cc"], hdrs = ["custom_op_data.h"], @@ -26,7 +28,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "custom_op", srcs = ["custom_op.cc"], hdrs = ["custom_op.h"], @@ -40,7 +42,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "custom_op_user_data_direct", srcs = ["custom_op_user_data_direct.cc"], hdrs = ["custom_op_user_data_direct.h"], @@ -53,7 +55,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_context_direct_header", hdrs = ["edgetpu_context_direct.h"], deps = [ @@ -72,7 +74,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_manager_direct_header", hdrs = ["edgetpu_manager_direct.h"], deps = [ @@ -92,7 +94,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_context_direct", srcs = ["edgetpu_context_direct.cc"], hdrs = ["edgetpu_context_direct.h"], @@ -113,7 +115,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_manager_direct", srcs = ["edgetpu_manager_direct.cc"], hdrs = ["edgetpu_manager_direct.h"], @@ -136,7 +138,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "custom_op_direct", srcs = ["custom_op_direct.cc"], deps = [ @@ -155,7 +157,7 @@ cc_library( alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "tensor_data_controller", srcs = ["tensor_data_controller.cc"], hdrs = ["tensor_data_controller.h"], @@ -166,7 +168,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_context_factory", srcs = ["edgetpu_context_factory.cc"], hdrs = ["edgetpu_context_factory.h"], @@ -184,7 +186,7 @@ config_setting( values = {"define": "EXTRA_LOGGING=DUMMY"}, ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_delegate_for_custom_op", srcs = [ "edgetpu_delegate_for_custom_op.cc", @@ -198,7 +200,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_delegate_for_custom_op_tflite_plugin", srcs = [ "edgetpu_delegate_for_custom_op_tflite_plugin.cc", @@ -206,13 +208,14 @@ cc_library( deps = [ ":edgetpu_delegate_for_custom_op", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", "//tflite/public:edgetpu", "@org_tensorflow//tensorflow/lite:kernel_api", ], alwayslink = 1, ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_c", srcs = [ "edgetpu_c.cc", diff --git a/tflite/edgetpu_context_factory.cc b/tflite/edgetpu_context_factory.cc index 167d96a..50319bc 100644 --- a/tflite/edgetpu_context_factory.cc +++ b/tflite/edgetpu_context_factory.cc @@ -25,9 +25,12 @@ namespace darwinn { namespace tflite { const char* EdgeTpuContextFactory::GetDescriptionForDeviceTypeOptions() { - static const std::string desc = - StringPrintf("Type of Edge TPU device. Possible choices are %s | %s | %s", - kDeviceTypeDefault, kDeviceTypeApexUsb, kDeviceTypeApexPci); + static const std::string desc = [] { + auto s = StringPrintf( + "Type of Edge TPU device. Possible choices are %s | %s | %s", + kDeviceTypeDefault, kDeviceTypeApexUsb, kDeviceTypeApexPci); + return s; + }(); return desc.c_str(); } diff --git a/tflite/edgetpu_delegate_for_custom_op_tflite_plugin.cc b/tflite/edgetpu_delegate_for_custom_op_tflite_plugin.cc index 725536b..2ed67b9 100644 --- a/tflite/edgetpu_delegate_for_custom_op_tflite_plugin.cc +++ b/tflite/edgetpu_delegate_for_custom_op_tflite_plugin.cc @@ -14,6 +14,7 @@ #include "absl/strings/match.h" #include "absl/strings/numbers.h" +#include "absl/types/optional.h" #include "tflite/edgetpu_delegate_for_custom_op.h" #include "tflite/public/edgetpu.h" #include "tensorflow/lite/builtin_op_data.h" @@ -30,7 +31,6 @@ typedef void (*ErrorHandler)(const char*); constexpr char kUsb[] = "usb"; constexpr char kPci[] = "pci"; constexpr char kOptionDevice[] = "device"; -constexpr int kAnyDevice = -1; bool MatchDevice(const std::string& s, const std::string& type, int* index) { const auto prefix(type + ":"); @@ -40,16 +40,25 @@ bool MatchDevice(const std::string& s, const std::string& type, int* index) { return true; } -std::shared_ptr GetEdgeTpuContext( - DeviceType type, int index, const EdgeTpuManager::DeviceOptions& options) { - auto* manager = EdgeTpuManager::GetSingleton(); - if (index < 0) { - return manager->OpenDevice(type); +std::shared_ptr GetEdgeTpuContext( + absl::optional device_type, absl::optional device_index, + const edgetpu::EdgeTpuManager::DeviceOptions& options) { + auto* manager = edgetpu::EdgeTpuManager::GetSingleton(); + if (!device_index.has_value()) { + return device_type.has_value() ? manager->OpenDevice(device_type.value()) + : manager->OpenDevice(); } else { - int i = 0; - for (auto& record : manager->EnumerateEdgeTpu()) - if (record.type == type && i++ == index) - return manager->OpenDevice(record.type, record.path); + const int index = device_index.value(); + auto tpus = manager->EnumerateEdgeTpu(); + if (device_type.has_value()) { + int i = 0; + for (auto& record : tpus) + if (record.type == device_type.value() && i++ == index) + return manager->OpenDevice(record.type, record.path, options); + } else { + if (index < tpus.size()) + return manager->OpenDevice(tpus[index].type, tpus[index].path, options); + } return nullptr; } } @@ -61,13 +70,17 @@ std::shared_ptr GetEdgeTpuContext( return EdgeTpuManager::GetSingleton()->OpenDevice(); } else { const auto& device = it->second; - if (device == kUsb) { - return GetEdgeTpuContext(DeviceType::kApexUsb, kAnyDevice, options); + if (device.empty()) { + return GetEdgeTpuContext(absl::nullopt, absl::nullopt, options); + } else if (device == kUsb) { + return GetEdgeTpuContext(DeviceType::kApexUsb, absl::nullopt, options); } else if (device == kPci) { - return GetEdgeTpuContext(DeviceType::kApexPci, kAnyDevice, options); + return GetEdgeTpuContext(DeviceType::kApexPci, absl::nullopt, options); } else { int index; - if (MatchDevice(device, kUsb, &index)) { + if (MatchDevice(device, "", &index)) { + return GetEdgeTpuContext(absl::nullopt, index, options); + } else if (MatchDevice(device, kUsb, &index)) { return GetEdgeTpuContext(DeviceType::kApexUsb, index, options); } else if (MatchDevice(device, kPci, &index)) { return GetEdgeTpuContext(DeviceType::kApexPci, index, options); @@ -77,22 +90,25 @@ std::shared_ptr GetEdgeTpuContext( } } } - } // namespace extern "C" { // Recognized input options: -// "device": ["usb", "usb:", "pci", "pci:"] +// "device", possible values: +// "" -- any TPU device +// ":" -- TPU device of any type according to enumeration order +// "usb" -- any TPU device of USB type +// "usb:" -- TPU device of USB type according to enumeration order +// "pci" -- any TPU device of PCIe type +// "pci:" -- TPU device of PCIe type according to enumeration order // -// "usb" or "pci" define any available USB/PCI TPU device. -// "usb:" or "pci:" define specific USB/PCI TPU device -// according to the enumeration order from -// `edgetpu::EdgeTpuManager::EnumerateEdgeTpu` call. +// Enumeration order is defined by `edgetpu::EdgeTpuManager::EnumerateEdgeTpu` +// call. // // All options are forwarded to `edgetpu::EdgeTpuManager::OpenDevice` -// call when "device" has a form of "usb:" or "pci:", i.e. the -// following are supported as well: +// call when "device" has a form of ":", "usb:", or "pci:", +// i.e. the following are supported as well: // "Performance": ["Low", "Medium", "High", "Max"] (Default is "Max") // "Usb.AlwaysDfu": ["True", "False"] (Default is "False") // "Usb.MaxBulkInQueueLength": ["0",.., "255"] (Default is "32") diff --git a/tflite/edgetpu_manager_direct.cc b/tflite/edgetpu_manager_direct.cc index f1e7a09..8d297d7 100644 --- a/tflite/edgetpu_manager_direct.cc +++ b/tflite/edgetpu_manager_direct.cc @@ -476,9 +476,15 @@ std::shared_ptr EdgeTpuManagerDirect::OpenDeviceInternal( } } - std::unique_ptr driver_wrapper = - MakeDriverWrapper(request_device_type, extended_device_path, options, - /*exclusive_ownership=*/false); + EdgeTpuManager::DeviceOptions device_options(options); + if (request_device_type == edgetpu::DeviceType::kApexUsb && + device_options.find("Usb.MaxBulkInQueueLength") == + device_options.end()) { + device_options["Usb.MaxBulkInQueueLength"] = "8"; + } + std::unique_ptr driver_wrapper = MakeDriverWrapper( + request_device_type, extended_device_path, device_options, + /*exclusive_ownership=*/false); if (!driver_wrapper) { // Returns a null pointer on error. diff --git a/tflite/public/BUILD b/tflite/public/BUILD index 63db841..e4c55f8 100644 --- a/tflite/public/BUILD +++ b/tflite/public/BUILD @@ -14,25 +14,12 @@ # BUILD rules for DarwiNN TfLite Custom-op public interface. +load("//:libedgetpu_cc_rules.bzl", "libedgetpu_cc_binary", "libedgetpu_cc_library") + package( default_visibility = ["//visibility:public"], ) -# This group is meant for users of USB TPUs who also have a dependency to -# libcap (ex: CNS file support depends on libcap). Using the default USB -# driver wouldn't work in that case as libUSB is dynamically linked and -# thus the global _cap_names in libcap ends up twice in your binary, -# causing an ODR violation. Also to comply with libUSB LGPL license, this -# version only exist for internal usage (no external release, this is why -# the user are limited to this group). -package_group( - name = "libusb_statically_linked_users", - packages = [ - "//lifescience/cad/ophthalmology/arda_camera/dagon/...", - "//vr/stargate/...", - ], -) - licenses(["notice"]) # Version number used in the soname of shared libraries. @@ -43,8 +30,39 @@ SHARED_LIBRARY_LINKOPTS = [ "-Wl,--version-script=$(location libedgetpu.lds)", ] +DLL_ADDITIONAL_LINKOPTS = [ + ":edgetpu.res", +] + +DLL_LINKOPTS = [ + "$(location edgetpu.res)", +] + +config_setting( + name = "dbg", + values = { + "compilation_mode": "dbg", + }, +) + +genrule( + name = "dll_res_gen", + srcs = [ + ":edgetpu.rc", + ], + outs = [ + "edgetpu.res", + ], + cmd_bat = "rc.exe " + + select({ + ":dbg": "/D_DEBUG", + "//conditions:default": "", + }) + + " /nologo /fo $(location edgetpu.res) $(location edgetpu.rc)", +) + # Header for external use. -cc_library( +libedgetpu_cc_library( name = "edgetpu", hdrs = [ "edgetpu.h", @@ -58,7 +76,7 @@ cc_library( ], ) -cc_library( +libedgetpu_cc_library( name = "edgetpu_c", hdrs = [ "edgetpu_c.h", @@ -68,40 +86,38 @@ cc_library( "//conditions:default": [], }), deps = [ - "@org_tensorflow//tensorflow/lite:context", + "@org_tensorflow//tensorflow/lite/c:common", ], ) # Shared library for external use. # Explicit variant for all(pci/usb). -cc_binary( +libedgetpu_cc_binary( name = "libedgetpu_direct_all.so", linkopts = SHARED_LIBRARY_LINKOPTS, linkshared = 1, linkstatic = 1, deps = [ "libedgetpu.lds", - "//driver/beagle:beagle_all_driver_provider", - "//tflite:custom_op_direct", + ":edgetpu_internal_direct_all", ], ) # Shared library for external use. # Explicit variant for Beagle PCIe. -cc_binary( +libedgetpu_cc_binary( name = "libedgetpu_direct_pci.so", linkopts = SHARED_LIBRARY_LINKOPTS, linkshared = 1, linkstatic = 1, deps = [ "libedgetpu.lds", - "//driver/beagle:beagle_pci_driver_provider", - "//tflite:custom_op_direct", + ":edgetpu_internal_direct_pci", ], ) # Shared library for linking of applications not depending on a particular driver provider. -cc_binary( +libedgetpu_cc_binary( name = "libedgetpu_bare.so", linkopts = SHARED_LIBRARY_LINKOPTS, linkshared = 1, @@ -114,19 +130,18 @@ cc_binary( # Shared library for external use. # Explicit variant for Beagle USB. -cc_binary( +libedgetpu_cc_binary( name = "libedgetpu_direct_usb.so", linkopts = SHARED_LIBRARY_LINKOPTS, linkshared = 1, linkstatic = 1, deps = [ "libedgetpu.lds", - "//driver/beagle:beagle_usb_driver_provider", - "//tflite:custom_op_direct", + ":edgetpu_internal_direct_usb", ], ) -cc_binary( +libedgetpu_cc_binary( name = "libedgetpu_direct_usb.dylib", linkopts = [ "-Wl,-install_name,@rpath/libedgetpu." + VERSION + ".dylib", @@ -139,22 +154,145 @@ cc_binary( "notap", ], deps = [ - "//driver/beagle:beagle_usb_driver_provider", - "//tflite:custom_op_direct", + ":edgetpu_internal_direct_usb", ], ) -cc_binary( +libedgetpu_cc_binary( name = "edgetpu_direct_usb.dll", + additional_linker_inputs = DLL_ADDITIONAL_LINKOPTS, + linkopts = DLL_LINKOPTS, + linkshared = 1, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [ + ":edgetpu_internal_direct_usb", + "@libusb//:shared", + ], +) + +libedgetpu_cc_binary( + name = "edgetpu_direct_pci.dll", + additional_linker_inputs = DLL_ADDITIONAL_LINKOPTS, + linkopts = DLL_LINKOPTS, linkshared = 1, tags = [ "manual", "nobuilder", "notap", ], + deps = [ + ":edgetpu_internal_direct_pci_windows", + ], +) + +filegroup( + name = "edgetpu_direct_pci_interface_library", + srcs = [":edgetpu_direct_pci.dll"], + output_group = "interface_library", +) + +cc_import( + name = "edgetpu_direct_pci", + hdrs = [ + "edgetpu.h", + ], + interface_library = ":edgetpu_direct_pci_interface_library", + shared_library = ":edgetpu_direct_pci.dll", +) + +libedgetpu_cc_binary( + name = "edgetpu_direct_all.dll", + additional_linker_inputs = DLL_ADDITIONAL_LINKOPTS, + linkopts = DLL_LINKOPTS, + linkshared = 1, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [ + ":edgetpu_internal_direct_all_windows", + "@libusb//:shared", + ], +) + +filegroup( + name = "edgetpu_direct_all_interface_library", + srcs = [":edgetpu_direct_all.dll"], + output_group = "interface_library", +) + +cc_import( + name = "edgetpu_direct_all", + hdrs = [ + "edgetpu.h", + ], + interface_library = ":edgetpu_direct_all_interface_library", + shared_library = ":edgetpu_direct_all.dll", +) + +libedgetpu_cc_library( + name = "edgetpu_internal_direct_all", + deps = [ + "//driver/beagle:beagle_all_driver_provider_linux", + "//tflite:custom_op_direct", + ], +) + +libedgetpu_cc_library( + name = "edgetpu_internal_direct_all_windows", + deps = [ + "//driver/beagle:beagle_all_driver_provider_windows", + "//tflite:custom_op_direct", + ], +) + +libedgetpu_cc_library( + name = "edgetpu_internal_direct_pci", + deps = [ + "//driver/beagle:beagle_pci_driver_provider_linux", + "//tflite:custom_op_direct", + ], +) + +libedgetpu_cc_library( + name = "edgetpu_internal_direct_pci_windows", + deps = [ + "//driver/beagle:beagle_pci_driver_provider_windows", + "//tflite:custom_op_direct", + ], +) + +libedgetpu_cc_library( + name = "edgetpu_internal_direct_usb", deps = [ "//driver/beagle:beagle_usb_driver_provider", "//tflite:custom_op_direct", - "@libusb//:shared", ], ) + +cc_library( + name = "oss_edgetpu_direct_usb", + deps = [":edgetpu_internal_direct_usb"], +) + +cc_library( + name = "oss_edgetpu_direct_pci", + deps = select({ + "//:windows": [":edgetpu_internal_direct_pci_windows"], + "//conditions:default": [":edgetpu_internal_direct_pci"], + }), +) + +cc_library( + name = "oss_edgetpu_direct_all", + deps = select({ + "//:windows": [":edgetpu_internal_direct_all_windows"], + "//:darwin": [":edgetpu_internal_direct_usb"], + "//conditions:default": [":edgetpu_internal_direct_all"], + }), +) diff --git a/tflite/public/edgetpu.h b/tflite/public/edgetpu.h index 2f8f18e..ed4e4e2 100644 --- a/tflite/public/edgetpu.h +++ b/tflite/public/edgetpu.h @@ -13,70 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// -// This header file defines EdgeTpuManager, and EdgeTpuContext. -// EdgeTpuContext is an object associated with one or more tflite::Interpreter. -// Instances of this class should be allocated through -// EdgeTpuManager::NewEdgeTpuContext. -// More than one Interpreter instances can point to the same context. This means -// the tasks from both would be executed under the same TPU context. -// The lifetime of this context must be longer than all associated -// tflite::Interpreter instances. -// -// Typical usage with NNAPI: -// -// std::unique_ptr interpreter; -// tflite::ops::builtin::BuiltinOpResolver resolver; -// auto model = -// tflite::FlatBufferModel::BuildFromFile(model_file_name.c_str()); -// -// // Registers edge TPU custom op handler with Tflite resolver. -// resolver.AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); -// -// tflite::InterpreterBuilder(*model, resolver)(&interpreter); -// -// interpreter->AllocateTensors(); -// .... (Prepare input tensors) -// interpreter->Invoke(); -// .... (retrieving the result from output tensors) -// -// // Releases interpreter instance to free up resources associated with -// // this custom op. -// interpreter.reset(); -// -// Typical usage with Non-NNAPI: -// -// // Sets up the tpu_context. -// auto tpu_context = -// edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice(); -// -// std::unique_ptr interpreter; -// tflite::ops::builtin::BuiltinOpResolver resolver; -// auto model = -// tflite::FlatBufferModel::BuildFromFile(model_file_name.c_str()); -// -// // Registers edge TPU custom op handler with Tflite resolver. -// resolver.AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); -// -// tflite::InterpreterBuilder(*model, resolver)(&interpreter); -// -// // Binds a context with a specific interpreter. -// interpreter->SetExternalContext(kTfLiteEdgeTpuContext, -// tpu_context.get()); -// -// // Note that all edge TPU context set ups should be done before this -// // function is called. -// interpreter->AllocateTensors(); -// .... (Prepare input tensors) -// interpreter->Invoke(); -// .... (retrieving the result from output tensors) -// -// // Releases interpreter instance to free up resources associated with -// // this custom op. -// interpreter.reset(); -// -// // Closes the edge TPU. -// tpu_context.reset(); +// @cond BEGIN doxygen exclude +// This header file defines EdgeTpuManager and EdgeTpuContext. +// See below for more details. #ifndef TFLITE_PUBLIC_EDGETPU_H_ #define TFLITE_PUBLIC_EDGETPU_H_ @@ -101,26 +40,34 @@ limitations under the License. #else #define EDGETPU_EXPORT __attribute__((visibility("default"))) #endif // _WIN32 +// END doxygen exclude @endcond namespace edgetpu { -// EdgeTPU custom op. +// Edge TPU custom op. static const char kCustomOp[] = "edgetpu-custom-op"; +// The device interface used with the host enum class DeviceType { + // PCIe Gen2 x1 kApexPci = 0, + // USB 2.0 or 3.1 Gen1 kApexUsb = 1, }; class EdgeTpuContext; -// Singleton edge TPU manager for allocating new TPU contexts. +// Singleton Edge TPU manager for allocating new TPU contexts. // Functions in this interface are thread-safe. class EDGETPU_EXPORT EdgeTpuManager { public: + // See EdgeTpuContext::GetDeviceOptions(). using DeviceOptions = std::unordered_map; + // Details about a particular Edge TPU struct DeviceEnumerationRecord { + // The Edge TPU device type, either PCIe or USB DeviceType type; + // System path for the Edge TPU device std::string path; // Returns true if two enumeration records point to the same device. @@ -136,10 +83,12 @@ class EDGETPU_EXPORT EdgeTpuManager { } }; - // Returns pointer to the singleton object, or nullptr if not supported on + // Returns a pointer to the singleton object, or nullptr if not supported on // this platform. static EdgeTpuManager* GetSingleton(); + // @cond BEGIN doxygen exclude for deprecated APIs. + // NewEdgeTpuContext family functions has been deprecated and will be removed // in the future. Please use OpenDevice for new code. // @@ -180,49 +129,63 @@ class EDGETPU_EXPORT EdgeTpuManager { virtual std::unique_ptr NewEdgeTpuContext( DeviceType device_type, const std::string& device_path, const DeviceOptions& options) = 0; + // END doxygen exclude for deprecated APIs @endcond + // Enumerates all connected Edge TPU devices. virtual std::vector EnumerateEdgeTpu() const = 0; - // OpenDevice family of functions return a shared_ptr to EdgeTpuContext, with - // the intention that the device can be shared among multiple software - // components. - // - // These functions seek shared ownership of the opened devices. As they - // cannot open devices already opened by NewEdgeTpuContext, and vice versa. - // The device would be closed after the last reference leaves scope. - // Opens the default Edge TPU device. // + // All `OpenDevice` functions return a shared_ptr to EdgeTpuContext, with + // the intention that the device can be shared among multiple software + // components. The device is closed after the last reference leaves scope. + // // Multiple invocations of this function could return handle to the same // device, but there is no guarantee. // - // Returns a shared pointer to Edge TPU device. The shared_ptr could point to + // You cannot open devices opened by `NewEdgeTpuContext`, and vice versa. + // + // @return A shared pointer to Edge TPU device. The shared_ptr could point to // nullptr in case of error. virtual std::shared_ptr OpenDevice() = 0; // Same as above, but the returned context is associated with the specified // type. + // + // @param device_type The DeviceType you want to open. virtual std::shared_ptr OpenDevice( DeviceType device_type) = 0; // Same as above, but the returned context is associated with the specified // type and device path. If path is empty, any device of the specified type // could be returned. + // + // @param device_type The DeviceType you want to open. + // @param device_path A path to the device you want. + // + // @return A shared pointer to Edge TPU device. The shared_ptr could point to + // nullptr in case of error. virtual std::shared_ptr OpenDevice( DeviceType device_type, const std::string& device_path) = 0; - // Same as above, but the specified options would used to create a new context + // Same as above, but the specified options are used to create a new context // if no existing device is compatible with the specified type and path. // - // If a device of compatible type and path can be found, the options could be + // If a device of compatible type and path is not found, the options could be // ignored. It is the caller's responsibility to verify if the returned - // context is desirable, through #EdgeTpuContext::GetDeviceOptions(). + // context is desirable, through EdgeTpuContext::GetDeviceOptions(). // + // @param device_type The DeviceType you want to open. + // @param device_path A path to the device you want. + // @param options Specific criteria for the device you want. // Available options are: // - "Performance": ["Low", "Medium", "High", "Max"] (Default is "Max") // - "Usb.AlwaysDfu": ["True", "False"] (Default is "False") // - "Usb.MaxBulkInQueueLength": ["0",.., "255"] (Default is "32") + // + // @return A shared pointer to Edge TPU device. The shared_ptr could point to + // nullptr in case of error. virtual std::shared_ptr OpenDevice( DeviceType device_type, const std::string& device_path, const DeviceOptions& options) = 0; @@ -233,11 +196,13 @@ class EDGETPU_EXPORT EdgeTpuManager { virtual std::vector> GetOpenedDevices() const = 0; - // Sets verbosity of operating logs related to edge TPU. - // Verbosity level can be set to [0-10], in which 10 is the most verbose. + // Sets the verbosity of operating logs related to each Edge TPU. + // + // @param verbosity The verbosity level, which may be 0 to 10. + // 10 is the most verbose; 0 is the default. virtual TfLiteStatus SetVerbosity(int verbosity) = 0; - // Returns the version of EdgeTPU runtime stack. + // Returns the version of the Edge TPU runtime stack. virtual std::string Version() const = 0; protected: @@ -245,12 +210,74 @@ class EDGETPU_EXPORT EdgeTpuManager { virtual ~EdgeTpuManager() = default; }; -// External context to be assigned through -// tflite::Interpreter::SetExternalContext. -// One should get hold of either shared_ptr from EdgeTpuManager::OpenDevice, or -// unique_ptr from EdgeTpuManager::NewEdgeTpuContext, to ensure ownership, and -// avoid using this pointer directly. +// EdgeTpuContext is an object associated with one or more tflite::Interpreter. +// Instances of this class should be allocated with EdgeTpuManager::OpenDevice. +// +// More than one Interpreter instances can point to the same context. This means +// the tasks from both would be executed under the same TPU context. +// The lifetime of this context must be longer than all associated +// tflite::Interpreter instances. +// // Functions in this interface are thread-safe. +// +// Typical usage with Coral: +// +// ``` +// // Sets up the tpu_context. +// auto tpu_context = +// edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice(); +// +// std::unique_ptr interpreter; +// tflite::ops::builtin::BuiltinOpResolver resolver; +// auto model = +// tflite::FlatBufferModel::BuildFromFile(model_file_name.c_str()); +// +// // Registers Edge TPU custom op handler with Tflite resolver. +// resolver.AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); +// +// tflite::InterpreterBuilder(*model, resolver)(&interpreter); +// +// // Binds a context with a specific interpreter. +// interpreter->SetExternalContext(kTfLiteEdgeTpuContext, +// tpu_context.get()); +// +// // Note that all edge TPU context set ups should be done before this +// // function is called. +// interpreter->AllocateTensors(); +// .... (Prepare input tensors) +// interpreter->Invoke(); +// .... (retrieving the result from output tensors) +// +// // Releases interpreter instance to free up resources associated with +// // this custom op. +// interpreter.reset(); +// +// // Closes the edge TPU. +// tpu_context.reset(); +// ``` +// +// Typical usage with Android NNAPI: +// +// ``` +// std::unique_ptr interpreter; +// tflite::ops::builtin::BuiltinOpResolver resolver; +// auto model = +// tflite::FlatBufferModel::BuildFromFile(model_file_name.c_str()); +// +// // Registers Edge TPU custom op handler with Tflite resolver. +// resolver.AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); +// +// tflite::InterpreterBuilder(*model, resolver)(&interpreter); +// +// interpreter->AllocateTensors(); +// .... (Prepare input tensors) +// interpreter->Invoke(); +// .... (retrieving the result from output tensors) +// +// // Releases interpreter instance to free up resources associated with +// // this custom op. +// interpreter.reset(); +// ``` class EdgeTpuContext : public TfLiteExternalContext { public: virtual ~EdgeTpuContext() = 0; @@ -276,7 +303,7 @@ class EdgeTpuContext : public TfLiteExternalContext { }; // Returns pointer to an instance of TfLiteRegistration to handle -// EdgeTPU custom ops, to be used with +// Edge TPU custom ops, to be used with // tflite::ops::builtin::BuiltinOpResolver::AddCustom EDGETPU_EXPORT TfLiteRegistration* RegisterCustomOp(); diff --git a/tflite/public/edgetpu.rc b/tflite/public/edgetpu.rc new file mode 100644 index 0000000..f378cad --- /dev/null +++ b/tflite/public/edgetpu.rc @@ -0,0 +1,42 @@ +// Redefine some constants we would usually get from winver.h +// Bazel doesn't know the correct include paths to pass along +// to rc to pick up the header. +#define VS_VERSION_INFO 1 +#define VS_FFI_FILEFLAGSMASK 0x3FL +#define VS_FF_DEBUG 0x1L +#define VOS__WINDOWS32 0x4L +#define VFT_DLL 0x2L + +#define VER_FILEVERSION 14,0,0,0 +#define VER_FILEVERSION_STR "14.0.0.0\0" + +VS_VERSION_INFO VERSIONINFO +FILEVERSION VER_FILEVERSION +PRODUCTVERSION VER_FILEVERSION +FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG +FILEFLAGS VS_FF_DEBUG +#else +FILEFLAGS 0 +#endif +FILEOS VOS__WINDOWS32 +FILETYPE VFT_DLL +FILESUBTYPE 0 +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" + BEGIN + VALUE "FileDescription", "EdgeTPU interface library\0" + VALUE "FileVersion", VER_FILEVERSION_STR + VALUE "InternalName", "edgetpu.dll\0" + VALUE "LegalCopyright", "(C) 2019-2020 Google, LLC\0" + VALUE "ProductName", "edgetpu\0" + VALUE "ProductVersion", VER_FILEVERSION_STR + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1252 + END +END \ No newline at end of file diff --git a/tflite/public/edgetpu_c.h b/tflite/public/edgetpu_c.h index 54f4f05..64b73de 100644 --- a/tflite/public/edgetpu_c.h +++ b/tflite/public/edgetpu_c.h @@ -55,7 +55,7 @@ limitations under the License. #ifndef TFLITE_PUBLIC_EDGETPU_C_H_ #define TFLITE_PUBLIC_EDGETPU_C_H_ -#include "tensorflow/lite/context.h" +#include "tensorflow/lite/c/common.h" #if defined(_WIN32) #ifdef EDGETPU_COMPILE_LIBRARY diff --git a/tflite/public/libedgetpu.lds b/tflite/public/libedgetpu.lds index 6a07c66..595f751 100644 --- a/tflite/public/libedgetpu.lds +++ b/tflite/public/libedgetpu.lds @@ -10,6 +10,33 @@ VER_1.0 { /* Edge TPU C API functions. */ edgetpu_*; + + /* Export `new` and `delete` variants to workaround b/167358360. + */ + _ZdaPv; + _ZdaPvRKSt9nothrow_t; + _ZdaPvSt11align_val_t; + _ZdaPvSt11align_val_tRKSt9nothrow_t; + _ZdaPvm; + _ZdaPvmSt11align_val_t; + _ZdlPv; + _ZdlPvRKSt9nothrow_t; + _ZdlPvSt11align_val_t; + _ZdlPvSt11align_val_tRKSt9nothrow_t; + _ZdlPvm; + _ZdlPvmSt11align_val_t; + _Znam; + _ZnamRKSt9nothrow_t; + _ZnamSt11align_val_t; + _ZnamSt11align_val_tRKSt9nothrow_t; + _Znwm; + _ZnwmRKSt9nothrow_t; + _ZnwmSt11align_val_t; + _ZnwmSt11align_val_tRKSt9nothrow_t; + + /* Export std::nothrow just in case (b/130758776). + */ + _ZSt7nothrow; local: *; }; diff --git a/workspace.bzl b/workspace.bzl new file mode 100644 index 0000000..5d05c69 --- /dev/null +++ b/workspace.bzl @@ -0,0 +1,109 @@ +""" +This module contains workspace definitions for building and using libedgetpu. +""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +TENSORFLOW_COMMIT = "f394a768719a55b5c351ed1ecab2ec6f16f99dd4" +TENSORFLOW_SHA256 = "cb286abee7ee9cf5c8701d85fcc88f0fd59e72492ec4f254156de486e3e905c1" + +IO_BAZEL_RULES_CLOSURE_COMMIT = "308b05b2419edb5c8ee0471b67a40403df940149" +IO_BAZEL_RULES_CLOSURE_SHA256 = "5b00383d08dd71f28503736db0500b6fb4dda47489ff5fc6bed42557c07c6ba9" + +CORAL_CROSSTOOL_COMMIT = "9e00d5be43bf001f883b5700f5d04882fea00229" +CORAL_CROSSTOOL_SHA256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb" + +def libedgetpu_dependencies( + tensorflow_commit = TENSORFLOW_COMMIT, + tensorflow_sha256 = TENSORFLOW_SHA256, + io_bazel_rules_closure_commit = IO_BAZEL_RULES_CLOSURE_COMMIT, + io_bazel_rules_closure_sha256 = IO_BAZEL_RULES_CLOSURE_SHA256, + coral_crosstool_commit = CORAL_CROSSTOOL_COMMIT, + coral_crosstool_sha256 = CORAL_CROSSTOOL_SHA256): + maybe( + http_archive, + name = "org_tensorflow", + urls = [ + "https://github.com/tensorflow/tensorflow/archive/" + tensorflow_commit + ".tar.gz", + ], + sha256 = tensorflow_sha256, + strip_prefix = "tensorflow-" + tensorflow_commit, + ) + + maybe( + http_archive, + name = "io_bazel_rules_closure", + urls = [ + "https://github.com/bazelbuild/rules_closure/archive/" + io_bazel_rules_closure_commit + ".tar.gz", # 2019-06-13 + ], + sha256 = io_bazel_rules_closure_sha256, + strip_prefix = "rules_closure-" + io_bazel_rules_closure_commit, + ) + + maybe( + http_archive, + name = "coral_crosstool", + urls = [ + "https://github.com/google-coral/crosstool/archive/" + coral_crosstool_commit + ".tar.gz", + ], + sha256 = coral_crosstool_sha256, + strip_prefix = "crosstool-" + coral_crosstool_commit, + ) + + maybe( + libusb_repository, + name = "libusb", + ) + +def _libusb_impl(ctx): + lower_name = ctx.os.name.lower() + if lower_name.startswith("linux"): + path = "/usr/include" + build_file_content = """ +cc_library( + name = "headers", + includes = ["root"], + hdrs = ["root/libusb-1.0/libusb.h"], + visibility = ["//visibility:public"], +) +""" + elif lower_name.startswith("windows"): + path = str(ctx.path(Label("@//:WORKSPACE"))) + "/../../libusb-1.0.22" + build_file_content = """ +cc_library( + name = "headers", + includes = ["root/include"], + hdrs = ["root/include/libusb-1.0/libusb.h"], + visibility = ["//visibility:public"], +) +cc_import( + name = "shared", + interface_library = "root/MS64/dll/libusb-1.0.lib", + shared_library = "root/MS64/dll/libusb-1.0.dll", + visibility = ["//visibility:public"], +) +""" + elif lower_name.startswith("mac os x"): + path = "/opt/local/include/" + build_file_content = """ +cc_library( + name = "headers", + includes = ["root"], + hdrs = ["root/libusb-1.0/libusb.h"], + visibility = ["//visibility:public"], +) +""" + else: + fail("Unsupported operating system.") + + ctx.symlink(path, "root") + ctx.file( + "BUILD", + content = build_file_content, + executable = False, + ) + +libusb_repository = repository_rule( + implementation = _libusb_impl, +)