Merge branch 'main' of github.com:isl-org/Open3D into realsense

3rdparty/find_dependencies.cmake

@@ -1339,6 +1339,7 @@ if(BUILD_GUI)
                     if (CPP_LIBRARY AND CPPABI_LIBRARY)
                         set(CLANG_LIBDIR ${llvm_lib_dir})
                         message(STATUS "CLANG_LIBDIR found in ubuntu-default: ${CLANG_LIBDIR}")
+                        set(LIBCPP_VERSION ${llvm_ver})
                         break()
                     endif()
                 endforeach()
@@ -1363,7 +1364,10 @@ if(BUILD_GUI)
                              llvm-8/lib
                              llvm-7/lib
                 )
+                file(REAL_PATH ${CPPABI_LIBRARY} CPPABI_LIBRARY)
                 get_filename_component(CLANG_LIBDIR ${CPPABI_LIBRARY} DIRECTORY)
+                string(REGEX MATCH "llvm-([0-9]+)/lib" _ ${CLANG_LIBDIR})
+                set(LIBCPP_VERSION ${CMAKE_MATCH_1})
             endif()
 
             # Find clang libraries at the exact path ${CLANG_LIBDIR}.
@@ -1379,6 +1383,11 @@ if(BUILD_GUI)
             target_link_libraries(3rdparty_filament INTERFACE -lstdc++
                                   ${CPP_LIBRARY} ${CPPABI_LIBRARY})
             message(STATUS "Filament C++ libraries: ${CPP_LIBRARY} ${CPPABI_LIBRARY}")
+            if (LIBCPP_VERSION GREATER 11)
+                message(WARNING "libc++ (LLVM) version ${LIBCPP_VERSION} > 11 includes libunwind that " 
+                "interferes with the system libunwind.so.8 and may crash Python code when exceptions "
+                "are used. Please consider using libc++ (LLVM) v11.")
+            endif()
         endif()
         if (APPLE)
             find_library(CORE_VIDEO CoreVideo)

cpp/open3d/core/kernel/ReductionCPU.cpp

@@ -122,13 +122,14 @@ class CPUReductionEngine {
         for (int64_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
             int64_t start = thread_idx * workload_per_thread;
             int64_t end = std::min(start + workload_per_thread, num_workloads);
+            scalar_t local_result = identity;
             for (int64_t workload_idx = start; workload_idx < end;
                  ++workload_idx) {
                 scalar_t* src = reinterpret_cast<scalar_t*>(
                         indexer.GetInputPtr(0, workload_idx));
-                thread_results[thread_idx] =
-                        element_kernel(*src, thread_results[thread_idx]);
+                local_result = element_kernel(*src, local_result);
             }
+            thread_results[thread_idx] = local_result;
         }
         scalar_t* dst = reinterpret_cast<scalar_t*>(indexer.GetOutputPtr(0));
         for (int64_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
@@ -190,14 +191,25 @@ class CPUArgReductionEngine {
         // elements. We need to keep track of the indices within each
         // sub-iteration.
         int64_t num_output_elements = indexer_.NumOutputElements();
+        if (num_output_elements <= 1) {
+            LaunchArgReductionKernelTwoPass(indexer_, reduce_func, identity);
+        } else {
+            LaunchArgReductionParallelDim(indexer_, reduce_func, identity);
+        }
+    }
 
+    template <typename scalar_t, typename func_t>
+    static void LaunchArgReductionParallelDim(const Indexer& indexer,
+                                              func_t reduce_func,
+                                              scalar_t identity) {
+        int64_t num_output_elements = indexer.NumOutputElements();
 #pragma omp parallel for schedule(static) \
         num_threads(utility::EstimateMaxThreads())
         for (int64_t output_idx = 0; output_idx < num_output_elements;
              output_idx++) {
             // sub_indexer.NumWorkloads() == ipo.
-            // sub_indexer's workload_idx is indexer_'s ipo_idx.
-            Indexer sub_indexer = indexer_.GetPerOutputIndexer(output_idx);
+            // sub_indexer's workload_idx is indexer's ipo_idx.
+            Indexer sub_indexer = indexer.GetPerOutputIndexer(output_idx);
             scalar_t dst_val = identity;
             for (int64_t workload_idx = 0;
                  workload_idx < sub_indexer.NumWorkloads(); workload_idx++) {
@@ -212,6 +224,52 @@ class CPUArgReductionEngine {
         }
     }
 
+    /// Create num_threads workers to compute partial arg reductions
+    /// and then reduce to the final results.
+    /// This only applies to arg reduction op with one output.
+    template <typename scalar_t, typename func_t>
+    static void LaunchArgReductionKernelTwoPass(const Indexer& indexer,
+                                                func_t reduce_func,
+                                                scalar_t identity) {
+        if (indexer.NumOutputElements() > 1) {
+            utility::LogError(
+                    "Internal error: two-pass arg reduction only works for "
+                    "single-output arg reduction ops.");
+        }
+        int64_t num_workloads = indexer.NumWorkloads();
+        int64_t num_threads = utility::EstimateMaxThreads();
+        int64_t workload_per_thread =
+                (num_workloads + num_threads - 1) / num_threads;
+        std::vector<int64_t> thread_results_idx(num_threads, 0);
+        std::vector<scalar_t> thread_results_val(num_threads, identity);
+
+#pragma omp parallel for schedule(static) \
+        num_threads(utility::EstimateMaxThreads())
+        for (int64_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
+            int64_t start = thread_idx * workload_per_thread;
+            int64_t end = std::min(start + workload_per_thread, num_workloads);
+            scalar_t local_result_val = identity;
+            int64_t local_result_idx = 0;
+            for (int64_t workload_idx = start; workload_idx < end;
+                 ++workload_idx) {
+                int64_t src_idx = workload_idx;
+                scalar_t* src_val = reinterpret_cast<scalar_t*>(
+                        indexer.GetInputPtr(0, workload_idx));
+                std::tie(local_result_idx, local_result_val) = reduce_func(
+                        src_idx, *src_val, local_result_idx, local_result_val);
+            }
+            thread_results_val[thread_idx] = local_result_val;
+            thread_results_idx[thread_idx] = local_result_idx;
+        }
+        scalar_t dst_val = identity;
+        int64_t* dst_idx = reinterpret_cast<int64_t*>(indexer.GetOutputPtr(0));
+        for (int64_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
+            std::tie(*dst_idx, dst_val) = reduce_func(
+                    thread_results_idx[thread_idx],
+                    thread_results_val[thread_idx], *dst_idx, dst_val);
+        }
+    }
+
 private:
     Indexer indexer_;
 };

cpp/open3d/core/nns/NanoFlannImpl.h

@@ -118,13 +118,6 @@ void _KnnSearchCPU(NanoFlannIndexHolderBase *holder,
         return;
     }
 
-    auto points_equal = [](const T *const p1, const T *const p2,
-                           size_t dimension) {
-        std::vector<T> p1_vec(p1, p1 + dimension);
-        std::vector<T> p2_vec(p2, p2 + dimension);
-        return p1_vec == p2_vec;
-    };
-
     std::vector<std::vector<TIndex>> neighbors_indices(num_queries);
     std::vector<std::vector<T>> neighbors_distances(num_queries);
     std::vector<uint32_t> neighbors_count(num_queries, 0);
@@ -147,8 +140,9 @@ void _KnnSearchCPU(NanoFlannIndexHolderBase *holder,
                     for (size_t valid_i = 0; valid_i < num_valid; ++valid_i) {
                         TIndex idx = result_indices[valid_i];
                         if (ignore_query_point &&
-                            points_equal(&queries[i * dimension],
-                                         &points[idx * dimension], dimension)) {
+                            std::equal(&queries[i * dimension],
+                                       &queries[i * dimension] + dimension,
+                                       &points[idx * dimension])) {
                             continue;
                         }
                         neighbors_indices[i].push_back(idx);
@@ -222,13 +216,6 @@ void _RadiusSearchCPU(NanoFlannIndexHolderBase *holder,
         return;
     }
 
-    auto points_equal = [](const T *const p1, const T *const p2,
-                           size_t dimension) {
-        std::vector<T> p1_vec(p1, p1 + dimension);
-        std::vector<T> p2_vec(p2, p2 + dimension);
-        return p1_vec == p2_vec;
-    };
-
     std::vector<std::vector<TIndex>> neighbors_indices(num_queries);
     std::vector<std::vector<T>> neighbors_distances(num_queries);
     std::vector<uint32_t> neighbors_count(num_queries, 0);
@@ -255,9 +242,9 @@ void _RadiusSearchCPU(NanoFlannIndexHolderBase *holder,
                     int num_neighbors = 0;
                     for (const auto &idx_dist : search_result) {
                         if (ignore_query_point &&
-                            points_equal(&queries[i * dimension],
-                                         &points[idx_dist.first * dimension],
-                                         dimension)) {
+                            std::equal(&queries[i * dimension],
+                                       &queries[i * dimension] + dimension,
+                                       &points[idx_dist.first * dimension])) {
                             continue;
                         }
                         neighbors_indices[i].push_back(idx_dist.first);

cpp/pybind/CMakeLists.txt

@@ -80,6 +80,9 @@ set(PYTHON_COMPILED_MODULE_DIR
 if (APPLE)
     set_target_properties(pybind PROPERTIES BUILD_RPATH "@loader_path;@loader_path/..")
 elseif (UNIX)
+    # Use RPATH instead of RUNPATH in pybind so that needed libc++.so can find child dependant libc++abi.so in RPATH
+    # https://stackoverflow.com/questions/69662319/managing-secondary-dependencies-of-shared-libraries
+    target_link_options(pybind PRIVATE "LINKER:--disable-new-dtags")
     set_target_properties(pybind PROPERTIES BUILD_RPATH "$ORIGIN;$ORIGIN/..")
 endif()
 set_target_properties(pybind PROPERTIES

cpp/pybind/t/geometry/boundingvolume.cpp

@@ -22,51 +22,19 @@ void pybind_boundingvolume_declarations(py::module& m) {
                std::shared_ptr<AxisAlignedBoundingBox>, Geometry,
                DrawableGeometry>
             aabb(m, "AxisAlignedBoundingBox",
-                 R"(A bounding box that is aligned along the coordinate axes
-and defined by the min_bound and max_bound."
-- (min_bound, max_bound): Lower and upper bounds of the bounding box for all
-axes.
-    - Usage
-        - AxisAlignedBoundingBox::GetMinBound()
-        - AxisAlignedBoundingBox::SetMinBound(const core::Tensor &min_bound)
-        - AxisAlignedBoundingBox::GetMaxBound()
-        - AxisAlignedBoundingBox::SetMaxBound(const core::Tensor &max_bound)
-    - Value tensor must have shape {3,}.
-    - Value tensor must have the same data type and device.
-    - Value tensor can only be float32 (default) or float64.
-    - The device of the tensor determines the device of the box.
+                 R"(A bounding box that is aligned along the coordinate axes and
+has the properties:
 
-- color: Color of the bounding box.
-    - Usage
-        - AxisAlignedBoundingBox::GetColor()
-        - AxisAlignedBoundingBox::SetColor(const core::Tensor &color)
-    - Value tensor must have shape {3,}.
-    - Value tensor can only be float32 (default) or float64.
-    - Value tensor can only be range [0.0, 1.0].)");
+- (``min_bound``, ``max_bound``): Lower and upper bounds of the bounding box for all axes. These are tensors with shape (3,) and a common data type and device. The data type can only be ``open3d.core.float32`` (default) or ``open3d.core.float64``. The device of the tensor determines the device of the box.
+- ``color``: Color of the bounding box is a tensor with shape (3,) and a data type ``open3d.core.float32`` (default) or ``open3d.core.float64``. Values can only be in the range [0.0, 1.0].)");
     py::class_<OrientedBoundingBox, PyGeometry<OrientedBoundingBox>,
                std::shared_ptr<OrientedBoundingBox>, Geometry, DrawableGeometry>
             obb(m, "OrientedBoundingBox",
-                R"(A bounding box oriented along an arbitrary frame of reference.
-- (center, rotation, extent): The oriented bounding box is defined by its
-center position, rotation maxtrix and extent.
-    - Usage
-        - OrientedBoundingBox::GetCenter()
-        - OrientedBoundingBox::SetCenter(const core::Tensor &center)
-        - OrientedBoundingBox::GetRotation()
-        - OrientedBoundingBox::SetRotation(const core::Tensor &rotation)
-    - Value tensor of center and extent must have shape {3,}.
-    - Value tensor of rotation must have shape {3, 3}.
-    - Value tensor must have the same data type and device.
-    - Value tensor can only be float32 (default) or float64.
-    - The device of the tensor determines the device of the box.
+                R"(A bounding box oriented along an arbitrary frame of reference
+with the properties:
 
-- color: Color of the bounding box.
-    - Usage
-        - OrientedBoundingBox::GetColor()
-        - OrientedBoundingBox::SetColor(const core::Tensor &color)
-    - Value tensor must have shape {3,}.
-    - Value tensor can only be float32 (default) or float64.
-    - Value tensor can only be range [0.0, 1.0].)");
+- (``center``, ``rotation``, ``extent``): The oriented bounding box is defined by its center position (shape (3,)), rotation maxtrix (shape (3,3)) and extent (shape (3,)).  Each of these tensors must have the same data type and device. The data type can only be ``open3d.core.float32`` (default) or ``open3d.core.float64``. The device of the tensor determines the device of the box.
+- ``color``: Color of the bounding box is a tensor with shape (3,) and a data type ``open3d.core.float32`` (default) or ``open3d.core.float64``. Values can only be in the range [0.0, 1.0].)");
 }
 void pybind_boundingvolume_definitions(py::module& m) {
     auto aabb = static_cast<py::class_<AxisAlignedBoundingBox,

cpp/tests/t/geometry/TensorMap.cpp

@@ -32,6 +32,13 @@ TEST_P(TensorMapPermuteDevices, Constructor) {
     // Primary key is required.
     EXPECT_ANY_THROW(t::geometry::TensorMap());
 
+    // Delete primary key.
+    EXPECT_ANY_THROW(tm0.Erase("positions"));
+
+    // Reserved keys.
+    EXPECT_ANY_THROW(tm0.insert(
+            {"primary_key", core::Tensor::Zeros({2, 3}, dtype, device)}));
+
     // Iterators.
     std::map<std::string, core::Tensor> tensor_map(
             {{"positions", core::Tensor::Zeros({10, 3}, dtype, device)},

docker/Dockerfile.ci

@@ -67,34 +67,48 @@ RUN if [ "${BUILD_SYCL_MODULE}" = "ON" ]; then \
         rm -rf /etc/apt/sources.list.d/oneAPI.list; \
     fi
 
-# Dependencies: basic
+# Dependencies: basic and python-build
 RUN apt-get update && apt-get install -y \
     git  \
     wget \
     curl \
     build-essential \
     pkg-config \
+    zlib1g \
+    zlib1g-dev \
+    libssl-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
  && rm -rf /var/lib/apt/lists/*
 
-# Miniconda or Intel conda
-# The **/open3d/bin paths are used during docker run, in this way docker run
+# pyenv or Intel Python
+# The pyenv python paths are used during docker run, in this way docker run
 # does not need to activate the environment again.
-ENV PATH="/root/miniconda3/bin:${PATH}"
-ENV PATH="/root/miniconda3/envs/open3d/bin:${PATH}"
+# The soft link from the python patch level version to the python mino version
+# ensures python wheel commands (i.e. open3d) are in PATH, since we don't know
+# which patch level pyenv will install (latest).
+ENV PYENV_ROOT=/root/.pyenv
+ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PYENV_ROOT/versions/$PYTHON_VERSION/bin:$PATH"
 ENV PATH="/opt/intel/oneapi/intelpython/latest/bin:${PATH}"
-ENV PATH="/opt/intel/oneapi/intelpython/latest/envs/open3d/bin:${PATH}"
 RUN if [ "${BUILD_SYCL_MODULE}" = "OFF" ]; then \
-        wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh; \
-        bash Miniconda3-latest-Linux-x86_64.sh -b; \
-        rm Miniconda3-latest-Linux-x86_64.sh; \
+        curl https://pyenv.run | bash \
+        && pyenv update \
+        && pyenv install $PYTHON_VERSION \
+        && pyenv global $PYTHON_VERSION \
+        && pyenv rehash \
+        && ln -s $PYENV_ROOT/versions/${PYTHON_VERSION}* $PYENV_ROOT/versions/${PYTHON_VERSION}; \
     fi
-RUN conda --version \
- && conda create -y -n open3d python=${PYTHON_VERSION}
+RUN python --version && pip --version
 
-# Activate open3d virtualenv
-# This works during docker build. It becomes the prefix of all RUN commands.
-# Ref: https://stackoverflow.com/a/60148365/1255535
-SHELL ["conda", "run", "-n", "open3d", "/bin/bash", "-o", "pipefail", "-c"]
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Dependencies: cmake
 ENV PATH=${HOME}/${CMAKE_VERSION}/bin:${PATH}

docker/docker_test.sh

@@ -141,7 +141,7 @@ cpp_python_linking_uninstall_test() {
     # Python test
     echo "pytest is randomized, add --randomly-seed=SEED to repeat the test sequence."
     ${docker_run} -i --rm "${DOCKER_TAG}" /bin/bash -c " \
-        python -m pytest python/test ${pytest_args} -s"
+        python  -W default -m pytest python/test ${pytest_args} -s"
     restart_docker_daemon_if_on_gcloud
 
     # Command-line tools test

docs/make_docs.py

@@ -77,9 +77,9 @@ def generate_rst(self):
             try:
                 module = self._try_import_module(module_name)
                 self._generate_module_class_function_docs(module_name, module)
-            except:
-                print("[Warning] Module {} cannot be imported.".format(
-                    module_name))
+            except Exception as _e:
+                print(f"[Warning] Module {module_name} cannot be imported: "
+                      f"{_e}.")
 
     @staticmethod
     def _get_documented_module_names():

docs/requirements.txt

@@ -1,9 +1,10 @@
 docutils==0.20.1
 furo==2023.9.10
-jinja2==3.1.3
+jinja2==3.1.4
 m2r2==0.3.3.post2
 matplotlib==3.7.3
 nbsphinx==0.9.3
 sphinx==7.1.2
 nbconvert==6.5.4
-lxml[html_clean]==5.2.1
+lxml==5.2.1
+lxml_html_clean==0.2.2