From b1ac0973a8ede188bd99806206dcbc48e322dc4e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 7 Nov 2024 13:34:41 +0100
Subject: [PATCH] fix count of interface elments

---
 include/OGL/MatrixWrapper/HostMatrix.H     |  3 ++-
 src/MatrixWrapper/HostMatrix.C             | 22 +++++++++++++---------
 unitTests/MatrixWrapper/HostMatrix_p2d.cpp | 16 +++++++++-------
 3 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/include/OGL/MatrixWrapper/HostMatrix.H b/include/OGL/MatrixWrapper/HostMatrix.H
index 13e4da01..7dcd050e 100644
--- a/include/OGL/MatrixWrapper/HostMatrix.H
+++ b/include/OGL/MatrixWrapper/HostMatrix.H
@@ -175,7 +175,8 @@ private:
     ** ret = [(1,2,1),(2,20, 2), (3, 20, 2) ...]
     **         i0   i1,   i...
     */
-    [[nodiscard]] std::vector<interface_locality> collect_cells_on_interfaces(
+    [[nodiscard]] std::pair<label, std::vector<interface_locality>>
+    collect_cells_on_interfaces(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
     /** Based on OpenFOAMs ldu matrix and interfaces this function computes
diff --git a/src/MatrixWrapper/HostMatrix.C b/src/MatrixWrapper/HostMatrix.C
index 70fe32a4..c45067d5 100644
--- a/src/MatrixWrapper/HostMatrix.C
+++ b/src/MatrixWrapper/HostMatrix.C
@@ -318,7 +318,8 @@ HostMatrixWrapper::create_communication_pattern() const
 }
 
 
-std::vector<interface_locality> HostMatrixWrapper::collect_cells_on_interfaces(
+std::pair<label,
+std::vector<interface_locality>> HostMatrixWrapper::collect_cells_on_interfaces(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // vector of neighbour cell idx connected to interface
@@ -327,9 +328,10 @@ std::vector<interface_locality> HostMatrixWrapper::collect_cells_on_interfaces(
     interface_idxs.reserve(interface_nnz);
     auto rank = get_exec_handler().get_rank();
 
-    label local_ctr = 0;
-    label interface_ctr = 0;
-    label interface_id = 0;
+    label local_ctr = 0; // count local interface element
+    label interface_ctr = 0; // count non_local interface elment
+    label interface_id = 0; // count number of interfaces
+    label total_ctr = 0;
 
     for (label i = 0; i < interfaces.size(); i++) {
         if (interface_getter(interfaces, i) == nullptr) {
@@ -338,6 +340,7 @@ std::vector<interface_locality> HostMatrixWrapper::collect_cells_on_interfaces(
         const auto iface{interface_getter(interfaces, i)};
         const auto &face_cells{iface->interface().faceCells()};
         const label interface_size = face_cells.size();
+        total_ctr += interface_size;
 
         if (isA<processorFvPatch>(iface->interface())) {
             const auto &patch =
@@ -386,7 +389,7 @@ std::vector<interface_locality> HostMatrixWrapper::collect_cells_on_interfaces(
 
     word msg = "done collecting neighbouring processor cell id";
     LOG_2(verbose_, msg)
-    return interface_idxs;
+    return {total_ctr, interface_idxs};
 }
 
 std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
@@ -394,8 +397,7 @@ std::tuple<std::vector<label>, std::vector<label>, std::vector<label>,
 HostMatrixWrapper::compute_interface_sparsity(
     std::shared_ptr<const gko::Executor> exec) const
 {
-    auto interface_loc_vec = collect_cells_on_interfaces(interfaces_);
-    label total_interface_nnz = interface_loc_vec.back().non_local_nnz_ctr + 1;
+    auto [total_interface_nnz, interface_loc_vec] = collect_cells_on_interfaces(interfaces_);
     std::vector<label> rows_vec(total_interface_nnz);
     std::vector<label> cols_vec(total_interface_nnz);
     std::vector<label> mapping_vec(total_interface_nnz);
@@ -445,8 +447,9 @@ HostMatrixWrapper::compute_sparsity_patterns(
     auto [non_local_rows, non_local_cols, non_local_mapping, non_local_ranks,
           non_local_spans] = compute_interface_sparsity(exec);
 
+    std::cout << __FILE__ << " rank " << rank << " non_local_rows before " << non_local_rows << "\n";
     // move all local interfaces to local rows and cols
-    std::vector<size_t> erase_non_local{};
+    // std::vector<size_t> erase_non_local{};
     std::vector<size_t> keep_non_local{};
     for (int interface_ctr = 0; interface_ctr < non_local_spans.size();
          interface_ctr++) {
@@ -461,7 +464,7 @@ HostMatrixWrapper::compute_sparsity_patterns(
                                  non_local_mapping.data() + begin,
                                  non_local_mapping.data() + end);
             local_spans.emplace_back(start, local_rows.size());
-            erase_non_local.push_back(interface_ctr);
+            // erase_non_local.push_back(interface_ctr);
         } else {
             keep_non_local.push_back(interface_ctr);
         }
@@ -500,6 +503,7 @@ HostMatrixWrapper::compute_sparsity_patterns(
 
         begin += length;
     }
+    std::cout << __FILE__ << " rank " << rank << " non_local_rows_copy " << non_local_rows_copy << "\n";
 
     auto local_sparsity = std::make_shared<SparsityPattern>(
         exec->get_master(), get_size(), local_rows, local_cols, local_mapping,
diff --git a/unitTests/MatrixWrapper/HostMatrix_p2d.cpp b/unitTests/MatrixWrapper/HostMatrix_p2d.cpp
index 6a214022..5fe4c5fe 100644
--- a/unitTests/MatrixWrapper/HostMatrix_p2d.cpp
+++ b/unitTests/MatrixWrapper/HostMatrix_p2d.cpp
@@ -261,10 +261,11 @@ TEST(HostMatrix, canGenerateNonLocalSparsityPattern)
         hostMatrix->compute_sparsity_patterns(exec->get_device_exec());
 
     // corresponds to cell ids
-    std::vector<std::vector<label>> rows_expected({{2, 5, 8, 6, 7, 8},
-                                                   {0, 3, 6, 6, 7, 8},
-                                                   {0, 1, 2, 2, 5, 8},
-                                                   {0, 1, 2, 0, 3, 6}});
+    std::vector<std::vector<label>> rows_expected{
+        {0, 1, 2, 3, 4, 5, 6, 7},
+        {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7},
+        {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7},
+        {0, 1, 2, 3, 4, 5, 6, 7}};
     // cols expected
     std::vector<std::vector<label>> cols_expected({{0, 3, 6, 0, 1, 2},
                                                    {2, 5, 8, 0, 1, 2},
@@ -277,19 +278,20 @@ TEST(HostMatrix, canGenerateNonLocalSparsityPattern)
     // as they are in compressed format
     std::vector<label> exp_send_idx_size{8, 16, 16, 8};
     EXPECT_EQ(nonLocalSparsity->num_nnz, exp_send_idx_size[rank]);
+    // number of interfaces
     std::vector<label> exp_spans_size{1, 2, 2, 1};
     EXPECT_EQ(nonLocalSparsity->spans.size(), exp_spans_size[rank]);
 
     // EXPECT_EQ(nonLocalSparsity->spans[0].begin, 0);
-    // EXPECT_EQ(nonLocalSparsity->spans[0].end, 3);
     // EXPECT_EQ(nonLocalSparsity->spans[1].begin, 3);
+    // EXPECT_EQ(nonLocalSparsity->spans[0].end, 3);
     // EXPECT_EQ(nonLocalSparsity->spans[1].end, 6);
 
     // auto res_size{nonLocalSparsity->row_idxs.get_size()};
-    // auto rows_res = convert_to_vector(nonLocalSparsity->row_idxs);
+    auto rows_res = convert_to_vector(nonLocalSparsity->row_idxs);
+    EXPECT_EQ(rows_expected[rank], rows_res);
     // auto cols_res = convert_to_vector(nonLocalSparsity->col_idxs);
     // auto mapping_res = convert_to_vector(nonLocalSparsity->ldu_mapping);
-    // EXPECT_EQ(rows_expected[comm->rank()], rows_res);
     // EXPECT_EQ(cols_expected[comm->rank()], cols_res);
     // EXPECT_EQ(mapping_expected, mapping_res);
 }