From 4d3899756ebd0e135cbde523453e741132dd9919 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 7 Nov 2024 16:56:04 +0100
Subject: [PATCH] fix mapping in the interface case

---
 include/OGL/MatrixWrapper/SparsityPattern.H |  4 +++
 src/MatrixWrapper/HostMatrix.C              | 27 +++++++++++++--------
 unitTests/MatrixWrapper/HostMatrix_p2d.cpp  | 12 ++++++---
 3 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/include/OGL/MatrixWrapper/SparsityPattern.H b/include/OGL/MatrixWrapper/SparsityPattern.H
index 682b252d..8d9b3dad 100644
--- a/include/OGL/MatrixWrapper/SparsityPattern.H
+++ b/include/OGL/MatrixWrapper/SparsityPattern.H
@@ -113,6 +113,10 @@ struct SparsityPattern {
 
     mutable gko::array<label> col_idxs;
 
+    // ldu_mapping is used to reorder copied matrix coefficients
+    // to match the row major ordering of the sparsity pattern
+    // in both cases (local and non local sparsity) the full list of interfaces
+    // is reordered at once. thus all indices from [0, end) should be present.
     // ldu_mapping[sorted_(csr)_position] =
     // unsorted_(consecutive_ldu_)_position
     mutable gko::array<label> ldu_mapping;
diff --git a/src/MatrixWrapper/HostMatrix.C b/src/MatrixWrapper/HostMatrix.C
index c45067d5..fece1754 100644
--- a/src/MatrixWrapper/HostMatrix.C
+++ b/src/MatrixWrapper/HostMatrix.C
@@ -447,7 +447,6 @@ HostMatrixWrapper::compute_sparsity_patterns(
     auto [non_local_rows, non_local_cols, non_local_mapping, non_local_ranks,
           non_local_spans] = compute_interface_sparsity(exec);
 
-    std::cout << __FILE__ << " rank " << rank << " non_local_rows before " << non_local_rows << "\n";
     // move all local interfaces to local rows and cols
     // std::vector<size_t> erase_non_local{};
     std::vector<size_t> keep_non_local{};
@@ -455,16 +454,20 @@ HostMatrixWrapper::compute_sparsity_patterns(
          interface_ctr++) {
         if (non_local_ranks[interface_ctr] == rank) {
             auto [begin, end] = non_local_spans[interface_ctr];
+            label interface_offs = local_cols.size();
             size_t start = local_rows.size();
             local_rows.insert(local_rows.end(), non_local_rows.data() + begin,
                               non_local_rows.data() + end);
             local_cols.insert(local_cols.end(), non_local_cols.data() + begin,
                               non_local_cols.data() + end);
-            local_mapping.insert(local_mapping.end(),
-                                 non_local_mapping.data() + begin,
-                                 non_local_mapping.data() + end);
+            // TODO FIXME
+            for (size_t map_el = begin; map_el < end; map_el++) {
+                local_mapping.push_back(map_el + interface_offs);
+            }
+            // local_mapping.insert(local_mapping.end(),
+            //                      non_local_mapping.data() + begin,
+            //                      non_local_mapping.data() + end);
             local_spans.emplace_back(start, local_rows.size());
-            // erase_non_local.push_back(interface_ctr);
         } else {
             keep_non_local.push_back(interface_ctr);
         }
@@ -485,6 +488,7 @@ HostMatrixWrapper::compute_sparsity_patterns(
         non_local_mapping_copy;
     std::vector<gko::span> non_local_spans_copy{};
     size_t begin = 0;
+    label interface_offset = 0;
     for (auto keep : keep_non_local) {
         auto span = non_local_spans[keep];
         size_t length{span.end - span.begin};
@@ -492,18 +496,21 @@ HostMatrixWrapper::compute_sparsity_patterns(
         non_local_rows_copy.insert(non_local_rows_copy.end(),
                                    non_local_rows.data() + span.begin,
                                    non_local_rows.data() + span.end);
-
         non_local_cols_copy.insert(non_local_cols_copy.end(),
                                    non_local_cols.data() + span.begin,
                                    non_local_cols.data() + span.end);
 
-        non_local_mapping_copy.insert(non_local_mapping_copy.end(),
-                                      non_local_mapping.data() + span.begin,
-                                      non_local_mapping.data() + span.end);
+        // mapping will be made consecutive in separate step
+        for (size_t j = 0; j < span.end - span.begin; j++) {
+            non_local_mapping_copy.push_back(j + interface_offset);
+        }
+        interface_offset += span.end - span.begin;
+        // non_local_mapping_copy.insert(non_local_mapping_copy.end(),
+        //                               non_local_mapping.data() + span.begin,
+        //                               non_local_mapping.data() + span.end);
 
         begin += length;
     }
-    std::cout << __FILE__ << " rank " << rank << " non_local_rows_copy " << non_local_rows_copy << "\n";
 
     auto local_sparsity = std::make_shared<SparsityPattern>(
         exec->get_master(), get_size(), local_rows, local_cols, local_mapping,
diff --git a/unitTests/MatrixWrapper/HostMatrix_p2d.cpp b/unitTests/MatrixWrapper/HostMatrix_p2d.cpp
index 5fe4c5fe..77b3bf74 100644
--- a/unitTests/MatrixWrapper/HostMatrix_p2d.cpp
+++ b/unitTests/MatrixWrapper/HostMatrix_p2d.cpp
@@ -272,7 +272,11 @@ TEST(HostMatrix, canGenerateNonLocalSparsityPattern)
                                                    {6, 7, 8, 0, 3, 6},
                                                    {6, 7, 8, 2, 5, 8}});
 
-    std::vector<label> mapping_expected({0, 1, 2, 3, 4, 5});
+    std::vector<std::vector<label>> mapping_expected{
+        {0, 1, 2, 3, 4, 5, 6, 7},
+        {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
+        {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
+        {0, 1, 2, 3, 4, 5, 6, 7}};
 
     // we dont test the cols expected for now,
     // as they are in compressed format
@@ -287,13 +291,13 @@ TEST(HostMatrix, canGenerateNonLocalSparsityPattern)
     // EXPECT_EQ(nonLocalSparsity->spans[0].end, 3);
     // EXPECT_EQ(nonLocalSparsity->spans[1].end, 6);
 
-    // auto res_size{nonLocalSparsity->row_idxs.get_size()};
     auto rows_res = convert_to_vector(nonLocalSparsity->row_idxs);
     EXPECT_EQ(rows_expected[rank], rows_res);
+    auto mapping_res = convert_to_vector(nonLocalSparsity->ldu_mapping);
+    EXPECT_EQ(mapping_expected[rank], mapping_res);
+
     // auto cols_res = convert_to_vector(nonLocalSparsity->col_idxs);
-    // auto mapping_res = convert_to_vector(nonLocalSparsity->ldu_mapping);
     // EXPECT_EQ(cols_expected[comm->rank()], cols_res);
-    // EXPECT_EQ(mapping_expected, mapping_res);
 }