From fd2e5d81ac1d0d259118d856c70d1b939c959a55 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 26 Oct 2023 14:06:03 +0200
Subject: [PATCH 001/125] base on sparse communicator branch

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7635b335..48033155e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT OGL_ALLOW_REFERENCE_ONLY)
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "4292ffdee"
+    "sparse-communicator"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)

From c9d2c0342345cf7bc0e82448b21b28d22e865cac Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 27 Oct 2023 13:41:53 +0200
Subject: [PATCH 002/125] adapt to new Ginkgo version

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 227 +++++++++---------
 DevicePersistent/Vector/Vector.H              |  70 +++---
 Preconditioner/Preconditioner.H               |   4 +-
 Solver/BiCGStab/GKOBiCGStab.H                 |   4 +-
 Solver/CG/GKOCG.H                             |   7 +-
 Solver/GMRES/GKOGMRES.H                       |   6 +-
 common/.common.H.swo                          | Bin 0 -> 24576 bytes
 common/common.C                               |  14 +-
 lduLduBase/lduLduBase.H                       |   4 +-
 9 files changed, 170 insertions(+), 166 deletions(-)
 create mode 100644 common/.common.H.swo

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index e575ac058..2eafe3661 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -185,14 +185,14 @@ struct MatrixInitFunctor {
 
 
             auto comm = exec_.get_gko_mpi_host_comm();
-            TIME_WITH_FIELDNAME(
-                verbose_, update_build_repartitioner, field_name_,
-                auto repartitioner = gko::share(
-                    gko::experimental::distributed::repartitioner<
-                        label, label>::create(*comm.get(),
-                                              partition_.get_host_partition(),
-                                              partition_.get_device_partition(),
-                                              false));)
+            // TIME_WITH_FIELDNAME(
+            //     verbose_, update_build_repartitioner, field_name_,
+            //     auto repartitioner = gko::share(
+            //         gko::experimental::distributed::repartitioner<
+            //             label, label>::create(*comm.get(),
+            //                                   partition_.get_host_partition(),
+            //                                   partition_.get_device_partition(),
+            //                                   false));)
 
             auto local_nnz =
                 local_scatter_map_pers.get_array()->get_num_elems();
@@ -223,16 +223,16 @@ struct MatrixInitFunctor {
                     recv_offs_pers.get_array()->get_num_elems());
 
 
-            TIME_WITH_FIELDNAME(
-                verbose_, update_repartitioned_existing, field_name_,
-                repartitioner->update_existing(
-                    *row_idxs_.get_array(), *non_local_row_idxs_.get_array(),
-                    *coeffs.get(), *non_local_coeffs.get(),
-                    *sorting_idx_pers.get_array().get(), send_sizes,
-                    send_offsets, recv_sizes, recv_offsets,
-                    *local_scatter_map_pers.get_array().get(),
-                    *non_local_scatter_map_pers.get_array().get(), local_values,
-                    non_local_values);)
+            // TIME_WITH_FIELDNAME(
+            //     verbose_, update_repartitioned_existing, field_name_,
+            //     repartitioner->update_existing(
+            //         *row_idxs_.get_array(), *non_local_row_idxs_.get_array(),
+            //         *coeffs.get(), *non_local_coeffs.get(),
+            //         *sorting_idx_pers.get_array().get(), send_sizes,
+            //         send_offsets, recv_sizes, recv_offsets,
+            //         *local_scatter_map_pers.get_array().get(),
+            //         *non_local_scatter_map_pers.get_array().get(), local_values,
+            //         non_local_values);)
 
             auto value_view =
                 val_array::view(device_exec, local_nnz, value_ptr);
@@ -359,8 +359,9 @@ struct MatrixInitFunctor {
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
             exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm());
-        dist_A->read_distributed(A_data, non_local_A_data,
-                                 partition_.get_host_partition().get());
+        // TODO FIXME implement me
+        // dist_A->read_distributed(A_data, non_local_A_data,
+        //                          partition_.get_host_partition().get());
 
         if (partition_.get_ranks_per_gpu() == 1) {
             auto device_mat = generate_dist_mtx_with_inner_type(
@@ -370,99 +371,99 @@ struct MatrixInitFunctor {
         }
 
         // TODO test if this needs to be persistent
-        auto repartitioner = gko::share(
-            gko::experimental::distributed::repartitioner<label, label>::create(
-                *comm.get(), partition_.get_host_partition(),
-                partition_.get_device_partition(), false));
-        auto to_mat = gko::share(
-            dist_mtx::create(exec, repartitioner->get_to_communicator()));
-
-        auto [local_scatter_map, non_local_scatter_map, sorting_idx, send_sizes,
-              send_offsets, recv_sizes, recv_offsets] =
-            repartitioner->gather(dist_A.get(), to_mat.get());
-
-        // TODO add field name
-        PersistentArray<label> local_scatter_map_pers{
-            local_scatter_map.get_data(),
-            "local_scatter_map",
-            db_,
-            exec_,
-            static_cast<label>(local_scatter_map.get_num_elems()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> non_local_scatter_map_pers{
-            non_local_scatter_map.get_data(),
-            "non_local_scatter_map",
-            db_,
-            exec_,
-            static_cast<label>(non_local_scatter_map.get_num_elems()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> sorting_idx_pers{
-            sorting_idx.get_data(),
-            "sorting_idx",
-            db_,
-            exec_,
-            static_cast<label>(sorting_idx.get_num_elems()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> send_sizes_pers{
-            send_sizes.data(),
-            "send_sizes",
-            db_,
-            exec_,
-            static_cast<label>(send_sizes.size()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> send_offs_pers{
-            send_offsets.data(),
-            "send_offs",
-            db_,
-            exec_,
-            static_cast<label>(send_offsets.size()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> recv_sizes_pers{
-            recv_sizes.data(),
-            "recv_sizes",
-            db_,
-            exec_,
-            static_cast<label>(recv_sizes.size()),
-            verbose_,
-            false,
-            false};
-
-        PersistentArray<label> recv_offs_pers{
-            recv_offsets.data(),
-            "recv_offs",
-            db_,
-            exec_,
-            static_cast<label>(recv_offsets.size()),
-            verbose_,
-            false,
-            false};
-
-        auto device_mat = generate_dist_mtx_with_inner_type(
-            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
-
-        to_mat->move_to(device_mat.get());
-
-        return device_mat;
+        // auto repartitioner = gko::share(
+        //     gko::experimental::distributed::repartitioner<label, label>::create(
+        //         *comm.get(), partition_.get_host_partition(),
+        //         partition_.get_device_partition(), false));
+        // auto to_mat = gko::share(
+        //     dist_mtx::create(exec, repartitioner->get_to_communicator()));
+        //
+        // auto [local_scatter_map, non_local_scatter_map, sorting_idx, send_sizes,
+        //       send_offsets, recv_sizes, recv_offsets] =
+        //     repartitioner->gather(dist_A.get(), to_mat.get());
+        //
+        // // TODO add field name
+        // PersistentArray<label> local_scatter_map_pers{
+        //     local_scatter_map.get_data(),
+        //     "local_scatter_map",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(local_scatter_map.get_num_elems()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> non_local_scatter_map_pers{
+        //     non_local_scatter_map.get_data(),
+        //     "non_local_scatter_map",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(non_local_scatter_map.get_num_elems()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> sorting_idx_pers{
+        //     sorting_idx.get_data(),
+        //     "sorting_idx",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(sorting_idx.get_num_elems()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> send_sizes_pers{
+        //     send_sizes.data(),
+        //     "send_sizes",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(send_sizes.size()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> send_offs_pers{
+        //     send_offsets.data(),
+        //     "send_offs",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(send_offsets.size()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> recv_sizes_pers{
+        //     recv_sizes.data(),
+        //     "recv_sizes",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(recv_sizes.size()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // PersistentArray<label> recv_offs_pers{
+        //     recv_offsets.data(),
+        //     "recv_offs",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(recv_offsets.size()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // auto device_mat = generate_dist_mtx_with_inner_type(
+        //     exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
+        //
+        // to_mat->move_to(device_mat.get());
+        //
+        // return device_mat;
     }
 };
 
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index f84b09263..a6e38986b 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -102,33 +102,33 @@ struct VectorInitFunctor {
             return gko::share(dist_vec::create(
                 exec, *exec_.get_gko_mpi_device_comm().get(),
                 vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)
-                    .get()));
+                    ));
         }
 
         // when having more ranks than GPUs we must first repartition and
         // gather the vector
-        auto host_vector = gko::share(dist_vec::create(
-            exec, *exec_.get_gko_mpi_host_comm().get(),
-            vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
-
-        auto target_vector = gko::share(dist_vec::create(
-            exec, *exec_.get_gko_mpi_host_comm().get(),
-            vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
-
-        // TODO test if this needs to be persistent
-        auto repartitioner = gko::share(
-            gko::experimental::distributed::repartitioner<label, label>::create(
-                *comm.get(), partition_.get_host_partition(),
-                partition_.get_device_partition(), true));
-
-        repartitioner->gather(host_vector.get(), target_vector.get());
-
-        auto device_vec = gko::share(
-            dist_vec::create(exec, *exec_.get_gko_mpi_device_comm().get()));
-
-        target_vector->move_to(device_vec.get());
-
-        return device_vec;
+        // auto host_vector = gko::share(dist_vec::create(
+        //     exec, *exec_.get_gko_mpi_host_comm().get(),
+        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
+        //
+        // auto target_vector = gko::share(dist_vec::create(
+        //     exec, *exec_.get_gko_mpi_host_comm().get(),
+        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
+        //
+        // // TODO test if this needs to be persistent
+        // auto repartitioner = gko::share(
+        //     gko::experimental::distributed::repartitioner<label, label>::create(
+        //         *comm.get(), partition_.get_host_partition(),
+        //         partition_.get_device_partition(), true));
+        //
+        // repartitioner->gather(host_vector.get(), target_vector.get());
+        //
+        // auto device_vec = gko::share(
+        //     dist_vec::create(exec, *exec_.get_gko_mpi_device_comm().get()));
+        //
+        // target_vector->move_to(device_vec.get());
+        //
+        // return device_vec;
     }
 };
 
@@ -223,18 +223,18 @@ public:
             gko::dim<2>{local_host_size, 1}, 1);
         scalar *host_buffer = host_device_vector->get_local_values();
 
-        if (partition_.get_ranks_per_gpu() != 1) {
-            auto repartitioner = gko::share(
-                gko::experimental::distributed::repartitioner<label, label>::
-                    create(*comm.get(),
-                           partition_.get_host_partition(),    // from
-                           partition_.get_device_partition(),  // to
-                           true));
-
-            repartitioner->scatter(host_device_vector.get(),
-                                   target_vector.get());
-            host_buffer = target_vector->get_local_values();
-        }
+        // if (partition_.get_ranks_per_gpu() != 1) {
+        //     auto repartitioner = gko::share(
+        //         gko::experimental::distributed::repartitioner<label, label>::
+        //             create(*comm.get(),
+        //                    partition_.get_host_partition(),    // from
+        //                    partition_.get_device_partition(),  // to
+        //                    true));
+        //
+        //     repartitioner->scatter(host_device_vector.get(),
+        //                            target_vector.get());
+        //     host_buffer = target_vector->get_local_values();
+        // }
 
         auto host_buffer_view = gko::array<T>::view(
             exec_.get_ref_exec(), local_host_size, host_buffer);
diff --git a/Preconditioner/Preconditioner.H b/Preconditioner/Preconditioner.H
index 7bcf5bbb3..2f883790e 100644
--- a/Preconditioner/Preconditioner.H
+++ b/Preconditioner/Preconditioner.H
@@ -76,7 +76,7 @@ public:
         if (Pstream::parRun()) {
             return gko::share(
                 ras::build()
-                    .with_generated_inner_solvers(gko::share(precond->generate(
+                    .with_generated_local_solver(gko::share(precond->generate(
                         gko::as<gko::experimental::distributed::Matrix<
                             scalar, label, label>>(gkomatrix)
                             ->get_local_matrix())))
@@ -95,7 +95,7 @@ public:
     {
         if (Pstream::parRun()) {
             return gko::share(ras::build()
-                                  .with_generated_inner_solvers(
+                                  .with_generated_local_solver(
                                       precond->generate(factorization))
                                   .on(device_exec)
                                   ->generate(gkomatrix));
diff --git a/Solver/BiCGStab/GKOBiCGStab.H b/Solver/BiCGStab/GKOBiCGStab.H
index 797cb8958..2cabd288c 100644
--- a/Solver/BiCGStab/GKOBiCGStab.H
+++ b/Solver/BiCGStab/GKOBiCGStab.H
@@ -73,7 +73,7 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto bicgstab = gko::solver::Bicgstab<scalar>::build()
-                            .with_criteria(stoppingCriterionVec_)
+                            // .with_criteria(stoppingCriterionVec_)
                             .on(exec);
         return bicgstab;
     };
@@ -83,7 +83,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto bicgstab = bicgstab::build()
-                            .with_criteria(stoppingCriterionVec_)
+                            // .with_criteria(stoppingCriterionVec_)
                             .with_generated_preconditioner(precond)
                             .on(exec);
         return bicgstab;
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 427a58ca9..66e1636ed 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -68,7 +68,9 @@ public:
     std::unique_ptr<cg::Factory, std::default_delete<cg::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
+        // TODO FIXME
+        // auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
+        auto cg = cg::build().on(exec);
         return cg;
     };
 
@@ -77,7 +79,8 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto cg = cg::build()
-                      .with_criteria(stoppingCriterionVec_)
+                      // TODO FIXME
+                      // .with_criteria(stoppingCriterionVec_)
                       .with_generated_preconditioner(precond)
                       .on(exec);
         return cg;
diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 46382d9f6..28efe1b9f 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -70,7 +70,9 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto gmres =
-            gmres::build().with_criteria(stoppingCriterionVec_).on(exec);
+            gmres::build()
+            // .with_criteria(stoppingCriterionVec_)
+            .on(exec);
         return gmres;
     };
 
@@ -79,7 +81,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto gmres = gmres::build()
-                         .with_criteria(stoppingCriterionVec_)
+                         // .with_criteria(stoppingCriterionVec_)
                          .with_generated_preconditioner(precond)
                          .on(exec);
         return gmres;
diff --git a/common/.common.H.swo b/common/.common.H.swo
new file mode 100644
index 0000000000000000000000000000000000000000..2ecc4277bb64ad17652e80614be1f89770ee053b
GIT binary patch
literal 24576
zcmeI3dyFK<S-_jvgludlcI<@MASmC#dUkJj_I&<>WcESU>wV$d+sp1=oa62odZuf3
z`mU#Y=<eCQ<#O1LP#gybJ3-izk%=Y#ASOzb7zIKi2NMz#L}E)2B$Nj)1)K=tU>k*?
zL^i*w>YnMDo!#5Bg9K5J_BW5}s;{cPdiD49x0Qo$TU-)vDju|WU1?cs&%fI)y*YgC
zL*E^?)<YS1QeD{n<$Lz%db4$?=-Pgxc*c#3(rNW6@@q!mm}AHGST%6u*z){gzBC$v
zpL`S-`jpF)at-7fc$pe##kJ8lU2*KDLpSQBzJK5K;_BBOdznJ>;<*NL4dfcgHIQo{
z*Fdg;Tm!iVat-`{(m>q2)ViGpzanXLA^H4~p3h5@-{Z;WaL@HilHZ4t&q~krA5DI*
zCZG59Tu*-HAGro{4dfcgHIQo{*Fdg;Tm!iVat-7f$Tg5_AlJbEf(C5IvR)jptWUif
z;XMCO?f;Ly*0Meb?}58v4lMZOYb@*i@IIJ@?_FV8AA{e42Vo6f16RQ1@QuqY>uLB3
zJPBWf^H7IXSb?91LHJpC_A=T6x5Fe{16RX$Xy4QD6sWfTAv_BAfCE(+hG*%^C*V=|
z7*yf)@E5PLtSvYIU;8o3x)-M5`Bz%jx8UF5-{2{D7#@HI+yXN&3`1}kyc(Xxq46K^
zIQ%KR7w&;ygyZm5xC*|A<K%8wf}e!H!%^}cu;FLmc^og#!J}{|yaj$7zKjFr5%^&0
zh!O3VKkLaxR4({4((|ISI3Ny*Vc{Q<?s~lvw8E%7bng7P)z05_snD?F&^=r9oYu7C
zgoUAEtJ#!c@;!2YRThSZMzV!XEonK&wbv=MEn`V6X~An-O<5{Mfwv*ULd}()bHr}Q
z5n-ww5$4KF;KyO$MO`)N23G?=iiP7=V>j^aa7zsKl+aCE2%^~3S1<)OgU}I!nfgp}
zr%hroX@QmP+y6yPYBlt}xqd9enq8HPemoP%S}ht8?KHmkD5*1mdmWdyu$zNcQb9&h
zx;c9rnoP@00cIMFHm#pi6XIwbMu%NSd9Yn+)M;>1-jvn(_1lDgr=wNdv%{%OS%mM}
z;JwJ`>GaE)Ud>j;bE_pY%^I!Pu6R;JvBRRM+o5#I%{ZKJowH>-4DGEc5eA!Oenw?H
ztE%sNC{hi)Jr&uoy|fm-wq(1GWMJ%y^hC0oB(-;4$?PcoE}M-ZxzdVUe_gcuB__rC
znV?jfR-K&iR83Q6OEMYTP3z8>jPAJA4BZVomL;p|*-<1q10qVoXt>XQBhFc|Y&1Oz
ziV0J&vjI$TekuK)*YE0E=QS}Q_MvzYo%HvOcarz;?WDD1reZZ{`Eg-r#|+yq>>^G2
zPI>?SBhyQ>VwCA*hjH?AXZii4(NjGatMxGO86T$=+G?jQSM4aC=(^N!t6>mHW`q+>
z6+}mx$rZF>q4aiw7(Q`$dHxhCH0|5R8138Mxvmr=&8>;T+h$j8US6FY66d-yT+|OE
zZBE#&I1m|4O?G!Silyyrm8)J*JyR<A!DeBo-)I|YT2@xk%+xOZ>{;4=N7h_liq*xX
zqlah9Yl};>X|G3w8qpCkxAQ1=E8=ADB;I*27@Vcmc{N5l4{GtolTmyLdOvCPbb}%9
z^n!|Z>H#fImb|Q{i`^W>rP3|ZkK{yq(VKjY_-t>!nJ7$i?RS0ts$R>H>8F@*IbQ5I
z2&b%0S}h1S?a(QEoHW_=hTY8mTzCC5>%o}$DAt=?><HOcET-Y9t8vh5vc=7Wsxnqg
zabaI~#>M3Y8y(AVY;k!t2xBrYI%juD+f~BRP_H~!Elw@6JE2e6`CFIGZr7=@H>ppl
z*|W0BstG%VdQw!GOh-exzPKDzPRnZCEv>dOGk1&0oXmGNhVsnv($eyg@&XcSKIx(>
z@#OI65AHQQX4Zy@Cnk`1m3!f;oO3-Haj)Ns>p@r&^K>8x#j+=T_l%fW58hI5MsB0%
z$n=1)y?~Q!(~aw5xheg*<>{rO5Q{b82Ql{_lCz~A$Ia5%*yiSDaoukfgK&L}Gha>S
zXl$xzsU>?n;5exVEzc2kdqWC_Q@R_{5w@rX%`FksL|m6*{>U*gFBxIm6G!!(gE(v?
zna(haMI#8MaAa(|USx#NEv_AzU0oG(%PV499GzZSTbwy|czQ(~J+^X`&6M`8(#{K1
zpqe>-<hZzPdSzw$$lCF7-D$P2%MI!4(k`c|OQTsyq3y@I*-Nu4GYjOOzIpNR;@WW$
zgqD(&>Pol~x9<@~Mx_-~)HRaoK~puzj)fajkLQVs6s<_sTHc5tr?_o#ZDIM?n(20d
zd|M2k2%2gE3<=xc66#zJ1$ITGmY=Rv^>8kvtE)jRrjh*O&?UZ{93w(9hMf}Prc~*P
zhDXw-Q-W+t78Nw2G~3Q&BkaiBg+z6FSlKcP;2fmyHPxIbX--M?LR4*^@$V=_ser9w
zOpmSO*0xA6{a&jp3Cvg}smUR)+QO14TIsA9`++W*la{jo3)sh(V>>JRKYf1xC2ajC
z;URFK3WwmovH8CXpN0G2UhrTM7T_RU2mg%i|0#GMY{9KC3NK*e{}ns{zYgnAfnS0f
z;YDoyZ-VjxJPhaI9Z-RJ7=fRF>);aj4DEgxZU^N9n1flk0aTy9M&Is&tN8w9KA(U;
zgU8_`@VoFJJOKB@eQ+=QCP*m5BwPho!sGZ4?t*tg1*YJQ@EpE`Pr^IlG_1iCTm#?7
zU+~B98?Xf~NO0g6;0AaDydGY_XYePW{0Zma46K6#r{E+MVH94(pYQ@a0iT0+!ktip
zH^X!I7rqV8z$35$Ef|9${5^h#55XTm1d~vJ8$f`1?dQJ1tuYQljExB&TRv)HhO*A=
zhMx8;{eCZFNSK3SMXsw20bAaw42@gb8W8I;E^Di`+zjPLS)Hm;)|ijBqO!S9D63jW
z%mV3Gm0)L3mW-BTL~3bNwGp@uTlU4(#0IT=5Zl{M8?-=qw#oTk-jLO7gKHgUhY`Sa
zF?@C;5vM&AvD=U|%j(mOOqoi*GTEBZ4TjFHm8-j`;ptRR(iDc#7-9O3aCdtY!^Je|
z9D7&X3faQima0IE?!zu}QX_S{ob*b0d`CHL^KeQS%XPby)S_+6Fq0tT2HRdHb1_qS
zQc(MX`jjy~qE1_r?ArS6lRdGD+s#5%1Lv&)&4Bi`m8)@h-~h=30}@Z3^y5s5>DGzN
zg;U%O>vAXBrB96uN^jQHK~eC6O&KbGK<2uboK(Xl1_w=1Re_l3as&!2E`+eCZO1XW
zgsRn#xm)Vfvi=QI?!Xh&bV_w6k)_h`-FaDYN&^F?r5E3OUG7A?bXV_<PE_wZ*E79u
zC+^yNol7^D!HBE@b=Ore<4oTs4~{EkP8ffyvKjex;DFYj$<(Q3QXE7RCbd0W3}7cU
zGqOp#wlv%QATj1HV06lCe)wAZx@xIKmort<|J};zOvc5VmeciZFQb#`VpjV0Tv}cy
zmm2WxnT)46m1L$5xsr7|I;9Q(^`C%BPQMMY&mNg38J`u7>W-H?5laPnPioj_WZ7(R
z6I@E%U<K`yT9GvwO1~;oOQ%$-hLW4h0`BN`OOj=t@qr;bvw^5)r|n7dnx(I@C7XlE
zaZcyc`**8Jk7c=z`#L?`j0#vCi+(Lo)_OV*&6-u7she)>O53M$DRcgUg)W$(v;9!b
z1ld`__$t(0dnCF0YzC3GK-KEyQj&_>HV|gw;UKA=QTxRJyL!~SZIPyTssR?vwu_`O
zIoK8S;Z0T)r>9?LAIoGQI~jDX*$llKVdh$98!)Tu<!^%hI+^rimoDzIFB&bP=Cp5R
zt<hqITP9zxFr{~2N+%L19h)hXSQyIgKzB;87X`wuDa)uCC<leMV`_N!V*M?3jikA~
zWUH_0hX^M1bgQ9DRv1fVcm%$;rR!AYsupbqUP(74aQ1Q!X!ej~LyMGY+(;T3aeqW%
zGV6_Bza`th=^t$^>qeSB_Bn6*&#26-D<<WO#z(2AO5cgruT0GiqgWT|Iz>(6Qo%Y?
zI*ZJ4sXS-bX9ePKj<?u}(!*389e5VS=@z$;c($;9m9blGg={%-s{A-i#)G3*OG_8{
zMIAnw17Noqq&LK@JIwfbo3<Mo&-{p9f>Tr3HH~h}!t5-aT~D#(z)%ZBO!96XP;~F|
z=EPD9MNIPEFH0iE%&eDi60(eTJUz9d(^Ig$cdI@8g2zJgl%(s#Y4?CSx1Bj0$myBW
zIZ|G7Rc(D<s47)M=D2ZvH#Ub2MjLLEc&psBWA+NR7ISy6?f<W0OMV^OQ`!HV#rpLk
zw*E8ld3YE;05uqe|HQt32EGL!fVaXW@Nw+>b+{Ikjjx{bkHHWW;0Cx1zKY%cH}DAD
z0qR+Q4J0TVe-1`L+4|qdZvP(qC43(4hu?<x!V(O?=djD~hPT5t@C5ewL-5Nm21R%V
zJNz+t2du&y;9uyo?swAX{3F*uu7O+wxdwj7G@y<}^U7nVGO-j{2Vx#OwLK&)kDbb6
zr-;x>TuX_U$~eUnkI$@*#O!w^<2UMeIq?!3+aQmfO1J0qGmo80HPPoYkDbyHV|nb<
z#l}vJTiwrZaTRF$|0?X@E$m@s|EK?-|4-QYe+wUmd*BZ6p$;k*paMg1ExZw43E#m7
z@Em*_z6sxer{UA^3Ai6lgA3QfKjJ6&2T*YW55TX0iW9gR)c*_oIeZvYtUv>f!IkiJ
z`~rUspMg)qWAJ;Rd<g;ga2vc1{uTegm*BJTI6MY_1RsaH;9c-`xD9?9ehR*V4?)EW
z{1&_)ZiNMyhgZN;_!FLlcY}%r_&K-~E`cwg$NS+<xC4}4&q4$0Fa{5zZ}sed8Lor>
z!vFFG_<hJ<xdw6#<QnMFz@EZz_LRxoQ0yV=9<pq|Iq6%dZ({XAF+>+t>Y_61X7?}A
zKef};3tiu?5tCBY1$j#a>`%YsfmKdrBoD0Off4_n(~cU-1FL$Y&yxRk?5HRYtm5gH
zdA@EQNaul7X)GO&NmamA9$3}=?0KidsgwT07g&`%<~E8T4n>8c=x~&FTtoj*OtK{b
zD}+{<EUB7$!aqF1q02f<CJDu)fQq~IMu-qlv292)0WjvhSM0!m5^MrU_&zw8sd#kC
o#Gp(}DuGIp&YT2lsGyJQlbEO^Ziuk3!jR6F#I>mJy9KTM51QLKTmS$7

literal 0
HcmV?d00001

diff --git a/common/common.C b/common/common.C
index 222de9a9f..71c1c38aa 100644
--- a/common/common.C
+++ b/common/common.C
@@ -65,16 +65,14 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
     std::ofstream stream{fn};
     stream << std::setprecision(15);
-    switch (matrixFormat){
-        case "Coo":
-            gko::write(stream, (const gko::matrix::Coo<scalar> *)A.get());
-            break;
-        case "Csr":
+    if (matrixFormat == "Coo"){
+        gko::write(stream, (const gko::matrix::Coo<scalar> *)A.get());
+            }
+    if (matrixFormat == "Csr"){
             gko::write(stream, (const gko::matrix::Csr<scalar> *)A.get());
-            break;
-        case "Ell":
+    }
+    if (matrixFormat == "Ell"){
             gko::write(stream, (const gko::matrix::Ell<scalar> *)A.get());
-            break;
     }
 }
 
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index c2ebd1f39..e91aba5a8 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -285,7 +285,7 @@ public:
         bool debug(solver_controls_.lookupOrDefault<Switch>("debug", false));
         if (debug && db_.time().writeTime()) {
             word matrix_format(
-              solverControls.lookupOrDefault<word>("matrixFormat", "Coo"));
+              solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"));
             LOG_0(verbose_, "Export system")
             export_mtx(
                 this->fieldName(),
@@ -312,7 +312,7 @@ public:
         LOG_1(verbose_, "solve solver done")
         TIME_WITH_FIELDNAME(
             verbose_, solve, this->fieldName(),
-            solver->apply(gko::lend(dist_b_v), gko::lend(dist_x_v));)
+            solver->apply(dist_b_v, dist_x_v);)
 
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)

From 913b79c92544e98819423f07064cd1c260da33ae Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 27 Oct 2023 14:30:23 +0200
Subject: [PATCH 003/125] Use localized_partition

---
 DevicePersistent/Partition/Partition.H | 91 ++++++++++++++------------
 lduLduBase/lduLduBase.H                |  1 -
 2 files changed, 50 insertions(+), 42 deletions(-)

diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index dad7403c0..2d0e20212 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -39,8 +39,8 @@ namespace Foam {
 
 
 struct PartitionInitFunctor {
-    using vec = gko::matrix::Dense<scalar>;
-    using dist_vec = gko::experimental::distributed::Vector<scalar>;
+    using part_type =
+        gko::experimental::distributed::localized_partition<label, label>;
 
     const ExecutorHandler &exec_;
 
@@ -63,47 +63,53 @@ struct PartitionInitFunctor {
           ranks_per_gpu_(ranks_per_gpu)
     {}
 
-    void update(
-        std::shared_ptr<gko::experimental::distributed::Partition<label, label>>
-            persistent_partition) const
+    void update(std::shared_ptr<part_type> persistent_partition) const
     {
         UNUSED(persistent_partition);
     }
 
-    std::shared_ptr<gko::experimental::distributed::Partition<label, label>>
-    init() const
+    std::shared_ptr<part_type> init() const
     {
         auto exec = exec_.get_ref_exec();
         auto comm = exec_.get_gko_mpi_host_comm();
 
+        const label local_size = rows_end_ - rows_start_;
         word msg{"initialising partition of size " +
-                 std::to_string(rows_end_ - rows_start_)};
+                 std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
-        auto host_partition = gko::share(
-            gko::experimental::distributed::build_partition_from_local_range<
-                label, label>(exec, *comm.get(),
-                              gko::span(rows_start_, rows_end_)));
+        std::vector<std::pair<index_set<index_type>, comm_index_type>>
+            send_idxs;
+        std::vector<std::pair<index_set<index_type>, comm_index_type>>
+            recv_idxs;
 
-        if (ranks_per_gpu_ == 1) return host_partition;
-
-        auto local_rank = comm->rank();
-        auto local_owning_rank = ranks_per_gpu_ * (local_rank / ranks_per_gpu_);
-        auto next_owning_rank = ranks_per_gpu_ + local_owning_rank;
+        auto host_partition = gko::experimental::distributed::
+            localized_partition::build_from_blocked_recv(
+                exec, rows_end_ - rows_start_, send_idxs, recv_idxs);
 
-        // The new partition ranges up to next owning rank elements
-        auto target_rows_end =
-            host_partition->get_range_bounds()[next_owning_rank];
-
-        // if we are not on an owning rank start rows are set to be the end
-        // row else the rows start are unchanged
-        auto target_rows_start =
-            (local_rank % ranks_per_gpu_ != 0) ? target_rows_end : rows_start_;
+        if (ranks_per_gpu_ == 1) return host_partition;
 
-        return gko::share(
-            gko::experimental::distributed::build_partition_from_local_range<
-                label, label>(exec, *comm.get(),
-                              gko::span(target_rows_start, target_rows_end)));
+        // TODO FIXME
+        // auto local_rank = comm->rank();
+        // auto local_owning_rank = ranks_per_gpu_ * (local_rank /
+        // ranks_per_gpu_); auto next_owning_rank = ranks_per_gpu_ +
+        // local_owning_rank;
+
+        // // The new partition ranges up to next owning rank elements
+        // auto target_rows_end =
+        //     host_partition->get_range_bounds()[next_owning_rank];
+
+        // // if we are not on an owning rank start rows are set to be the end
+        // // row else the rows start are unchanged
+        // auto target_rows_start =
+        //     (local_rank % ranks_per_gpu_ != 0) ? target_rows_end :
+        //     rows_start_;
+
+        // return gko::share(
+        //     gko::experimental::distributed::build_partition_from_local_range<
+        //         label, label>(exec, *comm.get(),
+        //                       gko::span(target_rows_start,
+        //                       target_rows_end)));
     }
 };
 
@@ -132,6 +138,8 @@ class PersistentPartition
 
 
 public:
+    using part_type =
+        gko::experimental::distributed::localized_partition<label, label>;
     /* PersistentPartition constructor using existing memory
      *
      * @param objectRegistry reference to registry for storage
@@ -144,9 +152,7 @@ public:
     PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
                         const label verbose, const int ranks_per_gpu,
                         const label offset, const label elements)
-        : PersistentBase<
-              gko::experimental::distributed::Partition<label, label>,
-              PartitionInitFunctor>(
+        : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, offset, offset + elements, verbose,
                                    ranks_per_gpu),
@@ -157,21 +163,24 @@ public:
           exec_(exec)
     {}
 
-    std::shared_ptr<gko::experimental::distributed::Partition<label, label>>
-    get_device_partition() const
+    std::shared_ptr<part_type> get_device_partition() const
     {
         return this->get_persistent_object();
     }
 
-    std::shared_ptr<
-        const gko::experimental::distributed::Partition<label, label>>
-    get_host_partition() const
+    std::shared_ptr<const part_type> get_host_partition() const
     {
         auto comm = exec_.get_gko_mpi_host_comm();
-        return gko::share(
-            gko::experimental::distributed::build_partition_from_local_range<
-                label, label>(exec_.get_ref_exec(), *comm.get(),
-                              gko::span(offset_, offset_ + elements_)));
+        const label local_size = rows_end_ - rows_start_;
+        std::vector<std::pair<index_set<index_type>, comm_index_type>>
+            send_idxs;
+        std::vector<std::pair<index_set<index_type>, comm_index_type>>
+            recv_idxs;
+        return gko::experimental::distributed::localized_partition::
+            build_from_blocked_recv(
+                exec, rows_end_ - rows_start_, send_idxs, recv_idxs
+
+            );
     }
 
     // number of elements on this rank on the host
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index e91aba5a8..e522d493a 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -147,7 +147,6 @@ private:
     using dist_mtx =
         gko::experimental::distributed::Matrix<scalar, label, label>;
     using GlobalIndexType = gko::int64;
-    using part_type = gko::experimental::distributed::Partition<label, label>;
     using mtx = gko::matrix::Csr<scalar>;
     using coo_mtx = gko::matrix::Coo<scalar>;
     using csr_mtx = gko::matrix::Csr<scalar>;

From 53b97c0b8c839b4407809ab9f6e7ba8108471ba5 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 27 Oct 2023 14:33:04 +0200
Subject: [PATCH 004/125] Fixup

---
 DevicePersistent/Partition/Partition.H | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 2d0e20212..77018214b 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -40,7 +40,7 @@ namespace Foam {
 
 struct PartitionInitFunctor {
     using part_type =
-        gko::experimental::distributed::localized_partition<label, label>;
+        gko::experimental::distributed::localized_partition;
 
     const ExecutorHandler &exec_;
 
@@ -126,7 +126,7 @@ struct PartitionInitFunctor {
  * */
 class PersistentPartition
     : public PersistentBase<
-          gko::experimental::distributed::Partition<label, label>,
+          gko::experimental::distributed::localized_partition,
           PartitionInitFunctor> {
     const int ranks_per_gpu_;
 
@@ -139,7 +139,7 @@ class PersistentPartition
 
 public:
     using part_type =
-        gko::experimental::distributed::localized_partition<label, label>;
+        gko::experimental::distributed::localized_partition;
     /* PersistentPartition constructor using existing memory
      *
      * @param objectRegistry reference to registry for storage

From 400b861564529ce8e573df0f44c2daba676eca2a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 11:10:00 +0100
Subject: [PATCH 005/125] construct sparse communicator

---
 DevicePersistent/Base/Base.H           |  7 ++-
 DevicePersistent/Partition/Partition.H | 72 +++++++++++++++-----------
 HostMatrix/HostMatrix.C                | 55 +++++++++++++++++++-
 HostMatrix/HostMatrix.H                | 24 +++++++++
 lduLduBase/lduLduBase.H                |  5 +-
 5 files changed, 131 insertions(+), 32 deletions(-)

diff --git a/DevicePersistent/Base/Base.H b/DevicePersistent/Base/Base.H
index e1bb611c6..5f8c8b9c2 100644
--- a/DevicePersistent/Base/Base.H
+++ b/DevicePersistent/Base/Base.H
@@ -86,6 +86,8 @@ private:
 
     const label verbose_;
 
+    const InitFunctor init_functor_;
+
     // This needs to be a pointer so that it can be assigned
     // during the constructor, since regIOobject needs arguments
     // to be constructed
@@ -99,7 +101,8 @@ public:
         : name_(name),
           db_(db),
           from_registry_{db.foundObject<regIOobject>(name_)},
-          verbose_(verbose)
+          verbose_(verbose),
+          init_functor_(f)
     {
         if (from_registry_) {
             word msg = "reading " + name_ + " from registry";
@@ -149,6 +152,8 @@ public:
 
     word get_name() const { return name_; }
 
+    const InitFunctor &get_init_functor() const { return init_functor_; }
+
     const objectRegistry &get_db() const { return db_; }
 
     bool get_stored() const { return from_registry_; }
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 77018214b..1c12bdf28 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -39,8 +39,7 @@ namespace Foam {
 
 
 struct PartitionInitFunctor {
-    using part_type =
-        gko::experimental::distributed::localized_partition;
+    using part_type = gko::experimental::distributed::localized_partition<>;
 
     const ExecutorHandler &exec_;
 
@@ -52,15 +51,25 @@ struct PartitionInitFunctor {
 
     const int ranks_per_gpu_;
 
+    // Array storing the ranks of the neighbouring processors
+    const gko::array<label> target_ids_;
+
+    // Array storing the ranks of the neighbouring processors
+    const gko::array<label> target_sizes_;
+
 
     PartitionInitFunctor(const ExecutorHandler &exec, const label rows_start,
                          const label rows_end, const label verbose,
-                         const int ranks_per_gpu)
+                         const int ranks_per_gpu,
+                         const gko::array<label> &target_ids,
+                         const gko::array<label> target_sizes)
         : exec_(exec),
           rows_end_(rows_end),
           rows_start_(rows_start),
           verbose_(verbose),
-          ranks_per_gpu_(ranks_per_gpu)
+          ranks_per_gpu_(ranks_per_gpu),
+          target_ids_(target_ids),
+          target_sizes_(target_sizes)
     {}
 
     void update(std::shared_ptr<part_type> persistent_partition) const
@@ -78,14 +87,11 @@ struct PartitionInitFunctor {
                  std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
-        std::vector<std::pair<index_set<index_type>, comm_index_type>>
-            send_idxs;
-        std::vector<std::pair<index_set<index_type>, comm_index_type>>
-            recv_idxs;
+        std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
 
         auto host_partition = gko::experimental::distributed::
-            localized_partition::build_from_blocked_recv(
-                exec, rows_end_ - rows_start_, send_idxs, recv_idxs);
+            localized_partition<>::build_from_blocked_recv(
+                exec, local_size, send_idxs, target_ids_, target_sizes_);
 
         if (ranks_per_gpu_ == 1) return host_partition;
 
@@ -126,7 +132,7 @@ struct PartitionInitFunctor {
  * */
 class PersistentPartition
     : public PersistentBase<
-          gko::experimental::distributed::localized_partition,
+          gko::experimental::distributed::localized_partition<>,
           PartitionInitFunctor> {
     const int ranks_per_gpu_;
 
@@ -138,8 +144,7 @@ class PersistentPartition
 
 
 public:
-    using part_type =
-        gko::experimental::distributed::localized_partition;
+    using part_type = gko::experimental::distributed::localized_partition<>;
     /* PersistentPartition constructor using existing memory
      *
      * @param objectRegistry reference to registry for storage
@@ -151,11 +156,13 @@ public:
      */
     PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
                         const label verbose, const int ranks_per_gpu,
-                        const label offset, const label elements)
+                        const label offset, const label elements,
+                         const gko::array<label> &target_ids,
+                         const gko::array<label> target_sizes)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, offset, offset + elements, verbose,
-                                   ranks_per_gpu),
+                                   ranks_per_gpu, target_ids, target_sizes),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           offset_(offset),
@@ -170,17 +177,20 @@ public:
 
     std::shared_ptr<const part_type> get_host_partition() const
     {
-        auto comm = exec_.get_gko_mpi_host_comm();
-        const label local_size = rows_end_ - rows_start_;
-        std::vector<std::pair<index_set<index_type>, comm_index_type>>
-            send_idxs;
-        std::vector<std::pair<index_set<index_type>, comm_index_type>>
-            recv_idxs;
-        return gko::experimental::distributed::localized_partition::
-            build_from_blocked_recv(
-                exec, rows_end_ - rows_start_, send_idxs, recv_idxs
-
-            );
+        // TODO FIXME can this be avoided? Why do we have this twice?
+        // auto comm = exec_.get_gko_mpi_host_comm();
+        // auto exec = exec_.get_ref_exec();
+        // const label local_size = rows_end_ - rows_start_;
+        // std::vector<std::pair<gko::index_set<label>, label>>
+        //     send_idxs;
+        // gko::array<label> target_ids;
+        // gko::array<label> target_sizes;
+        return {};
+        // gko::experimental::distributed::localized_partition<>::
+        //     build_from_blocked_recv(
+        //         exec, rows_end_ - rows_start_, send_idxs, recv_idxs
+        //
+        //     );
     }
 
     // number of elements on this rank on the host
@@ -189,14 +199,18 @@ public:
     // number of elements on this rank on the device
     label get_local_device_size() const
     {
-        auto comm = exec_.get_gko_mpi_device_comm();
-        return this->get_persistent_object()->get_part_size(comm->rank());
+        // TODO FIXME can this be avoided? Why do we have this twice?
+        // auto comm = exec_.get_gko_mpi_device_comm();
+        // return this->get_persistent_object()->get_part_size(comm->rank());
+        return 0;
     }
 
 
     label get_total_size() const
     {
-        return this->get_persistent_object()->get_size();
+        // TODO FIXME can this be avoided? Why do we have this twice?
+        // return this->get_persistent_object()->get_size();
+        return 0;
     }
 
     label get_ranks_per_gpu() const { return ranks_per_gpu_; }
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 731b20936..fc6cb2b14 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -39,7 +39,7 @@ const lduInterfaceField *interface_getter(
 #else
     return interfaces.operator()(i);
 #endif
-};
+}
 
 template <class MatrixType>
 label HostMatrixWrapper<MatrixType>::count_interface_nnz(
@@ -149,6 +149,59 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
     return local_interface_idxs;
 }
 
+template <class MatrixType>
+std::pair<gko::array<label>, gko::array<label>> HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    // temp vector to store neighbour proc number and number of cells to send
+    std::vector<std::pair<label, label>> neighbour_procs{};
+    // TODO FIXME use a lambda to avoid repeating
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+
+        const auto &iface{interface_getter(interfaces, i)};
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        if (isA<processorLduInterface>(iface->interface())) {
+            const processorLduInterface &pldui =
+                refCast<const processorLduInterface>(iface->interface());
+            const label neighbProcNo = pldui.neighbProcNo();
+
+            neighbour_procs.push_back(
+                std::pair<label, label>{neighbProcNo, interface_size});
+        }
+    }
+
+    // reduce vector
+    label n_procs = 0;
+    std::map<label, label> reduce_map{};
+    for (auto [proc, n_faces] : neighbour_procs) {
+        auto search = reduce_map.find(proc);
+        if (search == reduce_map.end()) {
+            n_procs+=1;
+            reduce_map.insert(proc, n_faces);
+        } else {
+            reduce_map[proc] = reduce_map[proc] + n_faces;
+        }
+    }
+
+    // convert to gko::array
+    gko::array<label> target_ids {exec_.get_ref_exec(), n_procs};
+    gko::array<label> target_sizes {exec_.get_ref_exec(), n_procs};
+
+    label iter = 0;
+    for (const auto& [proc, size] : reduce_map) {
+        target_ids.get_data()[iter] = proc;
+        target_sizes.get_data()[iter] = size;
+        iter++;
+    }
+
+    return std::pair(target_ids, target_sizes);
+}
+
 template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 6449d587c..93912b7ca 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -138,6 +138,10 @@ private:
     // non-local indices
     const label non_local_matrix_nnz_;
 
+    // Array storing the ranks of the neighbouring processors
+    const std::pair<gko::array<label>, gko::array<label>>
+        proc_target_id_and_sizes_;
+
     mutable PersistentSparsityPattern non_local_sparsity_;
 
     mutable PersistentArray<scalar> non_local_coeffs_;
@@ -196,6 +200,7 @@ public:
                      // matrix it will be put on the device
           },
           non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
+          proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
           non_local_sparsity_{
               fieldName + "_non_local", db,       exec_,
               non_local_matrix_nnz_,    verbose_,
@@ -269,6 +274,7 @@ public:
                      // matrix it will be put on the device
           },
           non_local_matrix_nnz_(),
+          // proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
           non_local_sparsity_{
               fieldName + "_non_local", db,       exec_,
               non_local_matrix_nnz_,    verbose_,
@@ -323,6 +329,24 @@ public:
     label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
                               bool proc_interfaces) const;
 
+    /* Iterates all interfaces and counts the number of unique neighbour
+     *processors and number of interfaces in total for this processor
+     **
+     ** @param interfaces The list of interfaces for the search
+     */
+    std::pair<gko::array<label>, gko::array<label>> assemble_proc_id_and_sizes(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /* Getter for proc_id and sizes arrays
+     *
+     */
+    const std::pair<gko::array<label>, gko::array<label>> &
+    get_proc_target_id_and_sizes() const
+    {
+        return proc_target_id_and_sizes_;
+    };
+
+
     /* Iterates all interfaces and collects non-local global column indices
     **
     ** Non-local global column indices are computed from the global cell-id
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index e522d493a..bc45a6b86 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -216,13 +216,16 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
+        const auto& [target_ids, target_sizes] = this->get_proc_target_id_and_sizes();
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
             verbose_,
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
             this->get_global_row_index().offset(Pstream::myProcNo()),  // offset
-            psi.size()  // number of elements
+            psi.size(),  // number of elements
+            target_ids,
+            target_sizes
         };
 
         // TODO this is now the local matrix part for dist_A

From ab0d0e29969e041b3226fe33e0166bcbd9bebf83 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 11:15:12 +0100
Subject: [PATCH 006/125] reformat

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H             | 10 ++++++----
 DevicePersistent/Partition/Partition.H              |  4 ++--
 DevicePersistent/Vector/Vector.H                    | 12 +++++++-----
 HostMatrix/HostMatrix.C                             | 11 ++++++-----
 HostMatrix/HostMatrix.H                             |  4 ++--
 lduLduBase/lduLduBase.H                             | 13 ++++++-------
 6 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 2eafe3661..5938d187d 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -231,8 +231,8 @@ struct MatrixInitFunctor {
             //         *sorting_idx_pers.get_array().get(), send_sizes,
             //         send_offsets, recv_sizes, recv_offsets,
             //         *local_scatter_map_pers.get_array().get(),
-            //         *non_local_scatter_map_pers.get_array().get(), local_values,
-            //         non_local_values);)
+            //         *non_local_scatter_map_pers.get_array().get(),
+            //         local_values, non_local_values);)
 
             auto value_view =
                 val_array::view(device_exec, local_nnz, value_ptr);
@@ -372,13 +372,15 @@ struct MatrixInitFunctor {
 
         // TODO test if this needs to be persistent
         // auto repartitioner = gko::share(
-        //     gko::experimental::distributed::repartitioner<label, label>::create(
+        //     gko::experimental::distributed::repartitioner<label,
+        //     label>::create(
         //         *comm.get(), partition_.get_host_partition(),
         //         partition_.get_device_partition(), false));
         // auto to_mat = gko::share(
         //     dist_mtx::create(exec, repartitioner->get_to_communicator()));
         //
-        // auto [local_scatter_map, non_local_scatter_map, sorting_idx, send_sizes,
+        // auto [local_scatter_map, non_local_scatter_map, sorting_idx,
+        // send_sizes,
         //       send_offsets, recv_sizes, recv_offsets] =
         //     repartitioner->gather(dist_A.get(), to_mat.get());
         //
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 1c12bdf28..178e10d09 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -157,8 +157,8 @@ public:
     PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
                         const label verbose, const int ranks_per_gpu,
                         const label offset, const label elements,
-                         const gko::array<label> &target_ids,
-                         const gko::array<label> target_sizes)
+                        const gko::array<label> &target_ids,
+                        const gko::array<label> target_sizes)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, offset, offset + elements, verbose,
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index a6e38986b..33bc807e3 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -101,23 +101,25 @@ struct VectorInitFunctor {
         if (partition_.get_ranks_per_gpu() == 1) {
             return gko::share(dist_vec::create(
                 exec, *exec_.get_gko_mpi_device_comm().get(),
-                vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)
-                    ));
+                vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)));
         }
 
         // when having more ranks than GPUs we must first repartition and
         // gather the vector
         // auto host_vector = gko::share(dist_vec::create(
         //     exec, *exec_.get_gko_mpi_host_comm().get(),
-        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
+        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view,
+        //     1).get()));
         //
         // auto target_vector = gko::share(dist_vec::create(
         //     exec, *exec_.get_gko_mpi_host_comm().get(),
-        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
+        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view,
+        //     1).get()));
         //
         // // TODO test if this needs to be persistent
         // auto repartitioner = gko::share(
-        //     gko::experimental::distributed::repartitioner<label, label>::create(
+        //     gko::experimental::distributed::repartitioner<label,
+        //     label>::create(
         //         *comm.get(), partition_.get_host_partition(),
         //         partition_.get_device_partition(), true));
         //
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index fc6cb2b14..4e272b1b3 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -150,7 +150,8 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
 }
 
 template <class MatrixType>
-std::pair<gko::array<label>, gko::array<label>> HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
+std::pair<gko::array<label>, gko::array<label>>
+HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // temp vector to store neighbour proc number and number of cells to send
@@ -181,7 +182,7 @@ std::pair<gko::array<label>, gko::array<label>> HostMatrixWrapper<MatrixType>::a
     for (auto [proc, n_faces] : neighbour_procs) {
         auto search = reduce_map.find(proc);
         if (search == reduce_map.end()) {
-            n_procs+=1;
+            n_procs += 1;
             reduce_map.insert(proc, n_faces);
         } else {
             reduce_map[proc] = reduce_map[proc] + n_faces;
@@ -189,11 +190,11 @@ std::pair<gko::array<label>, gko::array<label>> HostMatrixWrapper<MatrixType>::a
     }
 
     // convert to gko::array
-    gko::array<label> target_ids {exec_.get_ref_exec(), n_procs};
-    gko::array<label> target_sizes {exec_.get_ref_exec(), n_procs};
+    gko::array<label> target_ids{exec_.get_ref_exec(), n_procs};
+    gko::array<label> target_sizes{exec_.get_ref_exec(), n_procs};
 
     label iter = 0;
-    for (const auto& [proc, size] : reduce_map) {
+    for (const auto &[proc, size] : reduce_map) {
         target_ids.get_data()[iter] = proc;
         target_sizes.get_data()[iter] = size;
         iter++;
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 93912b7ca..8657519a9 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -340,8 +340,8 @@ public:
     /* Getter for proc_id and sizes arrays
      *
      */
-    const std::pair<gko::array<label>, gko::array<label>> &
-    get_proc_target_id_and_sizes() const
+    const std::pair<gko::array<label>, gko::array<label>>
+        &get_proc_target_id_and_sizes() const
     {
         return proc_target_id_and_sizes_;
     };
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index bc45a6b86..0313002ef 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -216,7 +216,8 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
-        const auto& [target_ids, target_sizes] = this->get_proc_target_id_and_sizes();
+        const auto &[target_ids, target_sizes] =
+            this->get_proc_target_id_and_sizes();
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
@@ -225,8 +226,7 @@ public:
             this->get_global_row_index().offset(Pstream::myProcNo()),  // offset
             psi.size(),  // number of elements
             target_ids,
-            target_sizes
-        };
+            target_sizes};
 
         // TODO this is now the local matrix part for dist_A
         MatrixWrapper dist_A{db_,
@@ -287,7 +287,7 @@ public:
         bool debug(solver_controls_.lookupOrDefault<Switch>("debug", false));
         if (debug && db_.time().writeTime()) {
             word matrix_format(
-              solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"));
+                solver_controls_.lookupOrDefault<word>("matrixFormat", "Coo"));
             LOG_0(verbose_, "Export system")
             export_mtx(
                 this->fieldName(),
@@ -312,9 +312,8 @@ public:
         TIME_WITH_FIELDNAME(verbose_, generate_solver, this->fieldName(),
                             auto solver = solver_gen->generate(dist_A_v);)
         LOG_1(verbose_, "solve solver done")
-        TIME_WITH_FIELDNAME(
-            verbose_, solve, this->fieldName(),
-            solver->apply(dist_b_v, dist_x_v);)
+        TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
+                            solver->apply(dist_b_v, dist_x_v);)
 
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)

From 6ebed5e4f79e7156d1a95b2360b783c0c143cb59 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 11:18:04 +0100
Subject: [PATCH 007/125] reformat

---
 Solver/GMRES/GKOGMRES.H |  7 +++----
 common/common.C         | 15 ++++++++-------
 common/common.H         |  3 ++-
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 28efe1b9f..7e6497627 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -69,10 +69,9 @@ public:
     std::unique_ptr<gmres::Factory, std::default_delete<gmres::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        auto gmres =
-            gmres::build()
-            // .with_criteria(stoppingCriterionVec_)
-            .on(exec);
+        auto gmres = gmres::build()
+                         // .with_criteria(stoppingCriterionVec_)
+                         .on(exec);
         return gmres;
     };
 
diff --git a/common/common.C b/common/common.C
index 71c1c38aa..644485c28 100644
--- a/common/common.C
+++ b/common/common.C
@@ -58,21 +58,22 @@ void export_vec(const word fieldName, const gko::matrix::Dense<scalar> *x,
 }
 
 void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
-                const word local, const objectRegistry &db, const word matrixFormat)
+                const word local, const objectRegistry &db,
+                const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
     std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
     std::ofstream stream{fn};
     stream << std::setprecision(15);
-    if (matrixFormat == "Coo"){
+    if (matrixFormat == "Coo") {
         gko::write(stream, (const gko::matrix::Coo<scalar> *)A.get());
-            }
-    if (matrixFormat == "Csr"){
-            gko::write(stream, (const gko::matrix::Csr<scalar> *)A.get());
     }
-    if (matrixFormat == "Ell"){
-            gko::write(stream, (const gko::matrix::Ell<scalar> *)A.get());
+    if (matrixFormat == "Csr") {
+        gko::write(stream, (const gko::matrix::Csr<scalar> *)A.get());
+    }
+    if (matrixFormat == "Ell") {
+        gko::write(stream, (const gko::matrix::Ell<scalar> *)A.get());
     }
 }
 
diff --git a/common/common.H b/common/common.H
index 46edf4ab6..f3b3bf194 100644
--- a/common/common.H
+++ b/common/common.H
@@ -118,7 +118,8 @@ void export_vec(const word fieldName, const gko::matrix::Dense<scalar> *x,
                 const word time);
 
 void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
-                const word local, const objectRegistry &db, const word matrixFormat);
+                const word local, const objectRegistry &db,
+                const word matrixFormat);
 
 void set_solve_prev_iters(const word sys_matrix_name, const objectRegistry &db,
                           label prev_solve_iters, const bool is_final);

From 214cde76d708dd599aa5cdf69687aae3081cdfb2 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 13:04:27 +0100
Subject: [PATCH 008/125] Allow - as separator

---
 .github/workflows/integration-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 85269a1d3..e255dbd75 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -111,7 +111,7 @@ jobs:
     - name: Get Ginkgo checkout version
       shell: bash
       run: |
-        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z]*" > GINKGO_CHECKOUT_VERSION
+        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
         export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
         echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
 

From dfaca2692d47702c09a479cf0581d3ac9c50475d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 13:05:10 +0100
Subject: [PATCH 009/125] Allow - as separator

---
 .github/workflows/build-foam.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index 70994fcb6..f1d7e251a 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -71,7 +71,7 @@ jobs:
     - name: Get Ginkgo checkout version
       shell: bash
       run: |
-        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z]*" > GINKGO_CHECKOUT_VERSION
+        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
         export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
         echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
 

From c33c27bffaae53614aa7c022dcdf0fa4c821938b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 14:27:59 +0100
Subject: [PATCH 010/125] switch gko version to include try-run patch

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 48033155e..9e3d69238 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT OGL_ALLOW_REFERENCE_ONLY)
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "sparse-communicator"
+    "sparse-communicator-ogl"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)

From 6eca357384cbf241fc6a414872a3ca1a918b77c7 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 15:54:16 +0100
Subject: [PATCH 011/125] add a todo fixme notes check

---
 .github/workflows/pr-formatting.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/pr-formatting.yml b/.github/workflows/pr-formatting.yml
index bb3734396..0c834e5c6 100644
--- a/.github/workflows/pr-formatting.yml
+++ b/.github/workflows/pr-formatting.yml
@@ -19,3 +19,8 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
         run: cp --preserve .github/format.sh /tmp && /tmp/format.sh
+      - name: check for todo fixme note
+        run: |
+          NTODOS="$(grep -r 'TODO FIXME' | wc -l)"
+          echo "Found $NTODOS TODO FIXME notes" 
+          ! grep -q -r "TODO FIXME"  

From d82ac3fd131e81bfbe2ddee22b6a2c234aab5d90 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 15:58:01 +0100
Subject: [PATCH 012/125] add a todo fixme notes check

---
 third_party/ginkgo/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/third_party/ginkgo/CMakeLists.txt b/third_party/ginkgo/CMakeLists.txt
index f63d5dda6..56195154f 100644
--- a/third_party/ginkgo/CMakeLists.txt
+++ b/third_party/ginkgo/CMakeLists.txt
@@ -15,4 +15,6 @@ ginkgo_load_git_package(
   "-DGINKGO_BUILD_HWLOC=OFF"
   "-DGINKGO_FORCE_GPU_AWARE_MPI=${GINKGO_FORCE_GPU_AWARE_MPI}"
   "-DCMAKE_INSTALL_PREFIX=$ENV{FOAM_USER_LIBBIN}/.."
-  "-DBUILD_SHARED_LIBS=ON")
+  "-DBUILD_SHARED_LIBS=ON"
+  "-DCMAKE_CXX_STANDARD=17"
+  )

From 27b2bda3eb131131104b1933368f87b636c945db Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 16:06:04 +0100
Subject: [PATCH 013/125] add a todo fixme notes check

---
 .github/workflows/format-check.yml  | 5 +++++
 .github/workflows/pr-formatting.yml | 5 -----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml
index a7d84810b..3d2519756 100644
--- a/.github/workflows/format-check.yml
+++ b/.github/workflows/format-check.yml
@@ -14,3 +14,8 @@ jobs:
         clang-format-version: '13'
         check-path: '.'
         fallback-style: 'Mozilla' # optional
+    - name: check for todo fixme note
+      run: |
+        NTODOS="$(grep -r 'TODO FIXME' | wc -l)"
+        echo "Found $NTODOS TODO FIXME notes" 
+        ! grep -q -r "TODO FIXME"  
diff --git a/.github/workflows/pr-formatting.yml b/.github/workflows/pr-formatting.yml
index 0c834e5c6..bb3734396 100644
--- a/.github/workflows/pr-formatting.yml
+++ b/.github/workflows/pr-formatting.yml
@@ -19,8 +19,3 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
         run: cp --preserve .github/format.sh /tmp && /tmp/format.sh
-      - name: check for todo fixme note
-        run: |
-          NTODOS="$(grep -r 'TODO FIXME' | wc -l)"
-          echo "Found $NTODOS TODO FIXME notes" 
-          ! grep -q -r "TODO FIXME"  

From 110310f46615b5058fb92db67d5265de41d2975e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 1 Nov 2023 17:43:38 +0100
Subject: [PATCH 014/125] add read_distributed with sparse comm

- removed repartitioning for now
- moved sorted validation to python script (not implemented yet)
---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 147 +-----------------
 DevicePersistent/Partition/Partition.H        |  82 +++-------
 DevicePersistent/Vector/Vector.H              |  54 +------
 common/common.H                               |   2 +
 lduLduBase/lduLduBase.H                       |   2 +-
 test/data_validation.py                       |  48 ++++++
 6 files changed, 84 insertions(+), 251 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 5938d187d..49aa7f45d 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -293,8 +293,8 @@ struct MatrixInitFunctor {
 
     std::shared_ptr<dist_mtx> init() const
     {
-        label nCells = partition_.get_local_host_size();
-        word msg{"init global csr matrix of size " + std::to_string(nCells)};
+        const label local_size = partition_.get_local_size();
+        word msg{"init global csr matrix of size " + std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
         auto coeffs = coeffs_.get_array();
@@ -305,45 +305,6 @@ struct MatrixInitFunctor {
         auto non_local_cols = non_local_col_idxs_.get_array();
         auto non_local_rows = non_local_row_idxs_.get_array();
 
-#ifdef DATA_VALIDATION
-        bool debug = true;
-#else
-        bool debug = false;
-#endif
-        // check if sorted
-        if (debug) {
-            bool is_sorted_rows = true;
-            bool is_sorted_cols = true;
-            auto rows_data = rows->get_const_data();
-            auto cols_data = cols->get_const_data();
-            for (size_t i = 1; i < cols->get_num_elems(); i++) {
-                if (rows_data[i] < rows_data[i - 1]) {
-                    is_sorted_rows = false;
-                    Info << "rows sorting error element " << i << " row[i] "
-                         << rows_data[i] << " row[i-1] " << rows_data[i - 1]
-                         << endl;
-                }
-                // same row but subsequent column is smaller
-                if (cols_data[i] < cols_data[i - 1] &&
-                    rows_data[i] == rows_data[i - 1]) {
-                    is_sorted_cols = false;
-                    Info << "cols sorting error element " << i << " row[i] "
-                         << rows_data[i] << " row[i-1] " << rows_data[i - 1]
-                         << " col[i] " << cols_data[i] << " col[i-1] "
-                         << cols_data[i - 1] << endl;
-                }
-            }
-
-            Info << "is_sorted rows " << is_sorted_rows << endl;
-            Info << "is_sorted cols " << is_sorted_cols << endl;
-
-            if (!is_sorted_cols || !is_sorted_rows) {
-                for (size_t i = 1; i < cols->get_num_elems(); i++) {
-                    Info << i << "sparsity (" << rows_data[i] << ","
-                         << cols_data[i] << ")\n";
-                }
-            }
-        }
         auto exec = exec_.get_ref_exec();
 
         auto num_rows = partition_.get_total_size();
@@ -359,9 +320,11 @@ struct MatrixInitFunctor {
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
             exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm());
-        // TODO FIXME implement me
-        // dist_A->read_distributed(A_data, non_local_A_data,
-        //                          partition_.get_host_partition().get());
+        dist_A->read_distributed(A_data, non_local_A_data,
+                gko::experimental::distributed::sparse_communicator::create(
+                    *comm.get(),
+                                 partition_.get_localized_partition()
+                                 ));
 
         if (partition_.get_ranks_per_gpu() == 1) {
             auto device_mat = generate_dist_mtx_with_inner_type(
@@ -370,102 +333,6 @@ struct MatrixInitFunctor {
             return device_mat;
         }
 
-        // TODO test if this needs to be persistent
-        // auto repartitioner = gko::share(
-        //     gko::experimental::distributed::repartitioner<label,
-        //     label>::create(
-        //         *comm.get(), partition_.get_host_partition(),
-        //         partition_.get_device_partition(), false));
-        // auto to_mat = gko::share(
-        //     dist_mtx::create(exec, repartitioner->get_to_communicator()));
-        //
-        // auto [local_scatter_map, non_local_scatter_map, sorting_idx,
-        // send_sizes,
-        //       send_offsets, recv_sizes, recv_offsets] =
-        //     repartitioner->gather(dist_A.get(), to_mat.get());
-        //
-        // // TODO add field name
-        // PersistentArray<label> local_scatter_map_pers{
-        //     local_scatter_map.get_data(),
-        //     "local_scatter_map",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(local_scatter_map.get_num_elems()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> non_local_scatter_map_pers{
-        //     non_local_scatter_map.get_data(),
-        //     "non_local_scatter_map",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(non_local_scatter_map.get_num_elems()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> sorting_idx_pers{
-        //     sorting_idx.get_data(),
-        //     "sorting_idx",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(sorting_idx.get_num_elems()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> send_sizes_pers{
-        //     send_sizes.data(),
-        //     "send_sizes",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(send_sizes.size()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> send_offs_pers{
-        //     send_offsets.data(),
-        //     "send_offs",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(send_offsets.size()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> recv_sizes_pers{
-        //     recv_sizes.data(),
-        //     "recv_sizes",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(recv_sizes.size()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // PersistentArray<label> recv_offs_pers{
-        //     recv_offsets.data(),
-        //     "recv_offs",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(recv_offsets.size()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // auto device_mat = generate_dist_mtx_with_inner_type(
-        //     exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
-        //
-        // to_mat->move_to(device_mat.get());
-        //
-        // return device_mat;
     }
 };
 
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 178e10d09..13743ef50 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -47,6 +47,8 @@ struct PartitionInitFunctor {
 
     const label rows_start_;
 
+    const label local_size_;
+
     const label verbose_;
 
     const int ranks_per_gpu_;
@@ -66,11 +68,13 @@ struct PartitionInitFunctor {
         : exec_(exec),
           rows_end_(rows_end),
           rows_start_(rows_start),
+          local_size_(rows_end_ - rows_start_),
           verbose_(verbose),
           ranks_per_gpu_(ranks_per_gpu),
           target_ids_(target_ids),
           target_sizes_(target_sizes)
-    {}
+    {
+    }
 
     void update(std::shared_ptr<part_type> persistent_partition) const
     {
@@ -82,40 +86,17 @@ struct PartitionInitFunctor {
         auto exec = exec_.get_ref_exec();
         auto comm = exec_.get_gko_mpi_host_comm();
 
-        const label local_size = rows_end_ - rows_start_;
         word msg{"initialising partition of size " +
-                 std::to_string(local_size)};
+                 std::to_string(local_size_)};
         LOG_1(verbose_, msg)
 
         std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
 
         auto host_partition = gko::experimental::distributed::
             localized_partition<>::build_from_blocked_recv(
-                exec, local_size, send_idxs, target_ids_, target_sizes_);
+                exec, local_size_, send_idxs, target_ids_, target_sizes_);
 
         if (ranks_per_gpu_ == 1) return host_partition;
-
-        // TODO FIXME
-        // auto local_rank = comm->rank();
-        // auto local_owning_rank = ranks_per_gpu_ * (local_rank /
-        // ranks_per_gpu_); auto next_owning_rank = ranks_per_gpu_ +
-        // local_owning_rank;
-
-        // // The new partition ranges up to next owning rank elements
-        // auto target_rows_end =
-        //     host_partition->get_range_bounds()[next_owning_rank];
-
-        // // if we are not on an owning rank start rows are set to be the end
-        // // row else the rows start are unchanged
-        // auto target_rows_start =
-        //     (local_rank % ranks_per_gpu_ != 0) ? target_rows_end :
-        //     rows_start_;
-
-        // return gko::share(
-        //     gko::experimental::distributed::build_partition_from_local_range<
-        //         label, label>(exec, *comm.get(),
-        //                       gko::span(target_rows_start,
-        //                       target_rows_end)));
     }
 };
 
@@ -138,7 +119,9 @@ class PersistentPartition
 
     const label offset_;
 
-    const label elements_;
+    const label local_elements_;
+
+    mutable label global_elements_;
 
     const ExecutorHandler &exec_;
 
@@ -166,51 +149,26 @@ public:
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           offset_(offset),
-          elements_(elements),
+          local_elements_(elements),
           exec_(exec)
-    {}
-
-    std::shared_ptr<part_type> get_device_partition() const
     {
-        return this->get_persistent_object();
-    }
+        auto comm = exec.get_gko_mpi_host_comm();
+        label local_elements = local_elements_;
+        comm->all_reduce(exec.get_ref_exec(), &local_elements, 1, MPI_SUM);
+        global_elements_ = local_elements;
+          }
 
-    std::shared_ptr<const part_type> get_host_partition() const
+    std::shared_ptr<part_type> get_localized_partition() const
     {
-        // TODO FIXME can this be avoided? Why do we have this twice?
-        // auto comm = exec_.get_gko_mpi_host_comm();
-        // auto exec = exec_.get_ref_exec();
-        // const label local_size = rows_end_ - rows_start_;
-        // std::vector<std::pair<gko::index_set<label>, label>>
-        //     send_idxs;
-        // gko::array<label> target_ids;
-        // gko::array<label> target_sizes;
-        return {};
-        // gko::experimental::distributed::localized_partition<>::
-        //     build_from_blocked_recv(
-        //         exec, rows_end_ - rows_start_, send_idxs, recv_idxs
-        //
-        //     );
+        return this->get_persistent_object();
     }
 
     // number of elements on this rank on the host
-    label get_local_host_size() const { return elements_; }
-
-    // number of elements on this rank on the device
-    label get_local_device_size() const
-    {
-        // TODO FIXME can this be avoided? Why do we have this twice?
-        // auto comm = exec_.get_gko_mpi_device_comm();
-        // return this->get_persistent_object()->get_part_size(comm->rank());
-        return 0;
-    }
-
+    label get_local_size() const { return local_elements_; }
 
     label get_total_size() const
     {
-        // TODO FIXME can this be avoided? Why do we have this twice?
-        // return this->get_persistent_object()->get_size();
-        return 0;
+        return global_elements_;
     }
 
     label get_ranks_per_gpu() const { return ranks_per_gpu_; }
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 33bc807e3..640c60c40 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -75,7 +75,7 @@ struct VectorInitFunctor {
     void update(std::shared_ptr<gko::experimental::distributed::Vector<T>>
                     persistent_vector) const
     {
-        const auto local_size = partition_.get_local_host_size();
+        const auto local_size = partition_.get_local_size();
         word msg{"updating array " + name_ + " of size " +
                  std::to_string(local_size)};
         LOG_1(verbose_, msg)
@@ -89,7 +89,7 @@ struct VectorInitFunctor {
             (on_device_) ? exec_.get_device_exec() : exec_.get_ref_exec();
         auto comm = exec_.get_gko_mpi_host_comm();
 
-        const auto local_size = partition_.get_local_host_size();
+        const auto local_size = partition_.get_local_size();
         word msg{"initialising vector " + name_ + " of size " +
                  std::to_string(local_size)};
         msg += (on_device_) ? " on device" : " on host";
@@ -103,34 +103,6 @@ struct VectorInitFunctor {
                 exec, *exec_.get_gko_mpi_device_comm().get(),
                 vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)));
         }
-
-        // when having more ranks than GPUs we must first repartition and
-        // gather the vector
-        // auto host_vector = gko::share(dist_vec::create(
-        //     exec, *exec_.get_gko_mpi_host_comm().get(),
-        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view,
-        //     1).get()));
-        //
-        // auto target_vector = gko::share(dist_vec::create(
-        //     exec, *exec_.get_gko_mpi_host_comm().get(),
-        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view,
-        //     1).get()));
-        //
-        // // TODO test if this needs to be persistent
-        // auto repartitioner = gko::share(
-        //     gko::experimental::distributed::repartitioner<label,
-        //     label>::create(
-        //         *comm.get(), partition_.get_host_partition(),
-        //         partition_.get_device_partition(), true));
-        //
-        // repartitioner->gather(host_vector.get(), target_vector.get());
-        //
-        // auto device_vec = gko::share(
-        //     dist_vec::create(exec, *exec_.get_gko_mpi_device_comm().get()));
-        //
-        // target_vector->move_to(device_vec.get());
-        //
-        // return device_vec;
     }
 };
 
@@ -209,8 +181,7 @@ public:
 
     void copy_back()
     {
-        const auto local_host_size = partition_.get_local_host_size();
-        const auto local_device_size = partition_.get_local_device_size();
+        const auto local_size = partition_.get_local_size();
         auto global_size = partition_.get_total_size();
 
         auto comm = exec_.get_gko_mpi_host_comm();
@@ -222,27 +193,14 @@ public:
         // repartitioner size should be sufficient
         auto target_vector = dist_vec::create(
             exec_.get_ref_exec(), *comm.get(), gko::dim<2>{global_size, 1},
-            gko::dim<2>{local_host_size, 1}, 1);
+            gko::dim<2>{local_size, 1}, 1);
         scalar *host_buffer = host_device_vector->get_local_values();
 
-        // if (partition_.get_ranks_per_gpu() != 1) {
-        //     auto repartitioner = gko::share(
-        //         gko::experimental::distributed::repartitioner<label, label>::
-        //             create(*comm.get(),
-        //                    partition_.get_host_partition(),    // from
-        //                    partition_.get_device_partition(),  // to
-        //                    true));
-        //
-        //     repartitioner->scatter(host_device_vector.get(),
-        //                            target_vector.get());
-        //     host_buffer = target_vector->get_local_values();
-        // }
-
         auto host_buffer_view = gko::array<T>::view(
-            exec_.get_ref_exec(), local_host_size, host_buffer);
+            exec_.get_ref_exec(), local_size, host_buffer);
 
         auto to_view = gko::array<T>::view(
-            exec_.get_ref_exec(), local_host_size, const_cast<T *>(memory_));
+            exec_.get_ref_exec(), local_size, const_cast<T *>(memory_));
         to_view = host_buffer_view;
     }
 
diff --git a/common/common.H b/common/common.H
index f3b3bf194..1c5998d94 100644
--- a/common/common.H
+++ b/common/common.H
@@ -106,6 +106,8 @@ namespace Foam {
 
 #define UNUSED(x) (void)(x)
 
+#define OGL_NOT_IMPLEMENTED GKO_NOT_IMPLEMENTED
+
 std::ostream &operator<<(
     std::ostream &os,
     const std::shared_ptr<gko::matrix::Dense<scalar>> array_in);
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 0313002ef..ff1eee075 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -318,7 +318,7 @@ public:
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)
         auto bandwidth_copy_back = sizeof(scalar) *
-                                   partition.get_local_device_size() /
+                                   partition.get_local_size() /
                                    delta_t_copy_x_back / 1000.0;
 
         solverPerf.initialResidual() = this->get_init_res_norm();
diff --git a/test/data_validation.py b/test/data_validation.py
index d3905a45d..fa31ce15a 100644
--- a/test/data_validation.py
+++ b/test/data_validation.py
@@ -106,6 +106,54 @@ def verify_local_matrix(lines, file_path):
                 )
                 sys.exit(1)
 
+def verify_matrix_is_in_row_major_order(lines, file_path):
+    """This function checks if matrix is sorted in row major order
+    """
+    from scipy.io import mmread
+
+    # bool is_sorted_rows = true;
+    # bool is_sorted_cols = true;
+    # auto rows_data = rows->get_const_data();
+    # auto cols_data = cols->get_const_data();
+    # for (size_t i = 1; i < cols->get_num_elems(); i++) {
+    #     if (rows_data[i] < rows_data[i - 1]) {
+    #         is_sorted_rows = false;
+    #         Info << "rows sorting error element " << i << " row[i] "
+    #              << rows_data[i] << " row[i-1] " << rows_data[i - 1]
+    #              << endl;
+    #     }
+    #     // same row but subsequent column is smaller
+    #     if (cols_data[i] < cols_data[i - 1] &&
+    #         rows_data[i] == rows_data[i - 1]) {
+    #         is_sorted_cols = false;
+    #         Info << "cols sorting error element " << i << " row[i] "
+    #              << rows_data[i] << " row[i-1] " << rows_data[i - 1]
+    #              << " col[i] " << cols_data[i] << " col[i-1] "
+    #              << cols_data[i - 1] << endl;
+    #     }
+    # }
+    # bool is_sorted_rows = true;
+    # bool is_sorted_cols = true;
+    # auto rows_data = rows->get_const_data();
+    # auto cols_data = cols->get_const_data();
+    # for (size_t i = 1; i < cols->get_num_elems(); i++) {
+    #     if (rows_data[i] < rows_data[i - 1]) {
+    #         is_sorted_rows = false;
+    #         Info << "rows sorting error element " << i << " row[i] "
+    #              << rows_data[i] << " row[i-1] " << rows_data[i - 1]
+    #              << endl;
+    #     }
+    #     // same row but subsequent column is smaller
+    #     if (cols_data[i] < cols_data[i - 1] &&
+    #         rows_data[i] == rows_data[i - 1]) {
+    #         is_sorted_cols = false;
+    #         Info << "cols sorting error element " << i << " row[i] "
+    #              << rows_data[i] << " row[i-1] " << rows_data[i - 1]
+    #              << " col[i] " << cols_data[i] << " col[i-1] "
+    #              << cols_data[i - 1] << endl;
+    #     }
+    # }
+
 
 def test_matrix_value_bounds(workspace):
     """This function checks if matrices are coeffs are within bounds"""

From 024acdb6a422cb4f0d9d66fcace94d877685a9f5 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 3 Nov 2023 15:42:32 +0100
Subject: [PATCH 015/125] use local indices

---
 DevicePersistent/Base/Base.C                  |  2 +-
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       |  4 +-
 HostMatrix/HostMatrix.C                       | 48 +++----------------
 HostMatrix/HostMatrix.H                       |  1 +
 Solver/CG/GKOCG.H                             | 14 ++++--
 StoppingCriterion/StoppingCriterion.H         |  2 +
 common/common.C                               |  3 +-
 7 files changed, 24 insertions(+), 50 deletions(-)

diff --git a/DevicePersistent/Base/Base.C b/DevicePersistent/Base/Base.C
index 123dfbeb7..487e9ccc8 100644
--- a/DevicePersistent/Base/Base.C
+++ b/DevicePersistent/Base/Base.C
@@ -23,7 +23,7 @@ defineTemplateTypeNameWithName(DevicePersistentBase<gko::LinOp>,
 
 // typedef needed  to avoid confusion with the comma separated template
 // arguments as macro arguments
-typedef gko::experimental::distributed::Partition<label, label> Partition;
+typedef gko::experimental::distributed::localized_partition<label> Partition;
 defineTemplateTypeNameWithName(DevicePersistentBase<Partition>,
                                "PersistentPartition");
 
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 49aa7f45d..383776a7c 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -307,13 +307,13 @@ struct MatrixInitFunctor {
 
         auto exec = exec_.get_ref_exec();
 
-        auto num_rows = partition_.get_total_size();
+        auto num_rows = partition_.get_local_size();
         gko::device_matrix_data<scalar, label> A_data(
             exec, gko::dim<2>(num_rows, num_rows), *rows.get(), *cols.get(),
             *coeffs.get());
 
         gko::device_matrix_data<scalar, label> non_local_A_data(
-            exec, gko::dim<2>(num_rows, num_rows), *non_local_rows.get(),
+            exec, gko::dim<2>(num_rows, non_local_cols->get_num_elems()), *non_local_rows.get(),
             *non_local_cols.get(), *non_local_coeffs.get());
 
 
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 4e272b1b3..2255967c5 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -183,7 +183,7 @@ HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
         auto search = reduce_map.find(proc);
         if (search == reduce_map.end()) {
             n_procs += 1;
-            reduce_map.insert(proc, n_faces);
+            reduce_map.insert(std::pair<label,label>{proc, n_faces});
         } else {
             reduce_map[proc] = reduce_map[proc] + n_faces;
         }
@@ -203,6 +203,11 @@ HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
     return std::pair(target_ids, target_sizes);
 }
 
+template
+std::pair<gko::array<label>, gko::array<label>>
+HostMatrixWrapper<lduMatrix>::assemble_proc_id_and_sizes(
+    const lduInterfaceFieldPtrsList &interfaces) const;
+
 template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
@@ -212,31 +217,6 @@ HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
     std::vector<std::tuple<label, label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
-    label startOfRequests = Pstream::nRequests();
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-
-        if (isA<processorLduInterface>(iface->interface())) {
-            const processorLduInterface &pldui =
-                refCast<const processorLduInterface>(iface->interface());
-            const label neighbProcNo = pldui.neighbProcNo();
-
-            word msg = "send face cells interface " + std::to_string(i) +
-                       " from proc " + std::to_string(Pstream::myProcNo()) +
-                       " to neighbour proc " + std::to_string(neighbProcNo);
-
-            LOG_2(verbose_, msg)
-            pldui.send(Pstream::commsTypes::nonBlocking, face_cells);
-        }
-    }
-    Pstream::waitRequests(startOfRequests);
-    LOG_2(verbose_, "send face cells done")
-
     label interface_ctr = 0;
     for (int i = 0; i < interfaces.size(); i++) {
         if (interface_getter(interfaces, i) == nullptr) {
@@ -248,23 +228,9 @@ HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
         const label interface_size = face_cells.size();
 
         if (isA<processorLduInterface>(iface->interface())) {
-            const processorLduInterface &pldui =
-                refCast<const processorLduInterface>(iface->interface());
-            const label neighbProcNo = pldui.neighbProcNo();
-
-            word msg_2 = "receive face cells interface " + std::to_string(i) +
-                         " from proc " + std::to_string(neighbProcNo);
-            LOG_2(verbose_, msg_2)
-
-            auto otherSide_tmp = pldui.receive<label>(
-                Pstream::commsTypes::nonBlocking, interface_size);
-            LOG_2(verbose_, "receive face cells done")
-
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto global_row = global_row_index_.toGlobal(
-                    neighbProcNo, otherSide_tmp()[cellI]);
                 non_local_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], global_row});
+                    {interface_ctr, face_cells[cellI], interface_ctr});
                 interface_ctr += 1;
             }
         }
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 8657519a9..9fb99043b 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -349,6 +349,7 @@ public:
 
     /* Iterates all interfaces and collects non-local global column indices
     **
+    ** TODO FIXME this is not uptodate
     ** Non-local global column indices are computed from the global cell-id
     ** (row) on the other side of an interface
     **
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 66e1636ed..7ad2ea3b5 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -68,9 +68,10 @@ public:
     std::unique_ptr<cg::Factory, std::default_delete<cg::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        // TODO FIXME
-        // auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
-        auto cg = cg::build().on(exec);
+        auto cg = cg::build().with_criteria(
+              gko::stop::Iteration::build().with_max_iters(20u)
+                // stoppingCriterionVec_
+                ).on(exec);
         return cg;
     };
 
@@ -79,8 +80,11 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto cg = cg::build()
-                      // TODO FIXME
-                      // .with_criteria(stoppingCriterionVec_)
+                      .with_criteria(
+                              gko::stop::Iteration::build().with_max_iters(20u)
+                              //
+                              // stoppingCriterionVec_
+                              )
                       .with_generated_preconditioner(precond)
                       .on(exec);
         return cg;
diff --git a/StoppingCriterion/StoppingCriterion.H b/StoppingCriterion/StoppingCriterion.H
index 64c8a7178..eda67ccbe 100644
--- a/StoppingCriterion/StoppingCriterion.H
+++ b/StoppingCriterion/StoppingCriterion.H
@@ -187,6 +187,8 @@ class StoppingCriterion {
                 *(parameters_.iter) += 1;
                 return false;
             }
+            // TODO FIXME
+            std::cout << "iteration\n ";
 
             const auto exec = this->get_executor();
 
diff --git a/common/common.C b/common/common.C
index 644485c28..ed9846fe6 100644
--- a/common/common.C
+++ b/common/common.C
@@ -62,8 +62,9 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
                 const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
-    std::filesystem::create_directories(folder);
+    //std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
+    std::cout  << "exporting " << fn <<  std::endl;
     std::ofstream stream{fn};
     stream << std::setprecision(15);
     if (matrixFormat == "Coo") {

From a6f341dcab28db679615349f307beccbfeb7ba80 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 3 Nov 2023 15:59:18 +0100
Subject: [PATCH 016/125] fixup

---
 .github/workflows/integration-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index e255dbd75..91edd8499 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -59,7 +59,7 @@ jobs:
     - name: Get Ginkgo checkout version
       shell: bash
       run: |
-        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z]*" > GINKGO_CHECKOUT_VERSION
+        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
         export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
         echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
 

From 7aa7c4cb5563102b6ef53212d87b3528aacbdf7d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 07:47:46 +0100
Subject: [PATCH 017/125] compute send_idxs

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 187 +++++-------------
 DevicePersistent/Partition/Partition.H        |  45 +++--
 HostMatrix/HostMatrix.C                       | 145 +++++++++-----
 HostMatrix/HostMatrix.H                       |  63 +++---
 common/common.H                               |   4 +
 lduLduBase/lduLduBase.H                       |  55 +++---
 6 files changed, 237 insertions(+), 262 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 383776a7c..344948570 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -143,127 +143,19 @@ struct MatrixInitFunctor {
             non_local_value_ptr = non_local_coo_ptr->get_values();
         }
 
-        if (partition_.get_ranks_per_gpu() != 1) {
-            // TODO add field name
-            PersistentArray<label> local_scatter_map_pers{
-                "local_scatter_map",  //
-                db_,
-                exec_,
-                0,
-                verbose_,
-                false,
-                false};
-
-            // TODO add field name
-            PersistentArray<label> non_local_scatter_map_pers{
-                "non_local_scatter_map",  //
-                db_,
-                exec_,
-                0,
-                verbose_,
-                false,
-                false};
-            // TODO add field name
-            PersistentArray<label> sorting_idx_pers{
-                "sorting_idx", db_, exec_, 0, verbose_, false, false};
-
-            // TODO add field name
-            PersistentArray<label> send_sizes_pers{
-                "send_sizes", db_, exec_, 0, verbose_, false, false};
-
-            // TODO add field name
-            PersistentArray<label> send_offs_pers{"send_offs", db_,   exec_, 0,
-                                                  verbose_,    false, false};
-
-            // TODO add field name
-            PersistentArray<label> recv_sizes_pers{
-                "recv_sizes", db_, exec_, 0, verbose_, false, false};
-
-            // TODO add field name
-            PersistentArray<label> recv_offs_pers{"recv_offs", db_,   exec_, 0,
-                                                  verbose_,    false, false};
-
-
-            auto comm = exec_.get_gko_mpi_host_comm();
-            // TIME_WITH_FIELDNAME(
-            //     verbose_, update_build_repartitioner, field_name_,
-            //     auto repartitioner = gko::share(
-            //         gko::experimental::distributed::repartitioner<
-            //             label, label>::create(*comm.get(),
-            //                                   partition_.get_host_partition(),
-            //                                   partition_.get_device_partition(),
-            //                                   false));)
-
-            auto local_nnz =
-                local_scatter_map_pers.get_array()->get_num_elems();
-            auto non_local_nnz =
-                non_local_scatter_map_pers.get_array()->get_num_elems();
-
-            gko::array<scalar> local_values{exec, local_nnz};
-            gko::array<scalar> non_local_values{exec, non_local_nnz};
-
-            std::vector<int> send_sizes;
-            std::vector<int> send_offsets;
-            std::vector<int> recv_sizes;
-            std::vector<int> recv_offsets;
-
-            send_sizes.assign(send_sizes_pers.get_array()->get_data(),
-                              send_sizes_pers.get_array()->get_data() +
-                                  send_sizes_pers.get_array()->get_num_elems());
-            send_offsets.assign(
-                send_offs_pers.get_array()->get_data(),
-                send_offs_pers.get_array()->get_data() +
-                    send_offs_pers.get_array()->get_num_elems());
-            recv_sizes.assign(recv_sizes_pers.get_array()->get_data(),
-                              recv_sizes_pers.get_array()->get_data() +
-                                  recv_sizes_pers.get_array()->get_num_elems());
-            recv_offsets.assign(
-                recv_offs_pers.get_array()->get_data(),
-                recv_offs_pers.get_array()->get_data() +
-                    recv_offs_pers.get_array()->get_num_elems());
-
-
-            // TIME_WITH_FIELDNAME(
-            //     verbose_, update_repartitioned_existing, field_name_,
-            //     repartitioner->update_existing(
-            //         *row_idxs_.get_array(), *non_local_row_idxs_.get_array(),
-            //         *coeffs.get(), *non_local_coeffs.get(),
-            //         *sorting_idx_pers.get_array().get(), send_sizes,
-            //         send_offsets, recv_sizes, recv_offsets,
-            //         *local_scatter_map_pers.get_array().get(),
-            //         *non_local_scatter_map_pers.get_array().get(),
-            //         local_values, non_local_values);)
-
-            auto value_view =
-                val_array::view(device_exec, local_nnz, value_ptr);
-
-            auto non_local_view = val_array::view(device_exec, non_local_nnz,
-                                                  non_local_value_ptr);
-
-            TIME_WITH_FIELDNAME(verbose_, update_offload_local_values,
-                                field_name_,
-                                local_values.set_executor(device_exec);
-                                value_view = local_values;)
-
-            TIME_WITH_FIELDNAME(verbose_, update_offload_non_local_values,
-                                field_name_,
-                                non_local_values.set_executor(device_exec);
-                                non_local_view = non_local_values;)
-        } else {
-            gko::array<scalar> device_values{device_exec, *coeffs.get()};
-            gko::array<scalar> device_non_local_values{device_exec,
-                                                       *non_local_coeffs.get()};
-
-            auto value_view = val_array::view(
-                device_exec, device_values.get_num_elems(), value_ptr);
-
-            auto non_local_view = val_array::view(
-                device_exec, device_non_local_values.get_num_elems(),
-                non_local_value_ptr);
-
-            value_view = device_values;
-            non_local_view = device_non_local_values;
-        }
+        gko::array<scalar> device_values{device_exec, *coeffs.get()};
+        gko::array<scalar> device_non_local_values{device_exec,
+                                                   *non_local_coeffs.get()};
+
+        auto value_view = val_array::view(
+            device_exec, device_values.get_num_elems(), value_ptr);
+
+        auto non_local_view = val_array::view(
+            device_exec, device_non_local_values.get_num_elems(),
+            non_local_value_ptr);
+
+        value_view = device_values;
+        non_local_view = device_non_local_values;
     }
 
     std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
@@ -294,7 +186,8 @@ struct MatrixInitFunctor {
     std::shared_ptr<dist_mtx> init() const
     {
         const label local_size = partition_.get_local_size();
-        word msg{"init global csr matrix of size " + std::to_string(local_size)};
+        word msg{"init global csr matrix of size " +
+                 std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
         auto coeffs = coeffs_.get_array();
@@ -313,26 +206,50 @@ struct MatrixInitFunctor {
             *coeffs.get());
 
         gko::device_matrix_data<scalar, label> non_local_A_data(
-            exec, gko::dim<2>(num_rows, non_local_cols->get_num_elems()), *non_local_rows.get(),
-            *non_local_cols.get(), *non_local_coeffs.get());
+            exec, gko::dim<2>(num_rows, non_local_cols->get_num_elems()),
+            *non_local_rows.get(), *non_local_cols.get(),
+            *non_local_coeffs.get());
 
 
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
             exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm());
-        dist_A->read_distributed(A_data, non_local_A_data,
-                gko::experimental::distributed::sparse_communicator::create(
-                    *comm.get(),
-                                 partition_.get_localized_partition()
-                                 ));
-
-        if (partition_.get_ranks_per_gpu() == 1) {
-            auto device_mat = generate_dist_mtx_with_inner_type(
-                exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
-            dist_A->move_to(device_mat.get());
-            return device_mat;
+
+        auto sparse_comm =
+            gko::experimental::distributed::sparse_communicator::create(
+                *comm.get(), partition_.get_localized_partition());
+
+        // TODO FIXME
+        auto recv_sizes = sparse_comm->get_recv_sizes();
+        auto recv_offs = sparse_comm->get_recv_offsets();
+        auto send_sizes = sparse_comm->get_send_sizes();
+        auto send_offs = sparse_comm->get_send_offsets();
+
+        if (Pstream::master() ) {
+        for (auto e: recv_sizes) {
+            std::cout << "recv_size: " << e << std::endl;
+        }
+
+        for (auto e: recv_offs) {
+            std::cout << "recv_offs: " << e << std::endl;
+        }
+
+        for (auto e: send_sizes) {
+            std::cout << "send_sizes: " << e << std::endl;
+        }
+
+        for (auto e: send_offs) {
+            std::cout << "send_offs: " << e << std::endl;
+        }
         }
 
+
+        dist_A->read_distributed(A_data, non_local_A_data, sparse_comm);
+
+        auto device_mat = generate_dist_mtx_with_inner_type(
+            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
+        dist_A->move_to(device_mat.get());
+        return device_mat;
     }
 };
 
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 13743ef50..eb8c8f9df 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -59,12 +59,14 @@ struct PartitionInitFunctor {
     // Array storing the ranks of the neighbouring processors
     const gko::array<label> target_sizes_;
 
-
-    PartitionInitFunctor(const ExecutorHandler &exec, const label rows_start,
-                         const label rows_end, const label verbose,
-                         const int ranks_per_gpu,
-                         const gko::array<label> &target_ids,
-                         const gko::array<label> target_sizes)
+    const std::vector<std::pair<gko::index_set<label>, label>> send_idxs_;
+
+    PartitionInitFunctor(
+        const ExecutorHandler &exec, const label rows_start,
+        const label rows_end, const label verbose, const int ranks_per_gpu,
+        const gko::array<label> &target_ids,
+        const gko::array<label> &target_sizes,
+        const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : exec_(exec),
           rows_end_(rows_end),
           rows_start_(rows_start),
@@ -72,9 +74,9 @@ struct PartitionInitFunctor {
           verbose_(verbose),
           ranks_per_gpu_(ranks_per_gpu),
           target_ids_(target_ids),
-          target_sizes_(target_sizes)
-    {
-    }
+          target_sizes_(target_sizes),
+          send_idxs_(send_idxs)
+    {}
 
     void update(std::shared_ptr<part_type> persistent_partition) const
     {
@@ -90,11 +92,9 @@ struct PartitionInitFunctor {
                  std::to_string(local_size_)};
         LOG_1(verbose_, msg)
 
-        std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
-
         auto host_partition = gko::experimental::distributed::
             localized_partition<>::build_from_blocked_recv(
-                exec, local_size_, send_idxs, target_ids_, target_sizes_);
+                exec, local_size_, send_idxs_, target_ids_, target_sizes_);
 
         if (ranks_per_gpu_ == 1) return host_partition;
     }
@@ -137,15 +137,17 @@ public:
      * @param offset
      * @param elements
      */
-    PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
-                        const label verbose, const int ranks_per_gpu,
-                        const label offset, const label elements,
-                        const gko::array<label> &target_ids,
-                        const gko::array<label> target_sizes)
+    PersistentPartition(
+        const objectRegistry &db, const ExecutorHandler &exec,
+        const label verbose, const int ranks_per_gpu, const label offset,
+        const label elements, const gko::array<label> &target_ids,
+        const gko::array<label> &target_sizes,
+        const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, offset, offset + elements, verbose,
-                                   ranks_per_gpu, target_ids, target_sizes),
+                                   ranks_per_gpu, target_ids, target_sizes,
+                                   send_idxs),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           offset_(offset),
@@ -156,7 +158,7 @@ public:
         label local_elements = local_elements_;
         comm->all_reduce(exec.get_ref_exec(), &local_elements, 1, MPI_SUM);
         global_elements_ = local_elements;
-          }
+    }
 
     std::shared_ptr<part_type> get_localized_partition() const
     {
@@ -166,10 +168,7 @@ public:
     // number of elements on this rank on the host
     label get_local_size() const { return local_elements_; }
 
-    label get_total_size() const
-    {
-        return global_elements_;
-    }
+    label get_total_size() const { return global_elements_; }
 
     label get_ranks_per_gpu() const { return ranks_per_gpu_; }
 };
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 2255967c5..196832952 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -102,60 +102,18 @@ HostMatrixWrapper<lduMatrix>::collect_interface_coeffs(
     const lduInterfaceFieldPtrsList &, const FieldField<Field, scalar> &,
     const bool) const;
 
-template <class MatrixType>
-std::vector<std::tuple<label, label, label>>
-HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
-    const lduInterfaceFieldPtrsList &interfaces) const
-{
-    std::vector<std::tuple<label, label, label>> local_interface_idxs{};
-    local_interface_idxs.reserve(local_interface_nnz_);
-
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        // TODO make this a separate specialized function
-        label interface_ctr = 0;
-        if (isA<cyclicLduInterface>(iface->interface())) {
-            const cyclicLduInterface &pldui =
-                refCast<const cyclicLduInterface>(iface->interface());
-            // const labelUList &rows = this->matrix().lduAddr().patchAddr(i);
-
-#ifdef WITH_ESI_VERSION
-            const label neighbPatchId = pldui.neighbPatchID();
-#else
-            const label neighbPatchId = pldui.nbrPatchID();
-#endif
-            const labelUList &cols =
-                this->matrix().lduAddr().patchAddr(neighbPatchId);
-            for (label cellI = 0; cellI < interface_size; cellI++) {
-                // DEBUG
-                // cout << "patch_ctr " << cellI << " ictr " << interface_ctr
-                //     << " patchId " << i << " neighPatchId " << neighbPatchId
-                //      << " row " << rows[cellI] << " col " << cols[cellI]
-                //      << "\n" << endl;
-                local_interface_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], cols[cellI]});
-                interface_ctr += 1;
-            }
-        }
-    }
-    return local_interface_idxs;
-}
 
 template <class MatrixType>
-std::pair<gko::array<label>, gko::array<label>>
-HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
+CommunicationPattern
+HostMatrixWrapper<MatrixType>::create_communication_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // temp vector to store neighbour proc number and number of cells to send
     std::vector<std::pair<label, label>> neighbour_procs{};
+
+    // temp map, mapping from neighbour rank interface cells
+    std::map<label, std::vector<label>> interface_cell_map{};
+
     // TODO FIXME use a lambda to avoid repeating
     for (int i = 0; i < interfaces.size(); i++) {
         if (interface_getter(interfaces, i) == nullptr) {
@@ -173,22 +131,54 @@ HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
 
             neighbour_procs.push_back(
                 std::pair<label, label>{neighbProcNo, interface_size});
+
+            //
+            auto search = interface_cell_map.find(neighbProcNo);
+            if (search == interface_cell_map.end()) {
+                interface_cell_map.insert(std::pair{
+                    neighbProcNo,
+                    std::vector<label>(face_cells.begin(), face_cells.end())});
+            } else {
+                // TODO FIXME reserve size
+                auto cur_face_cells = interface_cell_map[neighbProcNo];
+                for (auto face_cell : face_cells) {
+                    cur_face_cells.push_back(face_cell);
+                }
+                interface_cell_map[neighbProcNo] = cur_face_cells;
+            }
         }
     }
 
-    // reduce vector
+    // reduce vector of sizes
+    // this assumes that a rank can be connected to the same neighbour through
+    // different interfaces this migth not be necessary
     label n_procs = 0;
     std::map<label, label> reduce_map{};
     for (auto [proc, n_faces] : neighbour_procs) {
         auto search = reduce_map.find(proc);
         if (search == reduce_map.end()) {
             n_procs += 1;
-            reduce_map.insert(std::pair<label,label>{proc, n_faces});
+            reduce_map.insert(std::pair<label, label>{proc, n_faces});
         } else {
             reduce_map[proc] = reduce_map[proc] + n_faces;
         }
     }
 
+    // create index_sets
+    // currently this assumes that there is only one interface to a given
+    // neighbour rank
+    std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
+    for (auto [proc, interface_cells] : interface_cell_map) {
+        label interface_size = interface_cells.size();
+        auto exec = exec_.get_ref_exec();
+        send_idxs.push_back(std::pair<gko::index_set<label>, label>(
+            gko::index_set(
+                exec, interface_size,
+                gko::array<label>::view(exec, interface_size, interface_cells.data()),
+                true),
+            proc));
+    }
+
     // convert to gko::array
     gko::array<label> target_ids{exec_.get_ref_exec(), n_procs};
     gko::array<label> target_sizes{exec_.get_ref_exec(), n_procs};
@@ -200,14 +190,61 @@ HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
         iter++;
     }
 
-    return std::pair(target_ids, target_sizes);
+    return CommunicationPattern{target_ids, target_sizes, send_idxs};
 }
 
-template
-std::pair<gko::array<label>, gko::array<label>>
-HostMatrixWrapper<lduMatrix>::assemble_proc_id_and_sizes(
+template CommunicationPattern
+HostMatrixWrapper<lduMatrix>::create_communication_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const;
 
+
+template <class MatrixType>
+std::vector<std::tuple<label, label, label>>
+HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    std::vector<std::tuple<label, label, label>> local_interface_idxs{};
+    local_interface_idxs.reserve(local_interface_nnz_);
+
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+
+        const auto iface{interface_getter(interfaces, i)};
+
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        // TODO make this a separate specialized function
+        label interface_ctr = 0;
+        if (isA<cyclicLduInterface>(iface->interface())) {
+            const cyclicLduInterface &pldui =
+                refCast<const cyclicLduInterface>(iface->interface());
+            // const labelUList &rows = this->matrix().lduAddr().patchAddr(i);
+
+#ifdef WITH_ESI_VERSION
+            const label neighbPatchId = pldui.neighbPatchID();
+#else
+            const label neighbPatchId = pldui.nbrPatchID();
+#endif
+            const labelUList &cols =
+                this->matrix().lduAddr().patchAddr(neighbPatchId);
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                // DEBUG
+                // cout << "patch_ctr " << cellI << " ictr " << interface_ctr
+                //     << " patchId " << i << " neighPatchId " << neighbPatchId
+                //      << " row " << rows[cellI] << " col " << cols[cellI]
+                //      << "\n" << endl;
+                local_interface_idxs.push_back(
+                    {interface_ctr, face_cells[cellI], cols[cellI]});
+                interface_ctr += 1;
+            }
+        }
+    }
+    return local_interface_idxs;
+}
+
 template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 9fb99043b..fffb62684 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -39,6 +39,7 @@ SourceFiles
 namespace Foam {
 
 
+    // TODO Move this somewhere else
 struct PersistentSparsityPattern {
     PersistentSparsityPattern(
       const word &fieldName,
@@ -84,6 +85,20 @@ struct PersistentSparsityPattern {
     mutable PersistentArray<label> ldu_mapping_;
 };
 
+struct CommunicationPattern {
+    // an array storing to which rank to communicate
+    gko::array<label> target_ids;
+
+    // an array storing how many elements to communicate
+    // to the corresponding target_id
+    gko::array<label> target_sizes;
+
+    // send_idx stores the index_set ie which cells
+    // are owned by the interface and the corresponding target rank
+    std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
+};
+
+
 /* The HostMatrixWrapper class handles the conversion from OpenFOAMs lduMatrix
  * format into Ginkgo array data structures
  *
@@ -138,9 +153,7 @@ private:
     // non-local indices
     const label non_local_matrix_nnz_;
 
-    // Array storing the ranks of the neighbouring processors
-    const std::pair<gko::array<label>, gko::array<label>>
-        proc_target_id_and_sizes_;
+    const CommunicationPattern communication_pattern_;
 
     mutable PersistentSparsityPattern non_local_sparsity_;
 
@@ -200,7 +213,7 @@ public:
                      // matrix it will be put on the device
           },
           non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
-          proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
+          communication_pattern_(create_communication_pattern(interfaces)),
           non_local_sparsity_{
               fieldName + "_non_local", db,       exec_,
               non_local_matrix_nnz_,    verbose_,
@@ -311,42 +324,46 @@ public:
         const bool local) const;
 
 
-    /* Iterates all local interfaces and returns the relative order and
-     *corresponding row and column indices
-     **
-     */
-    std::vector<std::tuple<label, label, label>>
-    collect_local_interface_indices(
-        const lduInterfaceFieldPtrsList &interfaces_) const;
-
-
     /* Iterates all interfaces and counts the number of elements
     **
     ** @param interfaces The list of interfaces for the search
     ** @param proc_interfaces Count only elements on (true)
     *processorLduInterfaces or exclude processorLduInterfaces (false)
+     ** TODO make private and no discard
     */
     label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
                               bool proc_interfaces) const;
 
-    /* Iterates all interfaces and counts the number of unique neighbour
-     *processors and number of interfaces in total for this processor
+     /** Iterates all local interfaces and returns the relative order and
+     **corresponding row and column indices
+     **
+     ** @return vector of tuples containing the interface number, the local row, the local column
+     **/
+    std::vector<std::tuple<label, label, label>>
+    collect_local_interface_indices(
+        const lduInterfaceFieldPtrsList &interfaces_) const;
+
+
+    /** Iterates all interfaces and counts the number of unique neighbour
+     **processors and number of interfaces in total for this processor
+     ** and collects all interface cells of this rank
      **
      ** @param interfaces The list of interfaces for the search
+     ** @return the CommunicationPattern 
+     ** TODO make private and no discard
      */
-    std::pair<gko::array<label>, gko::array<label>> assemble_proc_id_and_sizes(
+    CommunicationPattern create_communication_pattern(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
-    /* Getter for proc_id and sizes arrays
-     *
-     */
-    const std::pair<gko::array<label>, gko::array<label>>
-        &get_proc_target_id_and_sizes() const
+    /** Getter for the CommunicationPattern
+     **
+     ** @return the CommunicationPattern 
+     **/
+    const CommunicationPattern &get_communication_pattern() const
     {
-        return proc_target_id_and_sizes_;
+        return communication_pattern_;
     };
 
-
     /* Iterates all interfaces and collects non-local global column indices
     **
     ** TODO FIXME this is not uptodate
diff --git a/common/common.H b/common/common.H
index 1c5998d94..5100477e6 100644
--- a/common/common.H
+++ b/common/common.H
@@ -60,6 +60,10 @@ namespace Foam {
         }                                                                   \
     }
 
+
+#define IN_USE_DO_NOT_REMOVE(FUNC) \
+std::cout << FUNC <<  " in "   << __FILENAME__ << ":" <<__LINE__ << " is in use do not remove"  << std::endl
+
 #define LOG_0(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 0, MSG, 0)
 #define LOG_1(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 1, MSG, 0)
 #define LOG_2(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 2, MSG, 0)
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index ff1eee075..8548f7f7e 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -123,7 +123,8 @@ public:
             int rank;
             MPI_Comm_rank(MPI_COMM_WORLD, &rank);
             word fn = "/tmp/mpi_debug_" + std::to_string(rank) + ".pid";
-            word msg = "writing pid to" + fn;
+            word msg =
+                "writing pid to" + fn + " waiting 20s for debugger to attach";
             LOG_0(verbose_, msg)
             std::ofstream os(fn);
             os << getpid() << std::endl;
@@ -216,8 +217,8 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
-        const auto &[target_ids, target_sizes] =
-            this->get_proc_target_id_and_sizes();
+        const auto communication_pattern = this->get_communication_pattern();
+
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
@@ -225,8 +226,9 @@ public:
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
             this->get_global_row_index().offset(Pstream::myProcNo()),  // offset
             psi.size(),  // number of elements
-            target_ids,
-            target_sizes};
+            communication_pattern.target_ids,
+            communication_pattern.target_sizes,
+            communication_pattern.send_idxs};
 
         // TODO this is now the local matrix part for dist_A
         MatrixWrapper dist_A{db_,
@@ -317,33 +319,32 @@ public:
 
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)
-        auto bandwidth_copy_back = sizeof(scalar) *
-                                   partition.get_local_size() /
+        auto bandwidth_copy_back = sizeof(scalar) * partition.get_local_size() /
                                    delta_t_copy_x_back / 1000.0;
 
         solverPerf.initialResidual() = this->get_init_res_norm();
         solverPerf.finalResidual() = this->get_res_norm();
         solverPerf.nIterations() = this->get_number_of_iterations();
-        this->store_number_of_iterations();
-        auto time_for_res_norm_eval = this->get_res_norm_time();
-        auto time_per_iter =
-            delta_t_solve / max(this->get_number_of_iterations(), 1);
-        scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
-        this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
-            this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1, 0);
-        this->set_prev_rel_res_cost(prev_rel_res_cost);
-        auto time_per_iter_and_dof =
-            time_per_iter * 1000.0 / partition.get_total_size();
-        word msg =
-            "\nStatistics:\n\tTime per iteration: " +
-            std::to_string(time_per_iter) +
-            std::string(" [mu s]\n\tTime per residual norm calculation: ") +
-            std::to_string(time_for_res_norm_eval) +
-            std::string(" [mu s]\n\tTime per iteration and DOF: ") +
-            std::to_string(time_per_iter_and_dof) + std::string(" [ns]") +
-            std::string("\n\tRetrieve results bandwidth ") +
-            std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
-        MLOG_0(verbose_, msg)
+        // this->store_number_of_iterations();
+        // auto time_for_res_norm_eval = this->get_res_norm_time();
+        // auto time_per_iter =
+        //     delta_t_solve / max(this->get_number_of_iterations(), 1);
+        // scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
+        // this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
+        //     this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1, 0);
+        // this->set_prev_rel_res_cost(prev_rel_res_cost);
+        // auto time_per_iter_and_dof =
+        //     time_per_iter * 1000.0 / partition.get_total_size();
+        // word msg =
+        //     "\nStatistics:\n\tTime per iteration: " +
+        //     std::to_string(time_per_iter) +
+        //     std::string(" [mu s]\n\tTime per residual norm calculation: ") +
+        //     std::to_string(time_for_res_norm_eval) +
+        //     std::string(" [mu s]\n\tTime per iteration and DOF: ") +
+        //     std::to_string(time_per_iter_and_dof) + std::string(" [ns]") +
+        //     std::string("\n\tRetrieve results bandwidth ") +
+        //     std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
+        // MLOG_0(verbose_, msg)
 
         return solverPerf;
     }

From 25e80f73be0f3e44fa4851b7242e0dfc71533a95 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 08:02:24 +0100
Subject: [PATCH 018/125] reorder header

---
 HostMatrix/HostMatrix.H | 151 ++++++++++++++++++++--------------------
 1 file changed, 76 insertions(+), 75 deletions(-)

diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index fffb62684..29bcf6811 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -159,6 +159,7 @@ private:
 
     mutable PersistentArray<scalar> non_local_coeffs_;
 
+    // FIXME TODO unused
     const word permutation_matrix_name_;
 
     const bool permutation_stored_;
@@ -166,6 +167,70 @@ private:
     mutable std::shared_ptr<gko::LinOp> P_;
 
 
+    /* Iterates all interfaces and collects the coefficients into a vector
+    **
+    ** @param local whether local or non local coefficients should be collected
+    */
+    std::vector<scalar> collect_interface_coeffs(
+        const lduInterfaceFieldPtrsList &interfaces_,
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        const bool local) const;
+
+
+    /* Iterates all interfaces and counts the number of elements
+    **
+    ** @param interfaces The list of interfaces for the search
+    ** @param proc_interfaces Count only elements on (true)
+    *processorLduInterfaces or exclude processorLduInterfaces (false)
+     ** TODO make private and no discard
+    */
+    label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
+                              bool proc_interfaces) const;
+
+     /** Iterates all local interfaces and returns the relative order and
+     **corresponding row and column indices
+     **
+     ** @return vector of tuples containing the interface number, the local row, the local column
+     **/
+    std::vector<std::tuple<label, label, label>>
+    collect_local_interface_indices(
+        const lduInterfaceFieldPtrsList &interfaces_) const;
+
+
+    /** Iterates all interfaces and counts the number of unique neighbour
+     **processors and number of interfaces in total for this processor
+     ** and collects all interface cells of this rank
+     **
+     ** @param interfaces The list of interfaces for the search
+     ** @return the CommunicationPattern
+     ** TODO make private and no discard
+     */
+    CommunicationPattern create_communication_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /* Iterates all interfaces and collects non-local global column indices
+    **
+    ** TODO FIXME this is not uptodate
+    ** Non-local global column indices are computed from the global cell-id
+    ** (row) on the other side of an interface
+    **
+    ** @return vector off size nnz_non_local, with sections for each interface
+    ** ret = [(1,2,3),(10,20,30), ...]
+    **         i0      i1,       i...
+    */
+    std::vector<std::tuple<label, label, label>> collect_non_local_col_indices(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /* Based on OpenFOAMs ldu matrix format this function computes two
+     *consecutive index arrays in row major ordering and scattering indices
+     **
+     */
+    void init_local_sparsity_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    void init_non_local_sparsity_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
 public:
     // segregated wrapper constructor
     // TODO - compute local_interface_nnzs these are matrix coefficient
@@ -314,94 +379,31 @@ public:
         // TODO reimplement this
     }
 
-    /* Iterates all interfaces and collects the coefficients into a vector
-    **
-    ** @param local whether local or non local coefficients should be collected
-    */
-    std::vector<scalar> collect_interface_coeffs(
-        const lduInterfaceFieldPtrsList &interfaces_,
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        const bool local) const;
 
+    // updater
 
-    /* Iterates all interfaces and counts the number of elements
-    **
-    ** @param interfaces The list of interfaces for the search
-    ** @param proc_interfaces Count only elements on (true)
-    *processorLduInterfaces or exclude processorLduInterfaces (false)
-     ** TODO make private and no discard
-    */
-    label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
-                              bool proc_interfaces) const;
+    void update_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
-     /** Iterates all local interfaces and returns the relative order and
-     **corresponding row and column indices
-     **
-     ** @return vector of tuples containing the interface number, the local row, the local column
-     **/
-    std::vector<std::tuple<label, label, label>>
-    collect_local_interface_indices(
-        const lduInterfaceFieldPtrsList &interfaces_) const;
+    void update_non_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
+    // getter
+    public:
 
-    /** Iterates all interfaces and counts the number of unique neighbour
-     **processors and number of interfaces in total for this processor
-     ** and collects all interface cells of this rank
-     **
-     ** @param interfaces The list of interfaces for the search
-     ** @return the CommunicationPattern 
-     ** TODO make private and no discard
-     */
-    CommunicationPattern create_communication_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
+    bool get_verbose() const { return verbose_; }
 
     /** Getter for the CommunicationPattern
      **
-     ** @return the CommunicationPattern 
+     ** @return the CommunicationPattern
      **/
     const CommunicationPattern &get_communication_pattern() const
     {
         return communication_pattern_;
     };
 
-    /* Iterates all interfaces and collects non-local global column indices
-    **
-    ** TODO FIXME this is not uptodate
-    ** Non-local global column indices are computed from the global cell-id
-    ** (row) on the other side of an interface
-    **
-    ** @return vector off size nnz_non_local, with sections for each interface
-    ** ret = [(1,2,3),(10,20,30), ...]
-    **         i0      i1,       i...
-    */
-    std::vector<std::tuple<label, label, label>> collect_non_local_col_indices(
-        const lduInterfaceFieldPtrsList &interfaces) const;
-
-    /* Based on OpenFOAMs ldu matrix format this function computes two
-     *consecutive index arrays in row major ordering and scattering indices
-     **
-     */
-    void init_local_sparsity_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
-
-    void init_non_local_sparsity_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
-
-
-    bool get_verbose() const { return verbose_; }
-
-    // updater
-
-    void update_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
-
-    void update_non_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
-
-    // getter
-
     label get_local_nrows() const { return nrows_; }
 
     label get_global_nrows() const { return global_row_index_.size(); }
@@ -413,7 +415,6 @@ public:
         return global_row_index_;
     }
 
-
     PersistentArray<scalar> &get_local_coeffs() const  //
     {
         return local_coeffs_;

From e3603503919918ef6275fde5b304d1cf7718332f Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 08:42:41 +0100
Subject: [PATCH 019/125] move ctr to body

---
 HostMatrix/HostMatrix.C | 123 +++++++++++++++++++++++++++++++---------
 1 file changed, 95 insertions(+), 28 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 196832952..a0dabdea6 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -62,8 +62,89 @@ label HostMatrixWrapper<MatrixType>::count_interface_nnz(
     return ctr;
 }
 
-template label HostMatrixWrapper<lduMatrix>::count_interface_nnz(
-    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const;
+
+template <class MatrixType>
+HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
+        const objectRegistry &db, const MatrixType &matrix,
+        // coeffs for cells on boundaries
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        // coeffs for internal cells
+        const FieldField<Field, scalar> &interfaceIntCoeffs,
+        // pointers to interfaces can be used to access concrete
+        // functions such as transferring indices, patch neighbours etc
+        const lduInterfaceFieldPtrsList &interfaces,
+        const dictionary &solverControls, const word &fieldName)
+        : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
+                             interfaceIntCoeffs, interfaces, solverControls),
+          exec_{db, solverControls, fieldName},
+          device_id_guard_{db, fieldName, exec_.get_device_exec()},
+          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+          scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
+          nrows_(matrix.diag().size()),
+          local_interface_nnz_(count_interface_nnz(interfaces, false)),
+          upper_nnz_(matrix.lduAddr().upperAddr().size()),
+          non_diag_nnz_(2 * upper_nnz_),
+          local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
+          local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
+                                         local_interface_nnz_),
+          global_row_index_{nrows_},
+          local_sparsity_{
+              fieldName + "_local",           db,       exec_,
+              local_matrix_w_interfaces_nnz_, verbose_,
+          },
+          local_coeffs_{
+              fieldName + "_local_coeffs",
+              db,
+              exec_,
+              local_matrix_w_interfaces_nnz_,
+              verbose_,
+              true,  // needs to be updated
+              false  // leave it on host once it is turned into a distributed
+                     // matrix it will be put on the device
+          },
+          non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
+          communication_pattern_(create_communication_pattern(interfaces)),
+          non_local_sparsity_{
+              fieldName + "_non_local", db,       exec_,
+              non_local_matrix_nnz_,    verbose_,
+          },
+          non_local_coeffs_{
+              fieldName + "_non_local_coeffs",
+              db,
+              exec_,
+              non_local_matrix_nnz_,
+              verbose_,
+              true,  // needs to be updated
+              false  // leave it on host once it is turned into a distributed
+                     // matrix it will be put on the device
+          },
+          permutation_matrix_name_{"PermutationMatrix"},
+          permutation_stored_{
+              db.template foundObject<regIOobject>(permutation_matrix_name_)},
+          P_{(permutation_stored_) ? db.template lookupObjectRef<
+                                           DevicePersistentBase<gko::LinOp>>(
+                                           permutation_matrix_name_)
+                                         .get_ptr()
+                                   : nullptr}
+    {
+        if (!local_sparsity_.col_idxs_.get_stored() ||
+            local_sparsity_.col_idxs_.get_update()) {
+            TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
+                                this->fieldName(),
+                                init_local_sparsity_pattern(interfaces);)
+            TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
+                                this->fieldName(),
+                                init_non_local_sparsity_pattern(interfaces);)
+        }
+        if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
+            TIME_WITH_FIELDNAME(
+                verbose_, update_local_matrix_data, this->fieldName(),
+                update_local_matrix_data(interfaces, interfaceBouCoeffs);)
+            TIME_WITH_FIELDNAME(
+                verbose_, update_non_local_matrix_data, this->fieldName(),
+                update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
+        }
+    }
 
 
 template <class MatrixType>
@@ -97,12 +178,6 @@ std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
 }
 
 
-template std::vector<scalar>
-HostMatrixWrapper<lduMatrix>::collect_interface_coeffs(
-    const lduInterfaceFieldPtrsList &, const FieldField<Field, scalar> &,
-    const bool) const;
-
-
 template <class MatrixType>
 CommunicationPattern
 HostMatrixWrapper<MatrixType>::create_communication_pattern(
@@ -193,10 +268,6 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     return CommunicationPattern{target_ids, target_sizes, send_idxs};
 }
 
-template CommunicationPattern
-HostMatrixWrapper<lduMatrix>::create_communication_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const;
-
 
 template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
@@ -278,10 +349,6 @@ HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
     return non_local_idxs;
 }
 
-template std::vector<std::tuple<label, label, label>>
-HostMatrixWrapper<lduMatrix>::collect_non_local_col_indices(
-    const lduInterfaceFieldPtrsList &interfaces) const;
-
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
@@ -326,9 +393,6 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     }
 }
 
-template void HostMatrixWrapper<lduMatrix>::init_non_local_sparsity_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const;
-
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
@@ -492,9 +556,6 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     LOG_1(verbose_, "done init host matrix")
 }
 
-template void HostMatrixWrapper<lduMatrix>::init_local_sparsity_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const;
-
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
@@ -590,10 +651,6 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     }
 }
 
-template void HostMatrixWrapper<lduMatrix>::update_local_matrix_data(
-    const lduInterfaceFieldPtrsList &interfaces,
-    const FieldField<Field, scalar> &interfaceBouCoeffs) const;
-
 
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
@@ -636,6 +693,16 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     i_device_view = tmp_contiguous_iface;
 }
 
-template void HostMatrixWrapper<lduMatrix>::update_non_local_matrix_data(
-    const lduInterfaceFieldPtrsList &, const FieldField<Field, scalar> &) const;
+template 
+HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
+        const objectRegistry &db, const lduMatrix &matrix,
+        // coeffs for cells on boundaries
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        // coeffs for internal cells
+        const FieldField<Field, scalar> &interfaceIntCoeffs,
+        // pointers to interfaces can be used to access concrete
+        // functions such as transferring indices, patch neighbours etc
+        const lduInterfaceFieldPtrsList &interfaces,
+        const dictionary &solverControls, const word &fieldName);
+
 }  // namespace Foam

From 0c96fffc9362c4ff98dbd9f4a0a27dcefa38c419 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 08:53:31 +0100
Subject: [PATCH 020/125] remove gko globalIndex

---
 DevicePersistent/Partition/Partition.H | 23 ++-----
 HostMatrix/HostMatrix.C                |  1 -
 HostMatrix/HostMatrix.H                | 86 +-------------------------
 lduLduBase/lduLduBase.H                |  1 -
 4 files changed, 8 insertions(+), 103 deletions(-)

diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index eb8c8f9df..beabad5c6 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -43,10 +43,6 @@ struct PartitionInitFunctor {
 
     const ExecutorHandler &exec_;
 
-    const label rows_end_;
-
-    const label rows_start_;
-
     const label local_size_;
 
     const label verbose_;
@@ -62,15 +58,13 @@ struct PartitionInitFunctor {
     const std::vector<std::pair<gko::index_set<label>, label>> send_idxs_;
 
     PartitionInitFunctor(
-        const ExecutorHandler &exec, const label rows_start,
-        const label rows_end, const label verbose, const int ranks_per_gpu,
+        const ExecutorHandler &exec, const label local_size
+        , const label verbose, const int ranks_per_gpu,
         const gko::array<label> &target_ids,
         const gko::array<label> &target_sizes,
         const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : exec_(exec),
-          rows_end_(rows_end),
-          rows_start_(rows_start),
-          local_size_(rows_end_ - rows_start_),
+          local_size_(local_size),
           verbose_(verbose),
           ranks_per_gpu_(ranks_per_gpu),
           target_ids_(target_ids),
@@ -107,9 +101,6 @@ struct PartitionInitFunctor {
  * Here device partitioning refers to the partitioning as used on for the Ginkgo
  * data structures and can also reside on the host if the executor is either
  * reference or omp.
- *
- *
- *
  * */
 class PersistentPartition
     : public PersistentBase<
@@ -117,15 +108,12 @@ class PersistentPartition
           PartitionInitFunctor> {
     const int ranks_per_gpu_;
 
-    const label offset_;
-
     const label local_elements_;
 
     mutable label global_elements_;
 
     const ExecutorHandler &exec_;
 
-
 public:
     using part_type = gko::experimental::distributed::localized_partition<>;
     /* PersistentPartition constructor using existing memory
@@ -139,18 +127,17 @@ public:
      */
     PersistentPartition(
         const objectRegistry &db, const ExecutorHandler &exec,
-        const label verbose, const int ranks_per_gpu, const label offset,
+        const label verbose, const int ranks_per_gpu,
         const label elements, const gko::array<label> &target_ids,
         const gko::array<label> &target_sizes,
         const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
-              PartitionInitFunctor(exec, offset, offset + elements, verbose,
+              PartitionInitFunctor(exec, elements, verbose,
                                    ranks_per_gpu, target_ids, target_sizes,
                                    send_idxs),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
-          offset_(offset),
           local_elements_(elements),
           exec_(exec)
     {
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index a0dabdea6..403a3d40a 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -87,7 +87,6 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
           local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
           local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
                                          local_interface_nnz_),
-          global_row_index_{nrows_},
           local_sparsity_{
               fieldName + "_local",           db,       exec_,
               local_matrix_w_interfaces_nnz_, verbose_,
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 29bcf6811..34e781cc7 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -104,7 +104,8 @@ struct CommunicationPattern {
  *
  * It mainly generates:
  * - sparsity pattern/device_matrix_data of the local_matrix and
- * non_local_matrix (with global columns)
+ * - non_local_matrix (with global columns)
+ * - CommuncationPattern
  * */
 template <class MatrixType>
 class HostMatrixWrapper : public MatrixType::solver {
@@ -144,8 +145,6 @@ private:
     // nnzs of local matrix including local interfaces
     const label local_matrix_w_interfaces_nnz_;
 
-    const gkoGlobalIndex global_row_index_;
-
     mutable PersistentSparsityPattern local_sparsity_;
 
     mutable PersistentArray<scalar> local_coeffs_;
@@ -248,78 +247,7 @@ public:
         // pointers to interfaces can be used to access concrete
         // functions such as transferring indices, patch neighbours etc
         const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls, const word &fieldName)
-        : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
-                             interfaceIntCoeffs, interfaces, solverControls),
-          exec_{db, solverControls, fieldName},
-          device_id_guard_{db, fieldName, exec_.get_device_exec()},
-          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-          scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
-          nrows_(matrix.diag().size()),
-          local_interface_nnz_(count_interface_nnz(interfaces, false)),
-          upper_nnz_(matrix.lduAddr().upperAddr().size()),
-          non_diag_nnz_(2 * upper_nnz_),
-          local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
-          local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
-                                         local_interface_nnz_),
-          global_row_index_{nrows_},
-          local_sparsity_{
-              fieldName + "_local",           db,       exec_,
-              local_matrix_w_interfaces_nnz_, verbose_,
-          },
-          local_coeffs_{
-              fieldName + "_local_coeffs",
-              db,
-              exec_,
-              local_matrix_w_interfaces_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
-          communication_pattern_(create_communication_pattern(interfaces)),
-          non_local_sparsity_{
-              fieldName + "_non_local", db,       exec_,
-              non_local_matrix_nnz_,    verbose_,
-          },
-          non_local_coeffs_{
-              fieldName + "_non_local_coeffs",
-              db,
-              exec_,
-              non_local_matrix_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          permutation_matrix_name_{"PermutationMatrix"},
-          permutation_stored_{
-              db.template foundObject<regIOobject>(permutation_matrix_name_)},
-          P_{(permutation_stored_) ? db.template lookupObjectRef<
-                                           DevicePersistentBase<gko::LinOp>>(
-                                           permutation_matrix_name_)
-                                         .get_ptr()
-                                   : nullptr}
-    {
-        if (!local_sparsity_.col_idxs_.get_stored() ||
-            local_sparsity_.col_idxs_.get_update()) {
-            TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
-                                this->fieldName(),
-                                init_local_sparsity_pattern(interfaces);)
-            TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
-                                this->fieldName(),
-                                init_non_local_sparsity_pattern(interfaces);)
-        }
-        if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
-            TIME_WITH_FIELDNAME(
-                verbose_, update_local_matrix_data, this->fieldName(),
-                update_local_matrix_data(interfaces, interfaceBouCoeffs);)
-            TIME_WITH_FIELDNAME(
-                verbose_, update_non_local_matrix_data, this->fieldName(),
-                update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
-        }
-    }
+        const dictionary &solverControls, const word &fieldName);
 
     // TODO reimplement this at some point
     // coupled  wrapper constructor
@@ -337,7 +265,6 @@ public:
           local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
           local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
                                          local_interface_nnz_),
-          global_row_index_{nrows_},
           local_sparsity_{
               fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
           },
@@ -406,15 +333,8 @@ public:
 
     label get_local_nrows() const { return nrows_; }
 
-    label get_global_nrows() const { return global_row_index_.size(); }
-
     const ExecutorHandler &get_exec_handler() const { return exec_; }
 
-    const gkoGlobalIndex &get_global_row_index() const
-    {
-        return global_row_index_;
-    }
-
     PersistentArray<scalar> &get_local_coeffs() const  //
     {
         return local_coeffs_;
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 8548f7f7e..813c84f7a 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -224,7 +224,6 @@ public:
             this->get_exec_handler(),
             verbose_,
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
-            this->get_global_row_index().offset(Pstream::myProcNo()),  // offset
             psi.size(),  // number of elements
             communication_pattern.target_ids,
             communication_pattern.target_sizes,

From 0eef355fc767eb6226e5d8484874536a80974a4f Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 09:11:55 +0100
Subject: [PATCH 021/125] add stopping criterion

---
 Solver/CG/GKOCG.H                     | 7 ++-----
 StoppingCriterion/StoppingCriterion.H | 2 --
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 7ad2ea3b5..3da54f949 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -69,8 +69,7 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto cg = cg::build().with_criteria(
-              gko::stop::Iteration::build().with_max_iters(20u)
-                // stoppingCriterionVec_
+                stoppingCriterionVec_
                 ).on(exec);
         return cg;
     };
@@ -81,9 +80,7 @@ public:
     {
         auto cg = cg::build()
                       .with_criteria(
-                              gko::stop::Iteration::build().with_max_iters(20u)
-                              //
-                              // stoppingCriterionVec_
+                              stoppingCriterionVec_
                               )
                       .with_generated_preconditioner(precond)
                       .on(exec);
diff --git a/StoppingCriterion/StoppingCriterion.H b/StoppingCriterion/StoppingCriterion.H
index eda67ccbe..64c8a7178 100644
--- a/StoppingCriterion/StoppingCriterion.H
+++ b/StoppingCriterion/StoppingCriterion.H
@@ -187,8 +187,6 @@ class StoppingCriterion {
                 *(parameters_.iter) += 1;
                 return false;
             }
-            // TODO FIXME
-            std::cout << "iteration\n ";
 
             const auto exec = this->get_executor();
 

From fa62373fb2d9675407041155b976b6a3e09693bb Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 10:21:17 +0100
Subject: [PATCH 022/125] switch gko branch

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9e3d69238..48033155e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT OGL_ALLOW_REFERENCE_ONLY)
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "sparse-communicator-ogl"
+    "sparse-communicator"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)

From 7ec263317467597cbc23a6ef6b565c274aec93fb Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 10:24:22 +0100
Subject: [PATCH 023/125] reformat

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       |  32 +--
 DevicePersistent/Partition/Partition.H        |  13 +-
 DevicePersistent/Vector/Vector.H              |  14 +-
 HostMatrix/HostMatrix.C                       | 183 +++++++++---------
 HostMatrix/HostMatrix.H                       |  18 +-
 Solver/CG/GKOCG.H                             |   8 +-
 common/common.C                               |   4 +-
 lduLduBase/lduLduBase.H                       |   3 +-
 8 files changed, 134 insertions(+), 141 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 344948570..91fc541e6 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -225,22 +225,22 @@ struct MatrixInitFunctor {
         auto send_sizes = sparse_comm->get_send_sizes();
         auto send_offs = sparse_comm->get_send_offsets();
 
-        if (Pstream::master() ) {
-        for (auto e: recv_sizes) {
-            std::cout << "recv_size: " << e << std::endl;
-        }
-
-        for (auto e: recv_offs) {
-            std::cout << "recv_offs: " << e << std::endl;
-        }
-
-        for (auto e: send_sizes) {
-            std::cout << "send_sizes: " << e << std::endl;
-        }
-
-        for (auto e: send_offs) {
-            std::cout << "send_offs: " << e << std::endl;
-        }
+        if (Pstream::master()) {
+            for (auto e : recv_sizes) {
+                std::cout << "recv_size: " << e << std::endl;
+            }
+
+            for (auto e : recv_offs) {
+                std::cout << "recv_offs: " << e << std::endl;
+            }
+
+            for (auto e : send_sizes) {
+                std::cout << "send_sizes: " << e << std::endl;
+            }
+
+            for (auto e : send_offs) {
+                std::cout << "send_offs: " << e << std::endl;
+            }
         }
 
 
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index beabad5c6..64cba12d4 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -58,8 +58,8 @@ struct PartitionInitFunctor {
     const std::vector<std::pair<gko::index_set<label>, label>> send_idxs_;
 
     PartitionInitFunctor(
-        const ExecutorHandler &exec, const label local_size
-        , const label verbose, const int ranks_per_gpu,
+        const ExecutorHandler &exec, const label local_size,
+        const label verbose, const int ranks_per_gpu,
         const gko::array<label> &target_ids,
         const gko::array<label> &target_sizes,
         const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
@@ -127,15 +127,14 @@ public:
      */
     PersistentPartition(
         const objectRegistry &db, const ExecutorHandler &exec,
-        const label verbose, const int ranks_per_gpu,
-        const label elements, const gko::array<label> &target_ids,
+        const label verbose, const int ranks_per_gpu, const label elements,
+        const gko::array<label> &target_ids,
         const gko::array<label> &target_sizes,
         const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
-              PartitionInitFunctor(exec, elements, verbose,
-                                   ranks_per_gpu, target_ids, target_sizes,
-                                   send_idxs),
+              PartitionInitFunctor(exec, elements, verbose, ranks_per_gpu,
+                                   target_ids, target_sizes, send_idxs),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           local_elements_(elements),
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 640c60c40..86c93daa4 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -191,16 +191,16 @@ public:
 
         // TODO why does it need its size
         // repartitioner size should be sufficient
-        auto target_vector = dist_vec::create(
-            exec_.get_ref_exec(), *comm.get(), gko::dim<2>{global_size, 1},
-            gko::dim<2>{local_size, 1}, 1);
+        auto target_vector = dist_vec::create(exec_.get_ref_exec(), *comm.get(),
+                                              gko::dim<2>{global_size, 1},
+                                              gko::dim<2>{local_size, 1}, 1);
         scalar *host_buffer = host_device_vector->get_local_values();
 
-        auto host_buffer_view = gko::array<T>::view(
-            exec_.get_ref_exec(), local_size, host_buffer);
+        auto host_buffer_view =
+            gko::array<T>::view(exec_.get_ref_exec(), local_size, host_buffer);
 
-        auto to_view = gko::array<T>::view(
-            exec_.get_ref_exec(), local_size, const_cast<T *>(memory_));
+        auto to_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
+                                           const_cast<T *>(memory_));
         to_view = host_buffer_view;
     }
 
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 403a3d40a..940fdd557 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -65,85 +65,83 @@ label HostMatrixWrapper<MatrixType>::count_interface_nnz(
 
 template <class MatrixType>
 HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
-        const objectRegistry &db, const MatrixType &matrix,
-        // coeffs for cells on boundaries
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        // coeffs for internal cells
-        const FieldField<Field, scalar> &interfaceIntCoeffs,
-        // pointers to interfaces can be used to access concrete
-        // functions such as transferring indices, patch neighbours etc
-        const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls, const word &fieldName)
-        : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
-                             interfaceIntCoeffs, interfaces, solverControls),
-          exec_{db, solverControls, fieldName},
-          device_id_guard_{db, fieldName, exec_.get_device_exec()},
-          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-          scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
-          nrows_(matrix.diag().size()),
-          local_interface_nnz_(count_interface_nnz(interfaces, false)),
-          upper_nnz_(matrix.lduAddr().upperAddr().size()),
-          non_diag_nnz_(2 * upper_nnz_),
-          local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
-          local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
-                                         local_interface_nnz_),
-          local_sparsity_{
-              fieldName + "_local",           db,       exec_,
-              local_matrix_w_interfaces_nnz_, verbose_,
-          },
-          local_coeffs_{
-              fieldName + "_local_coeffs",
-              db,
-              exec_,
-              local_matrix_w_interfaces_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
-          communication_pattern_(create_communication_pattern(interfaces)),
-          non_local_sparsity_{
-              fieldName + "_non_local", db,       exec_,
-              non_local_matrix_nnz_,    verbose_,
-          },
-          non_local_coeffs_{
-              fieldName + "_non_local_coeffs",
-              db,
-              exec_,
-              non_local_matrix_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          permutation_matrix_name_{"PermutationMatrix"},
-          permutation_stored_{
-              db.template foundObject<regIOobject>(permutation_matrix_name_)},
-          P_{(permutation_stored_) ? db.template lookupObjectRef<
-                                           DevicePersistentBase<gko::LinOp>>(
-                                           permutation_matrix_name_)
-                                         .get_ptr()
-                                   : nullptr}
-    {
-        if (!local_sparsity_.col_idxs_.get_stored() ||
-            local_sparsity_.col_idxs_.get_update()) {
-            TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
-                                this->fieldName(),
-                                init_local_sparsity_pattern(interfaces);)
-            TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
-                                this->fieldName(),
-                                init_non_local_sparsity_pattern(interfaces);)
-        }
-        if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
-            TIME_WITH_FIELDNAME(
-                verbose_, update_local_matrix_data, this->fieldName(),
-                update_local_matrix_data(interfaces, interfaceBouCoeffs);)
-            TIME_WITH_FIELDNAME(
-                verbose_, update_non_local_matrix_data, this->fieldName(),
-                update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
-        }
+    const objectRegistry &db, const MatrixType &matrix,
+    // coeffs for cells on boundaries
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    // coeffs for internal cells
+    const FieldField<Field, scalar> &interfaceIntCoeffs,
+    // pointers to interfaces can be used to access concrete
+    // functions such as transferring indices, patch neighbours etc
+    const lduInterfaceFieldPtrsList &interfaces,
+    const dictionary &solverControls, const word &fieldName)
+    : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
+                         interfaceIntCoeffs, interfaces, solverControls),
+      exec_{db, solverControls, fieldName},
+      device_id_guard_{db, fieldName, exec_.get_device_exec()},
+      verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
+      nrows_(matrix.diag().size()),
+      local_interface_nnz_(count_interface_nnz(interfaces, false)),
+      upper_nnz_(matrix.lduAddr().upperAddr().size()),
+      non_diag_nnz_(2 * upper_nnz_),
+      local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
+      local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
+      local_sparsity_{
+          fieldName + "_local",           db,       exec_,
+          local_matrix_w_interfaces_nnz_, verbose_,
+      },
+      local_coeffs_{
+          fieldName + "_local_coeffs",
+          db,
+          exec_,
+          local_matrix_w_interfaces_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
+      communication_pattern_(create_communication_pattern(interfaces)),
+      non_local_sparsity_{
+          fieldName + "_non_local", db, exec_, non_local_matrix_nnz_, verbose_,
+      },
+      non_local_coeffs_{
+          fieldName + "_non_local_coeffs",
+          db,
+          exec_,
+          non_local_matrix_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      permutation_matrix_name_{"PermutationMatrix"},
+      permutation_stored_{
+          db.template foundObject<regIOobject>(permutation_matrix_name_)},
+      P_{(permutation_stored_)
+             ? db.template lookupObjectRef<DevicePersistentBase<gko::LinOp>>(
+                     permutation_matrix_name_)
+                   .get_ptr()
+             : nullptr}
+{
+    if (!local_sparsity_.col_idxs_.get_stored() ||
+        local_sparsity_.col_idxs_.get_update()) {
+        TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
+                            this->fieldName(),
+                            init_local_sparsity_pattern(interfaces);)
+        TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
+                            this->fieldName(),
+                            init_non_local_sparsity_pattern(interfaces);)
     }
+    if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
+        TIME_WITH_FIELDNAME(
+            verbose_, update_local_matrix_data, this->fieldName(),
+            update_local_matrix_data(interfaces, interfaceBouCoeffs);)
+        TIME_WITH_FIELDNAME(
+            verbose_, update_non_local_matrix_data, this->fieldName(),
+            update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
+    }
+}
 
 
 template <class MatrixType>
@@ -246,10 +244,10 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
         label interface_size = interface_cells.size();
         auto exec = exec_.get_ref_exec();
         send_idxs.push_back(std::pair<gko::index_set<label>, label>(
-            gko::index_set(
-                exec, interface_size,
-                gko::array<label>::view(exec, interface_size, interface_cells.data()),
-                true),
+            gko::index_set(exec, interface_size,
+                           gko::array<label>::view(exec, interface_size,
+                                                   interface_cells.data()),
+                           true),
             proc));
     }
 
@@ -692,16 +690,15 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     i_device_view = tmp_contiguous_iface;
 }
 
-template 
-HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
-        const objectRegistry &db, const lduMatrix &matrix,
-        // coeffs for cells on boundaries
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        // coeffs for internal cells
-        const FieldField<Field, scalar> &interfaceIntCoeffs,
-        // pointers to interfaces can be used to access concrete
-        // functions such as transferring indices, patch neighbours etc
-        const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls, const word &fieldName);
+template HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
+    const objectRegistry &db, const lduMatrix &matrix,
+    // coeffs for cells on boundaries
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    // coeffs for internal cells
+    const FieldField<Field, scalar> &interfaceIntCoeffs,
+    // pointers to interfaces can be used to access concrete
+    // functions such as transferring indices, patch neighbours etc
+    const lduInterfaceFieldPtrsList &interfaces,
+    const dictionary &solverControls, const word &fieldName);
 
 }  // namespace Foam
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 34e781cc7..91d914107 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -39,7 +39,7 @@ SourceFiles
 namespace Foam {
 
 
-    // TODO Move this somewhere else
+// TODO Move this somewhere else
 struct PersistentSparsityPattern {
     PersistentSparsityPattern(
       const word &fieldName,
@@ -181,16 +181,17 @@ private:
     ** @param interfaces The list of interfaces for the search
     ** @param proc_interfaces Count only elements on (true)
     *processorLduInterfaces or exclude processorLduInterfaces (false)
-     ** TODO make private and no discard
+    ** TODO make private and no discard
     */
     label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
                               bool proc_interfaces) const;
 
-     /** Iterates all local interfaces and returns the relative order and
-     **corresponding row and column indices
-     **
-     ** @return vector of tuples containing the interface number, the local row, the local column
-     **/
+    /** Iterates all local interfaces and returns the relative order and
+    **corresponding row and column indices
+    **
+    ** @return vector of tuples containing the interface number, the local row,
+    *the local column
+    **/
     std::vector<std::tuple<label, label, label>>
     collect_local_interface_indices(
         const lduInterfaceFieldPtrsList &interfaces_) const;
@@ -318,8 +319,7 @@ public:
         const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
     // getter
-    public:
-
+public:
     bool get_verbose() const { return verbose_; }
 
     /** Getter for the CommunicationPattern
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 3da54f949..427a58ca9 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -68,9 +68,7 @@ public:
     std::unique_ptr<cg::Factory, std::default_delete<cg::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        auto cg = cg::build().with_criteria(
-                stoppingCriterionVec_
-                ).on(exec);
+        auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
         return cg;
     };
 
@@ -79,9 +77,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto cg = cg::build()
-                      .with_criteria(
-                              stoppingCriterionVec_
-                              )
+                      .with_criteria(stoppingCriterionVec_)
                       .with_generated_preconditioner(precond)
                       .on(exec);
         return cg;
diff --git a/common/common.C b/common/common.C
index ed9846fe6..d7ace7281 100644
--- a/common/common.C
+++ b/common/common.C
@@ -62,9 +62,9 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
                 const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
-    //std::filesystem::create_directories(folder);
+    // std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
-    std::cout  << "exporting " << fn <<  std::endl;
+    std::cout << "exporting " << fn << std::endl;
     std::ofstream stream{fn};
     stream << std::setprecision(15);
     if (matrixFormat == "Coo") {
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 813c84f7a..9114a532f 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -330,7 +330,8 @@ public:
         //     delta_t_solve / max(this->get_number_of_iterations(), 1);
         // scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
         // this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
-        //     this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1, 0);
+        //     this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1,
+        //     0);
         // this->set_prev_rel_res_cost(prev_rel_res_cost);
         // auto time_per_iter_and_dof =
         //     time_per_iter * 1000.0 / partition.get_total_size();

From 235c3c60fecc2dada22d5b0b61dfd5a44e44deef Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 10:25:52 +0100
Subject: [PATCH 024/125] format

---
 common/common.H | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/common/common.H b/common/common.H
index 5100477e6..5efbe9430 100644
--- a/common/common.H
+++ b/common/common.H
@@ -61,8 +61,9 @@ namespace Foam {
     }
 
 
-#define IN_USE_DO_NOT_REMOVE(FUNC) \
-std::cout << FUNC <<  " in "   << __FILENAME__ << ":" <<__LINE__ << " is in use do not remove"  << std::endl
+#define IN_USE_DO_NOT_REMOVE(FUNC)                                 \
+    std::cout << FUNC << " in " << __FILENAME__ << ":" << __LINE__ \
+              << " is in use do not remove" << std::endl
 
 #define LOG_0(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 0, MSG, 0)
 #define LOG_1(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 1, MSG, 0)

From 41cf412ab272cb8cd428078b8329b556326bc075 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 10:27:15 +0100
Subject: [PATCH 025/125] fix typo

---
 HostMatrix/HostMatrix.C | 2 +-
 HostMatrix/HostMatrix.H | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 940fdd557..5b69527a5 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -223,7 +223,7 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
 
     // reduce vector of sizes
     // this assumes that a rank can be connected to the same neighbour through
-    // different interfaces this migth not be necessary
+    // different interfaces this might not be necessary
     label n_procs = 0;
     std::map<label, label> reduce_map{};
     for (auto [proc, n_faces] : neighbour_procs) {
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 91d914107..12ff6bb18 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -105,7 +105,7 @@ struct CommunicationPattern {
  * It mainly generates:
  * - sparsity pattern/device_matrix_data of the local_matrix and
  * - non_local_matrix (with global columns)
- * - CommuncationPattern
+ * - CommunicationPattern
  * */
 template <class MatrixType>
 class HostMatrixWrapper : public MatrixType::solver {

From 63b1360148e301ed611bf07cb087ad1b77af12df Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 7 Nov 2023 14:48:18 +0100
Subject: [PATCH 026/125] Move ctr to .C file

---
 HostMatrix/HostMatrix.C | 241 ++++++++++++++++++++++------------------
 HostMatrix/HostMatrix.H |  83 ++------------
 2 files changed, 142 insertions(+), 182 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 5b69527a5..77799a638 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -27,6 +27,7 @@ SourceFiles
 
 #include "cyclicFvPatchField.H"
 #include "lduMatrix.H"
+#include "processorFvPatch.H"
 
 
 namespace Foam {
@@ -144,6 +145,65 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
 }
 
 
+template <class MatrixType>
+HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
+    const objectRegistry &db, const MatrixType &matrix,
+    const dictionary &solverControls, const word &fieldName)
+    : MatrixType::solver(fieldName, matrix, solverControls),
+      exec_{db, solverControls, fieldName},
+      device_id_guard_{db, fieldName, exec_.get_device_exec()},
+      verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
+      nrows_(matrix.diag().size()),
+      local_interface_nnz_(0),
+      upper_nnz_(matrix.lduAddr().upperAddr().size()),
+      non_diag_nnz_(2 * upper_nnz_),
+      local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
+      local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
+      local_sparsity_{
+          fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
+      },
+      local_coeffs_{
+          fieldName + "_coeffs",
+          db,
+          exec_,
+          local_matrix_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      non_local_matrix_nnz_(),
+      // proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
+      non_local_sparsity_{
+          fieldName + "_non_local", db, exec_, non_local_matrix_nnz_, verbose_,
+      },
+      non_local_coeffs_{
+          fieldName + "_non_local_coeffs",
+          db,
+          exec_,
+          non_local_matrix_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      permutation_matrix_name_{"PermutationMatrix"},
+      permutation_stored_{
+          db.template foundObject<regIOobject>(permutation_matrix_name_)},
+      P_{(permutation_stored_)
+             ? db.template lookupObjectRef<DevicePersistentBase<gko::LinOp>>(
+                     permutation_matrix_name_)
+                   .get_ptr()
+             : nullptr}
+{
+        FatalErrorInFunction
+           << "This constructor is currently not implemented" 
+           << abort(FatalError);
+
+}
+
+
 template <class MatrixType>
 std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
     const lduInterfaceFieldPtrsList &interfaces,
@@ -174,6 +234,24 @@ std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
     return ret;
 }
 
+template <class Sel, class Func>
+void interface_iterator(const lduInterfaceFieldPtrsList &interfaces, Func func)
+{
+    label element_ctr = 0;
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        if (isA<Sel>(iface->interface())) {
+            func(element_ctr, interface_size, iface);
+        }
+    }
+}
+
 
 template <class MatrixType>
 CommunicationPattern
@@ -186,17 +264,11 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     // temp map, mapping from neighbour rank interface cells
     std::map<label, std::vector<label>> interface_cell_map{};
 
-    // TODO FIXME use a lambda to avoid repeating
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto &iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        if (isA<processorLduInterface>(iface->interface())) {
+    //
+    interface_iterator<processorFvPatch>(
+        interfaces,
+        [&](label, const label interface_size, const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
             const processorLduInterface &pldui =
                 refCast<const processorLduInterface>(iface->interface());
             const label neighbProcNo = pldui.neighbProcNo();
@@ -204,22 +276,20 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
             neighbour_procs.push_back(
                 std::pair<label, label>{neighbProcNo, interface_size});
 
-            //
             auto search = interface_cell_map.find(neighbProcNo);
             if (search == interface_cell_map.end()) {
                 interface_cell_map.insert(std::pair{
                     neighbProcNo,
                     std::vector<label>(face_cells.begin(), face_cells.end())});
             } else {
-                // TODO FIXME reserve size
                 auto cur_face_cells = interface_cell_map[neighbProcNo];
+                cur_face_cells.reserve(interface_size);
                 for (auto face_cell : face_cells) {
                     cur_face_cells.push_back(face_cell);
                 }
                 interface_cell_map[neighbProcNo] = cur_face_cells;
             }
-        }
-    }
+        });
 
     // reduce vector of sizes
     // this assumes that a rank can be connected to the same neighbour through
@@ -236,6 +306,7 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
         }
     }
 
+
     // create index_sets
     // currently this assumes that there is only one interface to a given
     // neighbour rank
@@ -262,6 +333,12 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
         iter++;
     }
 
+    for (int n = 0; n < n_procs; n++) {
+        std::cout << Pstream::myProcNo()
+                  << "target_ids:" << target_ids.get_data()[n] << "target_size "
+                  << target_sizes.get_data()[n] << std::endl;
+    }
+
     return CommunicationPattern{target_ids, target_sizes, send_idxs};
 }
 
@@ -274,22 +351,13 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
     std::vector<std::tuple<label, label, label>> local_interface_idxs{};
     local_interface_idxs.reserve(local_interface_nnz_);
 
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
 
-        // TODO make this a separate specialized function
-        label interface_ctr = 0;
-        if (isA<cyclicLduInterface>(iface->interface())) {
+    interface_iterator<cyclicLduInterface>(
+        interfaces, [&](label &element_ctr, label interface_size,
+                        const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
             const cyclicLduInterface &pldui =
                 refCast<const cyclicLduInterface>(iface->interface());
-            // const labelUList &rows = this->matrix().lduAddr().patchAddr(i);
 
 #ifdef WITH_ESI_VERSION
             const label neighbPatchId = pldui.neighbPatchID();
@@ -299,47 +367,33 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
             const labelUList &cols =
                 this->matrix().lduAddr().patchAddr(neighbPatchId);
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                // DEBUG
-                // cout << "patch_ctr " << cellI << " ictr " << interface_ctr
-                //     << " patchId " << i << " neighPatchId " << neighbPatchId
-                //      << " row " << rows[cellI] << " col " << cols[cellI]
-                //      << "\n" << endl;
                 local_interface_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], cols[cellI]});
-                interface_ctr += 1;
+                    {element_ctr, face_cells[cellI], cols[cellI]});
+                element_ctr += 1;
             }
-        }
-    }
+        });
     return local_interface_idxs;
 }
 
 template <class MatrixType>
-std::vector<std::tuple<label, label, label>>
-HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
+std::vector<std::tuple<label, label>>
+HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // vector of local cell ids on other side
-    std::vector<std::tuple<label, label, label>> non_local_idxs{};
+    std::vector<std::tuple<label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
-    label interface_ctr = 0;
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        if (isA<processorLduInterface>(iface->interface())) {
+    interface_iterator<processorLduInterface>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                non_local_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], interface_ctr});
-                interface_ctr += 1;
+                non_local_idxs.push_back({element_ctr, face_cells[cellI]});
+                element_ctr += 1;
             }
-        }
-    }
+        });
+
     word msg = "done collecting neighbouring processor cell id";
 
     LOG_2(verbose_, msg)
@@ -350,44 +404,30 @@ template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
-    auto non_local_row_indices = collect_non_local_col_indices(interfaces);
+    auto non_local_row_indices = collect_cells_on_interface(interfaces);
     auto rows = non_local_sparsity_.row_idxs_.get_data();
     auto cols = non_local_sparsity_.col_idxs_.get_data();
     auto permute = non_local_sparsity_.ldu_mapping_.get_data();
 
     std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
               [&](const auto &a, const auto &b) {
-                  auto [interface_idx_a, row_a, col_a] = a;
-                  auto [interface_idx_b, row_b, col_b] = b;
-                  return std::tie(row_a, col_a) < std::tie(row_b, col_b);
+                  auto [interface_idx_a, row_a] = a;
+                  auto [interface_idx_b, row_b] = b;
+                  return std::tie(row_a, interface_idx_a) <
+                         std::tie(row_b, interface_idx_b);
               });
 
-    label interface_ctr = 0;
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto &iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        if (isA<processorLduInterface>(iface->interface())) {
-            // const processorLduInterface &pldui =
-            //     refCast<const processorLduInterface>(iface->interface());
-
-            // check if current cell is on the current patch
-            // NOTE cells can be several times on same patch
+    interface_iterator<processorLduInterface>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const lduInterfaceField *) {
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto [interface_idx, row, col] =
-                    non_local_row_indices[interface_ctr];
-                rows[interface_ctr] = row;
-                cols[interface_ctr] = col;
-                permute[interface_ctr] = interface_idx;
-                interface_ctr += 1;
+                auto [interface_idx, row] = non_local_row_indices[element_ctr];
+                rows[element_ctr] = row;
+                cols[element_ctr] = interface_idx;
+                permute[element_ctr] = interface_idx;
+                element_ctr += 1;
             }
-        }
-    }
+        });
 }
 
 template <class MatrixType>
@@ -542,7 +582,6 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
         // post insert if local interfaces were consumed but stuff remains in
         // rows_copy and cols_copy
         for (int i = total_ctr; i < local_matrix_w_interfaces_nnz_; i++) {
-            // std::cout << "insert missing values " << i << endl;
             rows[i] = rows_copy[current_idx_ctr];
             cols[i] = cols_copy[current_idx_ctr];
             permute[i] = permute_copy[current_idx_ctr];
@@ -593,10 +632,6 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
                     value = diag[pos - upper_nnz_];
                 }
                 if (pos >= upper_nnz_ + diag_nnz) {
-                    // std::cout <<  "idx ( "
-                    // << local_sparsity_.row_idxs_.get_data()[i] << ", "
-                    // << local_sparsity_.col_idxs_.get_data()[i] << "): "
-                    // << value << "\n" ;
                     value = -couple_coeffs[pos - upper_nnz_ - diag_nnz];
                 }
 
@@ -655,7 +690,6 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     const FieldField<Field, scalar> &interfaceBouCoeffs) const
 {
     auto ref_exec = exec_.get_ref_exec();
-
     auto interface_coeffs =
         collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
     auto permute = non_local_sparsity_.ldu_mapping_.get_data();
@@ -665,24 +699,15 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
         gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
     auto contiguous_iface = tmp_contiguous_iface.get_data();
 
-    label interface_ctr{0};
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-        const auto iface{interface_getter(interfaces, i)};
-        const label patch_size = iface->interface().faceCells().size();
-
-        if (!isA<processorLduInterface>(iface->interface())) {
-            continue;
-        }
-
-        for (label cellI = 0; cellI < patch_size; cellI++) {
-            contiguous_iface[interface_ctr] =
-                -interface_coeffs[permute[interface_ctr]];
-            interface_ctr += 1;
-        }
-    }
+    interface_iterator<processorLduInterface>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const lduInterfaceField *) {
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                contiguous_iface[element_ctr] =
+                    -interface_coeffs[permute[element_ctr]];
+                element_ctr++;
+            }
+        });
 
     // copy to persistent
     auto i_device_view = gko::array<scalar>::view(
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 12ff6bb18..9997268e1 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -39,7 +39,6 @@ SourceFiles
 namespace Foam {
 
 
-// TODO Move this somewhere else
 struct PersistentSparsityPattern {
     PersistentSparsityPattern(
       const word &fieldName,
@@ -181,7 +180,6 @@ private:
     ** @param interfaces The list of interfaces for the search
     ** @param proc_interfaces Count only elements on (true)
     *processorLduInterfaces or exclude processorLduInterfaces (false)
-    ** TODO make private and no discard
     */
     label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
                               bool proc_interfaces) const;
@@ -203,22 +201,18 @@ private:
      **
      ** @param interfaces The list of interfaces for the search
      ** @return the CommunicationPattern
-     ** TODO make private and no discard
      */
     CommunicationPattern create_communication_pattern(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
-    /* Iterates all interfaces and collects non-local global column indices
-    **
-    ** TODO FIXME this is not uptodate
-    ** Non-local global column indices are computed from the global cell-id
-    ** (row) on the other side of an interface
-    **
+    /** Iterates all interfaces and collect the corresponding cell id (row)
+    ** and a unique counter
+    ** 
     ** @return vector off size nnz_non_local, with sections for each interface
-    ** ret = [(1,2,3),(10,20,30), ...]
-    **         i0      i1,       i...
+    ** ret = [(1,2),(2,20), ...]
+    **         i0   i1,   i...
     */
-    std::vector<std::tuple<label, label, label>> collect_non_local_col_indices(
+    std::vector<std::tuple<label, label>> collect_cells_on_interface(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
     /* Based on OpenFOAMs ldu matrix format this function computes two
@@ -253,60 +247,7 @@ public:
     // TODO reimplement this at some point
     // coupled  wrapper constructor
     HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
-                      const dictionary &solverControls, const word &fieldName)
-        : MatrixType::solver(fieldName, matrix, solverControls),
-          exec_{db, solverControls, fieldName},
-          device_id_guard_{db, fieldName, exec_.get_device_exec()},
-          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-          scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
-          nrows_(matrix.diag().size()),
-          local_interface_nnz_(0),
-          upper_nnz_(matrix.lduAddr().upperAddr().size()),
-          non_diag_nnz_(2 * upper_nnz_),
-          local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
-          local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
-                                         local_interface_nnz_),
-          local_sparsity_{
-              fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
-          },
-          local_coeffs_{
-              fieldName + "_coeffs",
-              db,
-              exec_,
-              local_matrix_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          non_local_matrix_nnz_(),
-          // proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
-          non_local_sparsity_{
-              fieldName + "_non_local", db,       exec_,
-              non_local_matrix_nnz_,    verbose_,
-          },
-          non_local_coeffs_{
-              fieldName + "_non_local_coeffs",
-              db,
-              exec_,
-              non_local_matrix_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          permutation_matrix_name_{"PermutationMatrix"},
-          permutation_stored_{
-              db.template foundObject<regIOobject>(permutation_matrix_name_)},
-          P_{(permutation_stored_) ? db.template lookupObjectRef<
-                                           DevicePersistentBase<gko::LinOp>>(
-                                           permutation_matrix_name_)
-                                         .get_ptr()
-                                   : nullptr}
-    {
-        // TODO reimplement this
-    }
-
+                      const dictionary &solverControls, const word &fieldName);
 
     // updater
 
@@ -319,13 +260,10 @@ public:
         const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
     // getter
+
 public:
     bool get_verbose() const { return verbose_; }
 
-    /** Getter for the CommunicationPattern
-     **
-     ** @return the CommunicationPattern
-     **/
     const CommunicationPattern &get_communication_pattern() const
     {
         return communication_pattern_;
@@ -335,10 +273,7 @@ public:
 
     const ExecutorHandler &get_exec_handler() const { return exec_; }
 
-    PersistentArray<scalar> &get_local_coeffs() const  //
-    {
-        return local_coeffs_;
-    };
+    PersistentArray<scalar> &get_local_coeffs() const { return local_coeffs_; };
 
     PersistentArray<label> &get_local_col_idxs() const
     {

From ec020ebda3ae42f0055f8178c256b2d2e5f71b1a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 7 Nov 2023 15:00:08 +0100
Subject: [PATCH 027/125] use concrete types for interfaces

---
 HostMatrix/HostMatrix.C | 50 +++++++++++++++++------------------------
 1 file changed, 21 insertions(+), 29 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 77799a638..8a5bf21d1 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -197,10 +197,8 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
                    .get_ptr()
              : nullptr}
 {
-        FatalErrorInFunction
-           << "This constructor is currently not implemented" 
-           << abort(FatalError);
-
+    FatalErrorInFunction << "This constructor is currently not implemented"
+                         << abort(FatalError);
 }
 
 
@@ -245,9 +243,10 @@ void interface_iterator(const lduInterfaceFieldPtrsList &interfaces, Func func)
         const auto iface{interface_getter(interfaces, i)};
         const auto &face_cells{iface->interface().faceCells()};
         const label interface_size = face_cells.size();
+        const Sel &patch = refCast<const Sel>(iface->interface());
 
         if (isA<Sel>(iface->interface())) {
-            func(element_ctr, interface_size, iface);
+            func(element_ctr, interface_size, patch, iface);
         }
     }
 }
@@ -267,11 +266,10 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     //
     interface_iterator<processorFvPatch>(
         interfaces,
-        [&](label, const label interface_size, const lduInterfaceField *iface) {
+        [&](label, const label interface_size, const processorFvPatch &patch,
+            const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
-            const processorLduInterface &pldui =
-                refCast<const processorLduInterface>(iface->interface());
-            const label neighbProcNo = pldui.neighbProcNo();
+            const label neighbProcNo = patch.neighbProcNo();
 
             neighbour_procs.push_back(
                 std::pair<label, label>{neighbProcNo, interface_size});
@@ -333,12 +331,6 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
         iter++;
     }
 
-    for (int n = 0; n < n_procs; n++) {
-        std::cout << Pstream::myProcNo()
-                  << "target_ids:" << target_ids.get_data()[n] << "target_size "
-                  << target_sizes.get_data()[n] << std::endl;
-    }
-
     return CommunicationPattern{target_ids, target_sizes, send_idxs};
 }
 
@@ -352,17 +344,16 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
     local_interface_idxs.reserve(local_interface_nnz_);
 
 
-    interface_iterator<cyclicLduInterface>(
-        interfaces, [&](label &element_ctr, label interface_size,
-                        const lduInterfaceField *iface) {
+    interface_iterator<cyclicFvPatch>(
+        interfaces,
+        [&](label &element_ctr, label interface_size,
+            const cyclicFvPatch &patch, const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
-            const cyclicLduInterface &pldui =
-                refCast<const cyclicLduInterface>(iface->interface());
 
 #ifdef WITH_ESI_VERSION
-            const label neighbPatchId = pldui.neighbPatchID();
+            const label neighbPatchId = patch.neighbPatchID();
 #else
-            const label neighbPatchId = pldui.nbrPatchID();
+            const label neighbPatchId = patch.nbrPatchID();
 #endif
             const labelUList &cols =
                 this->matrix().lduAddr().patchAddr(neighbPatchId);
@@ -384,9 +375,10 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     std::vector<std::tuple<label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
-    interface_iterator<processorLduInterface>(
-        interfaces, [&](label &element_ctr, const label interface_size,
-                        const lduInterfaceField *iface) {
+    interface_iterator<processorFvPatch>(
+        interfaces,
+        [&](label &element_ctr, const label interface_size,
+            const processorFvPatch &, const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
             for (label cellI = 0; cellI < interface_size; cellI++) {
                 non_local_idxs.push_back({element_ctr, face_cells[cellI]});
@@ -417,9 +409,9 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
                          std::tie(row_b, interface_idx_b);
               });
 
-    interface_iterator<processorLduInterface>(
+    interface_iterator<processorFvPatch>(
         interfaces, [&](label &element_ctr, const label interface_size,
-                        const lduInterfaceField *) {
+                        const processorFvPatch &, const lduInterfaceField *) {
             for (label cellI = 0; cellI < interface_size; cellI++) {
                 auto [interface_idx, row] = non_local_row_indices[element_ctr];
                 rows[element_ctr] = row;
@@ -699,9 +691,9 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
         gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
     auto contiguous_iface = tmp_contiguous_iface.get_data();
 
-    interface_iterator<processorLduInterface>(
+    interface_iterator<processorFvPatch>(
         interfaces, [&](label &element_ctr, const label interface_size,
-                        const lduInterfaceField *) {
+                        const processorFvPatch &, const lduInterfaceField *) {
             for (label cellI = 0; cellI < interface_size; cellI++) {
                 contiguous_iface[element_ctr] =
                     -interface_coeffs[permute[element_ctr]];

From 3d8572b6f950d2717df066641c0a7ec4815759b2 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 8 Nov 2023 07:11:40 +0100
Subject: [PATCH 028/125] clean-up

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 91fc541e6..e3fd5feb3 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -225,25 +225,6 @@ struct MatrixInitFunctor {
         auto send_sizes = sparse_comm->get_send_sizes();
         auto send_offs = sparse_comm->get_send_offsets();
 
-        if (Pstream::master()) {
-            for (auto e : recv_sizes) {
-                std::cout << "recv_size: " << e << std::endl;
-            }
-
-            for (auto e : recv_offs) {
-                std::cout << "recv_offs: " << e << std::endl;
-            }
-
-            for (auto e : send_sizes) {
-                std::cout << "send_sizes: " << e << std::endl;
-            }
-
-            for (auto e : send_offs) {
-                std::cout << "send_offs: " << e << std::endl;
-            }
-        }
-
-
         dist_A->read_distributed(A_data, non_local_A_data, sparse_comm);
 
         auto device_mat = generate_dist_mtx_with_inner_type(

From 109f6bf34c61a5a833009e4dc21ac75f363fd807 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 8 Nov 2023 07:12:51 +0100
Subject: [PATCH 029/125] add fs creation

---
 common/common.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/common.C b/common/common.C
index d7ace7281..a79cecc8a 100644
--- a/common/common.C
+++ b/common/common.C
@@ -62,7 +62,7 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
                 const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
-    // std::filesystem::create_directories(folder);
+    std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
     std::cout << "exporting " << fn << std::endl;
     std::ofstream stream{fn};

From 62c3ae5a01b09a67d5e8af5e3db7a8724aec6a83 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 8 Nov 2023 08:15:19 +0100
Subject: [PATCH 030/125] Fixup pass stopping vector

---
 Solver/BiCGStab/GKOBiCGStab.H | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Solver/BiCGStab/GKOBiCGStab.H b/Solver/BiCGStab/GKOBiCGStab.H
index 2cabd288c..797cb8958 100644
--- a/Solver/BiCGStab/GKOBiCGStab.H
+++ b/Solver/BiCGStab/GKOBiCGStab.H
@@ -73,7 +73,7 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto bicgstab = gko::solver::Bicgstab<scalar>::build()
-                            // .with_criteria(stoppingCriterionVec_)
+                            .with_criteria(stoppingCriterionVec_)
                             .on(exec);
         return bicgstab;
     };
@@ -83,7 +83,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto bicgstab = bicgstab::build()
-                            // .with_criteria(stoppingCriterionVec_)
+                            .with_criteria(stoppingCriterionVec_)
                             .with_generated_preconditioner(precond)
                             .on(exec);
         return bicgstab;

From a344ef35794b320b053cc637f70bc6895768fc5b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 8 Nov 2023 10:44:00 +0100
Subject: [PATCH 031/125] add stopping crit vect to GMRES, format

---
 HostMatrix/HostMatrix.H | 2 +-
 Solver/GMRES/GKOGMRES.H | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 9997268e1..96bb5de58 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -207,7 +207,7 @@ private:
 
     /** Iterates all interfaces and collect the corresponding cell id (row)
     ** and a unique counter
-    ** 
+    **
     ** @return vector off size nnz_non_local, with sections for each interface
     ** ret = [(1,2),(2,20), ...]
     **         i0   i1,   i...
diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 7e6497627..4d3b253af 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -70,7 +70,7 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto gmres = gmres::build()
-                         // .with_criteria(stoppingCriterionVec_)
+                         .with_criteria(stoppingCriterionVec_)
                          .on(exec);
         return gmres;
     };
@@ -80,7 +80,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto gmres = gmres::build()
-                         // .with_criteria(stoppingCriterionVec_)
+                         .with_criteria(stoppingCriterionVec_)
                          .with_generated_preconditioner(precond)
                          .on(exec);
         return gmres;

From 26138ee4e2a23aa2512cc7fd3d7ff547f87c791e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 13:55:59 +0100
Subject: [PATCH 032/125] pass CommunicationPattern, move free functions to
 separate TU

---
 CMakeLists.txt                         |   1 +
 DevicePersistent/Partition/Partition.H |  21 ++--
 HostMatrix/HostMatrix.C                | 137 ++++++++-----------------
 HostMatrix/HostMatrix.H                |  81 ++++++++++++---
 HostMatrix/HostMatrixFreeFunctions.C   | 117 +++++++++++++++++++++
 lduLduBase/lduLduBase.H                |   7 +-
 6 files changed, 236 insertions(+), 128 deletions(-)
 create mode 100644 HostMatrix/HostMatrixFreeFunctions.C

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 48033155e..4c4564bc1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -151,6 +151,7 @@ target_sources(
           BaseWrapper/lduBase/GKOlduBase.C
           BaseWrapper/CoupledLduBase/GKOCoupledLduBase.C
           HostMatrix/HostMatrix.C
+          HostMatrix/HostMatrixFreeFunctions.C
           Solver/CG/GKOCG.C
           Solver/BiCGStab/GKOBiCGStab.C
 #Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 64cba12d4..d83de9894 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -31,6 +31,7 @@ SourceFiles
 
 #include <ginkgo/ginkgo.hpp>
 
+#include "HostMatrix/HostMatrix.H"
 #include "DevicePersistent/Base/Base.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "common/common.H"
@@ -55,21 +56,20 @@ struct PartitionInitFunctor {
     // Array storing the ranks of the neighbouring processors
     const gko::array<label> target_sizes_;
 
-    const std::vector<std::pair<gko::index_set<label>, label>> send_idxs_;
+    const std::vector<std::pair<gko::array<label>, label>> send_idxs_;
 
     PartitionInitFunctor(
         const ExecutorHandler &exec, const label local_size,
         const label verbose, const int ranks_per_gpu,
-        const gko::array<label> &target_ids,
-        const gko::array<label> &target_sizes,
-        const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
+        const CommunicationPattern& communication_pattern
+        )
         : exec_(exec),
           local_size_(local_size),
           verbose_(verbose),
           ranks_per_gpu_(ranks_per_gpu),
-          target_ids_(target_ids),
-          target_sizes_(target_sizes),
-          send_idxs_(send_idxs)
+          target_ids_(communication_pattern.target_ids),
+          target_sizes_(communication_pattern.target_sizes),
+          send_idxs_(communication_pattern.send_idxs)
     {}
 
     void update(std::shared_ptr<part_type> persistent_partition) const
@@ -128,13 +128,12 @@ public:
     PersistentPartition(
         const objectRegistry &db, const ExecutorHandler &exec,
         const label verbose, const int ranks_per_gpu, const label elements,
-        const gko::array<label> &target_ids,
-        const gko::array<label> &target_sizes,
-        const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
+        const CommunicationPattern &communication_pattern
+        )
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, elements, verbose, ranks_per_gpu,
-                                   target_ids, target_sizes, send_idxs),
+                                   communication_pattern),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           local_elements_(elements),
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 8a5bf21d1..d5c16ffd7 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -32,38 +32,6 @@ SourceFiles
 
 namespace Foam {
 
-const lduInterfaceField *interface_getter(
-    const lduInterfaceFieldPtrsList &interfaces, const label i)
-{
-#ifdef WITH_ESI_VERSION
-    return interfaces.get(i);
-#else
-    return interfaces.operator()(i);
-#endif
-}
-
-template <class MatrixType>
-label HostMatrixWrapper<MatrixType>::count_interface_nnz(
-    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
-{
-    label ctr{0};
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-        const auto iface{interface_getter(interfaces, i)};
-
-        bool count = (proc_interfaces)
-                         ? !!isA<processorLduInterface>(iface->interface())
-                         : !isA<processorLduInterface>(iface->interface());
-        if (count) {
-            ctr += iface->interface().faceCells().size();
-        }
-    }
-    return ctr;
-}
-
-
 template <class MatrixType>
 HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
     const objectRegistry &db, const MatrixType &matrix,
@@ -80,6 +48,8 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
       exec_{db, solverControls, fieldName},
       device_id_guard_{db, fieldName, exec_.get_device_exec()},
       verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      reorder_on_copy_(
+          solverControls.lookupOrDefault<Switch>("reorderOnHost", true)),
       scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
       nrows_(matrix.diag().size()),
       local_interface_nnz_(count_interface_nnz(interfaces, false)),
@@ -153,6 +123,8 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
       exec_{db, solverControls, fieldName},
       device_id_guard_{db, fieldName, exec_.get_device_exec()},
       verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      reorder_on_copy_(
+          solverControls.lookupOrDefault<Switch>("reorderOnHost", true)),
       scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
       nrows_(matrix.diag().size()),
       local_interface_nnz_(0),
@@ -308,21 +280,20 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     // create index_sets
     // currently this assumes that there is only one interface to a given
     // neighbour rank
-    std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
+    std::vector<std::pair<gko::array<label>, label>> send_idxs;
     for (auto [proc, interface_cells] : interface_cell_map) {
-        label interface_size = interface_cells.size();
         auto exec = exec_.get_ref_exec();
-        send_idxs.push_back(std::pair<gko::index_set<label>, label>(
-            gko::index_set(exec, interface_size,
-                           gko::array<label>::view(exec, interface_size,
-                                                   interface_cells.data()),
-                           true),
+        send_idxs.push_back(std::pair<gko::array<label>, label>(
+            gko::array<label>(exec, interface_cells.begin(),
+                              interface_cells.end()),
             proc));
     }
 
     // convert to gko::array
-    gko::array<label> target_ids{exec_.get_ref_exec(), n_procs};
-    gko::array<label> target_sizes{exec_.get_ref_exec(), n_procs};
+    gko::array<label> target_ids{exec_.get_ref_exec(),
+                                 static_cast<size_t>(n_procs)};
+    gko::array<label> target_sizes{exec_.get_ref_exec(),
+                                   static_cast<size_t>(n_procs)};
 
     label iter = 0;
     for (const auto &[proc, size] : reduce_map) {
@@ -386,7 +357,7 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
             }
         });
 
-    word msg = "done collecting neighbouring processor cell id";
+    word msg = "done collecting neighbouring processor cell ids";
 
     LOG_2(verbose_, msg)
     return non_local_idxs;
@@ -584,6 +555,19 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     LOG_1(verbose_, "done init host matrix")
 }
 
+
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        dense[i] =
+            scale * (pos >= upper_nnz) ? diag[pos - upper_nnz] : upper[pos];
+    }
+}
+
+
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
@@ -611,66 +595,25 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
         auto couple_coeffs =
             collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
         if (is_symmetric) {
-            for (label i = 0; i < local_matrix_w_interfaces_nnz_; ++i) {
-                // where the element is stored in a combined array
-                const label pos{permute[i]};
-                scalar value;
-                // all values up to upper_nnz_ are upper_nnz_ values
-                if (pos < upper_nnz_) {
-                    value = upper[pos];
-                }
-                // all values up to upper_nnz_ are upper_nnz_ values
-                if (pos >= upper_nnz_ && pos < upper_nnz_ + diag_nnz) {
-                    value = diag[pos - upper_nnz_];
-                }
-                if (pos >= upper_nnz_ + diag_nnz) {
-                    value = -couple_coeffs[pos - upper_nnz_ - diag_nnz];
-                }
-
-                dense[i] = scaling_ * value;
-            }
+            symmetric_update_w_interface(
+                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute, scaling_,
+                diag.data(), upper.data(), lower.data(), couple_coeffs.data(),
+                dense);
         } else {
-            for (label i = 0; i < local_matrix_w_interfaces_nnz_; ++i) {
-                const label pos{permute[i]};
-                scalar value;
-                if (pos < upper_nnz_) {
-                    value = upper[pos];
-                }
-                if (pos >= upper_nnz_ && pos < 2 * upper_nnz_) {
-                    value = lower[pos - upper_nnz_];
-                }
-                if (pos >= 2 * upper_nnz_ && pos < 2 * upper_nnz_ + diag_nnz) {
-                    value = diag[pos - 2 * upper_nnz_];
-                }
-                if (pos >= 2 * upper_nnz_ + diag_nnz) {
-                    value = -couple_coeffs[pos - 2 * upper_nnz_ - diag_nnz];
-                }
-                dense[i] = scaling_ * value;
-            }
+            non_symmetric_update_w_interface(
+                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute, scaling_,
+                diag.data(), upper.data(), lower.data(), couple_coeffs.data(),
+                dense);
         }
     } else {
-        // TODO move this to a function fill_non_interfaces
         if (is_symmetric) {
-            for (label i = 0; i < local_matrix_nnz_; ++i) {
-                const label pos{permute[i]};
-                dense[i] = scaling_ * (pos >= upper_nnz_)
-                               ? diag[pos - upper_nnz_]
-                               : upper[pos];
-            }
-            return;
+            symmetric_update(local_matrix_nnz_, upper_nnz_,
+                                          permute, scaling_, diag.data(),
+                                          upper.data(), dense);
         } else {
-            for (label i = 0; i < local_matrix_nnz_; ++i) {
-                const label pos{permute[i]};
-                if (pos < upper_nnz_) {
-                    dense[i] = scaling_ * upper[pos];
-                    continue;
-                }
-                if (pos >= upper_nnz_ && pos < 2 * upper_nnz_) {
-                    dense[i] = scaling_ * lower[pos - upper_nnz_];
-                    continue;
-                }
-                dense[i] = scaling_ * diag[pos - 2 * upper_nnz_];
-            }
+            non_symmetric_update(local_matrix_nnz_, upper_nnz_,
+                                          permute, scaling_, diag.data(),
+                                          upper.data(), lower.data(), dense);
         }
     }
 }
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 96bb5de58..424975e65 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -94,10 +94,40 @@ struct CommunicationPattern {
 
     // send_idx stores the index_set ie which cells
     // are owned by the interface and the corresponding target rank
-    std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
+    std::vector<std::pair<gko::array<label>, label>> send_idxs;
 };
 
 
+// Free functions
+const lduInterfaceField *interface_getter(
+    const lduInterfaceFieldPtrsList &interfaces, const label i);
+
+
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper,
+                      const scalar *lower, scalar *dense);
+
+
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, const label upper_nnz,
+                                  const label *permute, const scalar scale,
+                                  const scalar *diag, const scalar *upper,
+                                  const scalar *lower,
+                                  const scalar *couple_coeffs, scalar *dense);
+
+
+void non_symmetric_update(const label total_nnz, const label upper_nnz,
+                          const label *permute, const scalar scale,
+                          const scalar *diag, const scalar *upper,
+                          const scalar *lower, scalar *dense);
+
+
+void non_symmetric_update_w_interface(
+    const label total_nnz,const label diag_nnz, const label upper_nnz, const label *permute,
+    const scalar scale, const scalar *diag, const scalar *upper,
+    const scalar *lower, const scalar *couple_coeffs, scalar *dense);
+
+
 /* The HostMatrixWrapper class handles the conversion from OpenFOAMs lduMatrix
  * format into Ginkgo array data structures
  *
@@ -117,6 +147,11 @@ private:
 
     const label verbose_;
 
+    // Whether the matrix coefficients should be reordered
+    // during copy or on device
+    // NOTE Not implemented
+    const bool reorder_on_copy_;
+
     // multiply the complete system by this factor, ie sAx=sb
     // NOTE this could be also achieved by just fliping the sign
     const scalar scaling_;
@@ -215,16 +250,41 @@ private:
     std::vector<std::tuple<label, label>> collect_cells_on_interface(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
-    /* Based on OpenFOAMs ldu matrix format this function computes two
-     *consecutive index arrays in row major ordering and scattering indices
-     **
-     */
+    /** Based on OpenFOAMs ldu matrix format this function computes two
+     ** consecutive index arrays (local_sparsisty_.row_idxs and col_idxs) in row
+     ** major ordering and the permutation index (local_sparsity_.ldu_mapping),
+     ** which are required to create to a ginkgo matrix
+     **/
     void init_local_sparsity_pattern(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
+    /** Based on OpenFOAMs interfaces this function computes two
+     ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
+     ** row *major ordering and the permutation index
+     ** (non_local_sparsity_.ldu_mapping), which are required to create a ginkgo
+     ** matrix
+     **/
     void init_non_local_sparsity_pattern(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
+    // updater
+
+    /** Copies the LDU matrix coefficients to local_coeffs without changing or
+     ** reinstantiating the sparsity pattern.
+     ** This uses the local_sparsity_.ldu_mapping to permute the data already
+     ** on the host to be in row major order
+     **/
+    void update_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
+
+    /** Copies the interface matrix coefficients to non_local_coeffs without
+    ** changing or reinstantiating the sparsity pattern.
+    **/
+    void update_non_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
+
 public:
     // segregated wrapper constructor
     // TODO - compute local_interface_nnzs these are matrix coefficient
@@ -245,19 +305,10 @@ public:
         const dictionary &solverControls, const word &fieldName);
 
     // TODO reimplement this at some point
-    // coupled  wrapper constructor
+    // coupled wrapper constructor
     HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
                       const dictionary &solverControls, const word &fieldName);
 
-    // updater
-
-    void update_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
-
-    void update_non_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
     // getter
 
diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
new file mode 100644
index 000000000..0865947c2
--- /dev/null
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -0,0 +1,117 @@
+#include "HostMatrix.H"
+
+const lduInterfaceField *interface_getter(
+    const lduInterfaceFieldPtrsList &interfaces, const label i)
+{
+#ifdef WITH_ESI_VERSION
+    return interfaces.get(i);
+#else
+    return interfaces.operator()(i);
+#endif
+}
+
+
+// free functions - for unit testing
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper,
+                      const scalar *lower, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        dense[i] =
+            scale * (pos >= upper_nnz) ? diag[pos - upper_nnz] : upper[pos];
+    }
+}
+
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, const label upper_nnz,
+                                  const label *permute, const scalar scale,
+                                  const scalar *diag, const scalar *upper,
+                                  const scalar *lower,
+                                  const scalar *couple_coeffs, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        // where the element is stored in a combined array
+        const label pos{permute[i]};
+        scalar value;
+        // all values up to upper_nnz_ are upper_nnz_ values
+        if (pos < upper_nnz) {
+            value = upper[pos];
+        }
+        // all values up to upper_nnz_ are upper_nnz_ values
+        if (pos >= upper_nnz && pos < upper_nnz + diag_nnz) {
+            value = diag[pos - upper_nnz];
+        }
+        if (pos >= upper_nnz + diag_nnz) {
+            value = -couple_coeffs[pos - upper_nnz - diag_nnz];
+        }
+        dense[i] = scale * value;
+    }
+}
+
+void non_symmetric_update_w_interface(
+    const label total_nnz,const label diag_nnz, const label upper_nnz, const label *permute,
+    const scalar scale, const scalar *diag, const scalar *upper,
+    const scalar *lower, const scalar *couple_coeffs, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        scalar value;
+        if (pos < upper_nnz) {
+            value = upper[pos];
+        }
+        if (pos >= upper_nnz && pos < 2 * upper_nnz) {
+            value = lower[pos - upper_nnz];
+        }
+        if (pos >= 2 * upper_nnz && pos < 2 * upper_nnz + diag_nnz) {
+            value = diag[pos - 2 * upper_nnz];
+        }
+        if (pos >= 2 * upper_nnz + diag_nnz) {
+            value = -couple_coeffs[pos - 2 * upper_nnz - diag_nnz];
+        }
+        dense[i] = scale * value;
+    }
+}
+
+
+void non_symmetric_update(const label total_nnz, const label upper_nnz,
+                          const label *permute, const scalar scale,
+                          const scalar *diag, const scalar *upper,
+                          const scalar *lower, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        if (pos < upper_nnz) {
+            dense[i] = scale * upper[pos];
+            continue;
+        }
+        if (pos >= upper_nnz && pos < 2 * upper_nnz) {
+            dense[i] = scale * lower[pos - upper_nnz];
+            continue;
+        }
+        dense[i] = scale * diag[pos - 2 * upper_nnz];
+    }
+}
+
+
+template <class MatrixType>
+label HostMatrixWrapper<MatrixType>::count_interface_nnz(
+    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
+{
+    label ctr{0};
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+
+        bool count = (proc_interfaces)
+                         ? !!isA<processorLduInterface>(iface->interface())
+                         : !isA<processorLduInterface>(iface->interface());
+        if (count) {
+            ctr += iface->interface().faceCells().size();
+        }
+    }
+
+    return ctr;
+}
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 9114a532f..91bb35d4a 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -217,17 +217,14 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
-        const auto communication_pattern = this->get_communication_pattern();
-
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
             verbose_,
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
             psi.size(),  // number of elements
-            communication_pattern.target_ids,
-            communication_pattern.target_sizes,
-            communication_pattern.send_idxs};
+            this->get_communication_pattern()
+        };
 
         // TODO this is now the local matrix part for dist_A
         MatrixWrapper dist_A{db_,

From bd9a2a895e43f8cd3e5c264148bbe5ab855d0972 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 13:58:37 +0100
Subject: [PATCH 033/125] format

---
 HostMatrix/HostMatrixFreeFunctions.C | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index 0865947c2..58f32b259 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -24,10 +24,10 @@ void symmetric_update(const label total_nnz, const label upper_nnz,
     }
 }
 
-void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, const label upper_nnz,
-                                  const label *permute, const scalar scale,
-                                  const scalar *diag, const scalar *upper,
-                                  const scalar *lower,
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
+                                  const label upper_nnz, const label *permute,
+                                  const scalar scale, const scalar *diag,
+                                  const scalar *upper, const scalar *lower,
                                   const scalar *couple_coeffs, scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
@@ -50,9 +50,10 @@ void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, c
 }
 
 void non_symmetric_update_w_interface(
-    const label total_nnz,const label diag_nnz, const label upper_nnz, const label *permute,
-    const scalar scale, const scalar *diag, const scalar *upper,
-    const scalar *lower, const scalar *couple_coeffs, scalar *dense)
+    const label total_nnz, const label diag_nnz, const label upper_nnz,
+    const label *permute, const scalar scale, const scalar *diag,
+    const scalar *upper, const scalar *lower, const scalar *couple_coeffs,
+    scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
         const label pos{permute[i]};

From a96154bf5dd82bc20ab9b389d9e371956b256fb4 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 15:53:11 +0100
Subject: [PATCH 034/125] improve documentation, remove unneeded parameters

---
 HostMatrix/HostMatrix.C              |  46 +++++++----
 HostMatrix/HostMatrix.H              | 109 +++++++++++++++++++++++----
 HostMatrix/HostMatrixFreeFunctions.C |  35 ++-------
 3 files changed, 135 insertions(+), 55 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index d5c16ffd7..58345a68c 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -173,6 +173,27 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
                          << abort(FatalError);
 }
 
+template <class MatrixType>
+label HostMatrixWrapper<MatrixType>::count_interface_nnz(
+    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
+{
+    label ctr{0};
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+
+        bool count = (proc_interfaces)
+                         ? !!isA<processorLduInterface>(iface->interface())
+                         : !isA<processorLduInterface>(iface->interface());
+        if (count) {
+            ctr += iface->interface().faceCells().size();
+        }
+    }
+
+    return ctr;
+}
 
 template <class MatrixType>
 std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
@@ -595,25 +616,24 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
         auto couple_coeffs =
             collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
         if (is_symmetric) {
-            symmetric_update_w_interface(
-                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute, scaling_,
-                diag.data(), upper.data(), lower.data(), couple_coeffs.data(),
-                dense);
+            symmetric_update_w_interface(local_matrix_w_interfaces_nnz_,
+                                         diag_nnz, upper_nnz_, permute,
+                                         scaling_, diag.data(), upper.data(),
+                                         couple_coeffs.data(), dense);
         } else {
             non_symmetric_update_w_interface(
-                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute, scaling_,
-                diag.data(), upper.data(), lower.data(), couple_coeffs.data(),
-                dense);
+                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute,
+                scaling_, diag.data(), upper.data(), lower.data(),
+                couple_coeffs.data(), dense);
         }
     } else {
         if (is_symmetric) {
-            symmetric_update(local_matrix_nnz_, upper_nnz_,
-                                          permute, scaling_, diag.data(),
-                                          upper.data(), dense);
+            symmetric_update(local_matrix_nnz_, upper_nnz_, permute, scaling_,
+                             diag.data(), upper.data(), dense);
         } else {
-            non_symmetric_update(local_matrix_nnz_, upper_nnz_,
-                                          permute, scaling_, diag.data(),
-                                          upper.data(), lower.data(), dense);
+            non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                 scaling_, diag.data(), upper.data(),
+                                 lower.data(), dense);
         }
     }
 }
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 424975e65..393cc665c 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -103,29 +103,110 @@ const lduInterfaceField *interface_getter(
     const lduInterfaceFieldPtrsList &interfaces, const label i);
 
 
+/** Perform update of the coefficients array in row major order from a symmetric
+ *ldu matrix
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ *upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param out pointer to data of output matrix
+ **/
 void symmetric_update(const label total_nnz, const label upper_nnz,
                       const label *permute, const scalar scale,
-                      const scalar *diag, const scalar *upper,
-                      const scalar *lower, scalar *dense);
-
-
-void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, const label upper_nnz,
-                                  const label *permute, const scalar scale,
-                                  const scalar *diag, const scalar *upper,
-                                  const scalar *lower,
-                                  const scalar *couple_coeffs, scalar *dense);
-
-
+                      const scalar *diag, const scalar *upper, scalar *out);
+
+
+/** Perform update of the coefficients array in row major order from a
+ *non-symmetric ldu matrix
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ *upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for lower entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param lower pointer to lower triangular entries of the original ldu matrix
+ ** @param out pointer to data of output matrix
+ **/
 void non_symmetric_update(const label total_nnz, const label upper_nnz,
                           const label *permute, const scalar scale,
                           const scalar *diag, const scalar *upper,
                           const scalar *lower, scalar *dense);
 
 
+/** Perform update of the coefficients array in row major order from a
+ ** symmetric ldu matrix including local interface data.
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ ** upper
+ **
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for interface entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param interface pointer to interface entries
+ ** @param out pointer to data of output matrix
+ **/
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
+                                  const label upper_nnz, const label *permute,
+                                  const scalar scale, const scalar *diag,
+                                  const scalar *upper, const scalar *interface,
+                                  scalar *dense);
+
+
+/** Perform update of the coefficients array in row major order from a
+ ** non-symmetric ldu matrix including local interface data.
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ ** upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for lower entries
+ **        2*upper_nnz + diag_nnz <= permute for interface entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param lower pointer to lower triangular entries of the original ldu matrix
+ ** @param interface pointer to interface entries
+ ** @param out pointer to data of output matrix
+ **/
 void non_symmetric_update_w_interface(
-    const label total_nnz,const label diag_nnz, const label upper_nnz, const label *permute,
-    const scalar scale, const scalar *diag, const scalar *upper,
-    const scalar *lower, const scalar *couple_coeffs, scalar *dense);
+    const label total_nnz, const label diag_nnz, const label upper_nnz,
+    const label *permute, const scalar scale, const scalar *diag,
+    const scalar *upper, const scalar *lower, const scalar *couple_coeffs,
+    scalar *dense);
 
 
 /* The HostMatrixWrapper class handles the conversion from OpenFOAMs lduMatrix
diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index 58f32b259..6a7cd8338 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -15,11 +15,11 @@ const lduInterfaceField *interface_getter(
 void symmetric_update(const label total_nnz, const label upper_nnz,
                       const label *permute, const scalar scale,
                       const scalar *diag, const scalar *upper,
-                      const scalar *lower, scalar *dense)
+                      scalar *out)
 {
     for (label i = 0; i < total_nnz; ++i) {
         const label pos{permute[i]};
-        dense[i] =
+        out[i] =
             scale * (pos >= upper_nnz) ? diag[pos - upper_nnz] : upper[pos];
     }
 }
@@ -27,8 +27,8 @@ void symmetric_update(const label total_nnz, const label upper_nnz,
 void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
                                   const label upper_nnz, const label *permute,
                                   const scalar scale, const scalar *diag,
-                                  const scalar *upper, const scalar *lower,
-                                  const scalar *couple_coeffs, scalar *dense)
+                                  const scalar *upper, 
+                                  const scalar *interface, scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
         // where the element is stored in a combined array
@@ -43,7 +43,7 @@ void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
             value = diag[pos - upper_nnz];
         }
         if (pos >= upper_nnz + diag_nnz) {
-            value = -couple_coeffs[pos - upper_nnz - diag_nnz];
+            value = -interface[pos - upper_nnz - diag_nnz];
         }
         dense[i] = scale * value;
     }
@@ -52,7 +52,7 @@ void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
 void non_symmetric_update_w_interface(
     const label total_nnz, const label diag_nnz, const label upper_nnz,
     const label *permute, const scalar scale, const scalar *diag,
-    const scalar *upper, const scalar *lower, const scalar *couple_coeffs,
+    const scalar *upper, const scalar *lower, const scalar *interface,
     scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
@@ -68,7 +68,7 @@ void non_symmetric_update_w_interface(
             value = diag[pos - 2 * upper_nnz];
         }
         if (pos >= 2 * upper_nnz + diag_nnz) {
-            value = -couple_coeffs[pos - 2 * upper_nnz - diag_nnz];
+            value = -interface[pos - 2 * upper_nnz - diag_nnz];
         }
         dense[i] = scale * value;
     }
@@ -95,24 +95,3 @@ void non_symmetric_update(const label total_nnz, const label upper_nnz,
 }
 
 
-template <class MatrixType>
-label HostMatrixWrapper<MatrixType>::count_interface_nnz(
-    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
-{
-    label ctr{0};
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-        const auto iface{interface_getter(interfaces, i)};
-
-        bool count = (proc_interfaces)
-                         ? !!isA<processorLduInterface>(iface->interface())
-                         : !isA<processorLduInterface>(iface->interface());
-        if (count) {
-            ctr += iface->interface().faceCells().size();
-        }
-    }
-
-    return ctr;
-}

From ea9284eb83f16f89af7f10b320df9cf86f466bd9 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 15:56:21 +0100
Subject: [PATCH 035/125] format

---
 HostMatrix/HostMatrixFreeFunctions.C | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index 6a7cd8338..460587279 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -14,8 +14,7 @@ const lduInterfaceField *interface_getter(
 // free functions - for unit testing
 void symmetric_update(const label total_nnz, const label upper_nnz,
                       const label *permute, const scalar scale,
-                      const scalar *diag, const scalar *upper,
-                      scalar *out)
+                      const scalar *diag, const scalar *upper, scalar *out)
 {
     for (label i = 0; i < total_nnz; ++i) {
         const label pos{permute[i]};
@@ -27,8 +26,8 @@ void symmetric_update(const label total_nnz, const label upper_nnz,
 void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
                                   const label upper_nnz, const label *permute,
                                   const scalar scale, const scalar *diag,
-                                  const scalar *upper, 
-                                  const scalar *interface, scalar *dense)
+                                  const scalar *upper, const scalar *interface,
+                                  scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
         // where the element is stored in a combined array
@@ -49,11 +48,13 @@ void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
     }
 }
 
-void non_symmetric_update_w_interface(
-    const label total_nnz, const label diag_nnz, const label upper_nnz,
-    const label *permute, const scalar scale, const scalar *diag,
-    const scalar *upper, const scalar *lower, const scalar *interface,
-    scalar *dense)
+void non_symmetric_update_w_interface(const label total_nnz,
+                                      const label diag_nnz,
+                                      const label upper_nnz,
+                                      const label *permute, const scalar scale,
+                                      const scalar *diag, const scalar *upper,
+                                      const scalar *lower,
+                                      const scalar *interface, scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
         const label pos{permute[i]};
@@ -93,5 +94,3 @@ void non_symmetric_update(const label total_nnz, const label upper_nnz,
         dense[i] = scale * diag[pos - 2 * upper_nnz];
     }
 }
-
-

From 2b33020e598d5e0f7fcc8bb23028d9b700155c5b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Thu, 26 Oct 2023 14:06:03 +0200
Subject: [PATCH 036/125] base on sparse communicator branch

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7635b335..48033155e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT OGL_ALLOW_REFERENCE_ONLY)
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "4292ffdee"
+    "sparse-communicator"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)

From 29bd1cb62e36de65ebb9f9451bdb5cbdbc3a04e8 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 27 Oct 2023 13:41:53 +0200
Subject: [PATCH 037/125] adapt to new Ginkgo version

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 227 +++++++++---------
 DevicePersistent/Vector/Vector.H              |  70 +++---
 Preconditioner/Preconditioner.H               |   4 +-
 Solver/BiCGStab/GKOBiCGStab.H                 |   4 +-
 Solver/CG/GKOCG.H                             |   7 +-
 Solver/GMRES/GKOGMRES.H                       |   6 +-
 common/.common.H.swo                          | Bin 0 -> 24576 bytes
 lduLduBase/lduLduBase.H                       |   2 +-
 8 files changed, 163 insertions(+), 157 deletions(-)
 create mode 100644 common/.common.H.swo

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index e575ac058..2eafe3661 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -185,14 +185,14 @@ struct MatrixInitFunctor {
 
 
             auto comm = exec_.get_gko_mpi_host_comm();
-            TIME_WITH_FIELDNAME(
-                verbose_, update_build_repartitioner, field_name_,
-                auto repartitioner = gko::share(
-                    gko::experimental::distributed::repartitioner<
-                        label, label>::create(*comm.get(),
-                                              partition_.get_host_partition(),
-                                              partition_.get_device_partition(),
-                                              false));)
+            // TIME_WITH_FIELDNAME(
+            //     verbose_, update_build_repartitioner, field_name_,
+            //     auto repartitioner = gko::share(
+            //         gko::experimental::distributed::repartitioner<
+            //             label, label>::create(*comm.get(),
+            //                                   partition_.get_host_partition(),
+            //                                   partition_.get_device_partition(),
+            //                                   false));)
 
             auto local_nnz =
                 local_scatter_map_pers.get_array()->get_num_elems();
@@ -223,16 +223,16 @@ struct MatrixInitFunctor {
                     recv_offs_pers.get_array()->get_num_elems());
 
 
-            TIME_WITH_FIELDNAME(
-                verbose_, update_repartitioned_existing, field_name_,
-                repartitioner->update_existing(
-                    *row_idxs_.get_array(), *non_local_row_idxs_.get_array(),
-                    *coeffs.get(), *non_local_coeffs.get(),
-                    *sorting_idx_pers.get_array().get(), send_sizes,
-                    send_offsets, recv_sizes, recv_offsets,
-                    *local_scatter_map_pers.get_array().get(),
-                    *non_local_scatter_map_pers.get_array().get(), local_values,
-                    non_local_values);)
+            // TIME_WITH_FIELDNAME(
+            //     verbose_, update_repartitioned_existing, field_name_,
+            //     repartitioner->update_existing(
+            //         *row_idxs_.get_array(), *non_local_row_idxs_.get_array(),
+            //         *coeffs.get(), *non_local_coeffs.get(),
+            //         *sorting_idx_pers.get_array().get(), send_sizes,
+            //         send_offsets, recv_sizes, recv_offsets,
+            //         *local_scatter_map_pers.get_array().get(),
+            //         *non_local_scatter_map_pers.get_array().get(), local_values,
+            //         non_local_values);)
 
             auto value_view =
                 val_array::view(device_exec, local_nnz, value_ptr);
@@ -359,8 +359,9 @@ struct MatrixInitFunctor {
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
             exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm());
-        dist_A->read_distributed(A_data, non_local_A_data,
-                                 partition_.get_host_partition().get());
+        // TODO FIXME implement me
+        // dist_A->read_distributed(A_data, non_local_A_data,
+        //                          partition_.get_host_partition().get());
 
         if (partition_.get_ranks_per_gpu() == 1) {
             auto device_mat = generate_dist_mtx_with_inner_type(
@@ -370,99 +371,99 @@ struct MatrixInitFunctor {
         }
 
         // TODO test if this needs to be persistent
-        auto repartitioner = gko::share(
-            gko::experimental::distributed::repartitioner<label, label>::create(
-                *comm.get(), partition_.get_host_partition(),
-                partition_.get_device_partition(), false));
-        auto to_mat = gko::share(
-            dist_mtx::create(exec, repartitioner->get_to_communicator()));
-
-        auto [local_scatter_map, non_local_scatter_map, sorting_idx, send_sizes,
-              send_offsets, recv_sizes, recv_offsets] =
-            repartitioner->gather(dist_A.get(), to_mat.get());
-
-        // TODO add field name
-        PersistentArray<label> local_scatter_map_pers{
-            local_scatter_map.get_data(),
-            "local_scatter_map",
-            db_,
-            exec_,
-            static_cast<label>(local_scatter_map.get_num_elems()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> non_local_scatter_map_pers{
-            non_local_scatter_map.get_data(),
-            "non_local_scatter_map",
-            db_,
-            exec_,
-            static_cast<label>(non_local_scatter_map.get_num_elems()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> sorting_idx_pers{
-            sorting_idx.get_data(),
-            "sorting_idx",
-            db_,
-            exec_,
-            static_cast<label>(sorting_idx.get_num_elems()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> send_sizes_pers{
-            send_sizes.data(),
-            "send_sizes",
-            db_,
-            exec_,
-            static_cast<label>(send_sizes.size()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> send_offs_pers{
-            send_offsets.data(),
-            "send_offs",
-            db_,
-            exec_,
-            static_cast<label>(send_offsets.size()),
-            verbose_,
-            false,
-            false};
-
-        // TODO add field name
-        PersistentArray<label> recv_sizes_pers{
-            recv_sizes.data(),
-            "recv_sizes",
-            db_,
-            exec_,
-            static_cast<label>(recv_sizes.size()),
-            verbose_,
-            false,
-            false};
-
-        PersistentArray<label> recv_offs_pers{
-            recv_offsets.data(),
-            "recv_offs",
-            db_,
-            exec_,
-            static_cast<label>(recv_offsets.size()),
-            verbose_,
-            false,
-            false};
-
-        auto device_mat = generate_dist_mtx_with_inner_type(
-            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
-
-        to_mat->move_to(device_mat.get());
-
-        return device_mat;
+        // auto repartitioner = gko::share(
+        //     gko::experimental::distributed::repartitioner<label, label>::create(
+        //         *comm.get(), partition_.get_host_partition(),
+        //         partition_.get_device_partition(), false));
+        // auto to_mat = gko::share(
+        //     dist_mtx::create(exec, repartitioner->get_to_communicator()));
+        //
+        // auto [local_scatter_map, non_local_scatter_map, sorting_idx, send_sizes,
+        //       send_offsets, recv_sizes, recv_offsets] =
+        //     repartitioner->gather(dist_A.get(), to_mat.get());
+        //
+        // // TODO add field name
+        // PersistentArray<label> local_scatter_map_pers{
+        //     local_scatter_map.get_data(),
+        //     "local_scatter_map",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(local_scatter_map.get_num_elems()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> non_local_scatter_map_pers{
+        //     non_local_scatter_map.get_data(),
+        //     "non_local_scatter_map",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(non_local_scatter_map.get_num_elems()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> sorting_idx_pers{
+        //     sorting_idx.get_data(),
+        //     "sorting_idx",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(sorting_idx.get_num_elems()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> send_sizes_pers{
+        //     send_sizes.data(),
+        //     "send_sizes",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(send_sizes.size()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> send_offs_pers{
+        //     send_offsets.data(),
+        //     "send_offs",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(send_offsets.size()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // // TODO add field name
+        // PersistentArray<label> recv_sizes_pers{
+        //     recv_sizes.data(),
+        //     "recv_sizes",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(recv_sizes.size()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // PersistentArray<label> recv_offs_pers{
+        //     recv_offsets.data(),
+        //     "recv_offs",
+        //     db_,
+        //     exec_,
+        //     static_cast<label>(recv_offsets.size()),
+        //     verbose_,
+        //     false,
+        //     false};
+        //
+        // auto device_mat = generate_dist_mtx_with_inner_type(
+        //     exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
+        //
+        // to_mat->move_to(device_mat.get());
+        //
+        // return device_mat;
     }
 };
 
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index f84b09263..a6e38986b 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -102,33 +102,33 @@ struct VectorInitFunctor {
             return gko::share(dist_vec::create(
                 exec, *exec_.get_gko_mpi_device_comm().get(),
                 vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)
-                    .get()));
+                    ));
         }
 
         // when having more ranks than GPUs we must first repartition and
         // gather the vector
-        auto host_vector = gko::share(dist_vec::create(
-            exec, *exec_.get_gko_mpi_host_comm().get(),
-            vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
-
-        auto target_vector = gko::share(dist_vec::create(
-            exec, *exec_.get_gko_mpi_host_comm().get(),
-            vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
-
-        // TODO test if this needs to be persistent
-        auto repartitioner = gko::share(
-            gko::experimental::distributed::repartitioner<label, label>::create(
-                *comm.get(), partition_.get_host_partition(),
-                partition_.get_device_partition(), true));
-
-        repartitioner->gather(host_vector.get(), target_vector.get());
-
-        auto device_vec = gko::share(
-            dist_vec::create(exec, *exec_.get_gko_mpi_device_comm().get()));
-
-        target_vector->move_to(device_vec.get());
-
-        return device_vec;
+        // auto host_vector = gko::share(dist_vec::create(
+        //     exec, *exec_.get_gko_mpi_host_comm().get(),
+        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
+        //
+        // auto target_vector = gko::share(dist_vec::create(
+        //     exec, *exec_.get_gko_mpi_host_comm().get(),
+        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
+        //
+        // // TODO test if this needs to be persistent
+        // auto repartitioner = gko::share(
+        //     gko::experimental::distributed::repartitioner<label, label>::create(
+        //         *comm.get(), partition_.get_host_partition(),
+        //         partition_.get_device_partition(), true));
+        //
+        // repartitioner->gather(host_vector.get(), target_vector.get());
+        //
+        // auto device_vec = gko::share(
+        //     dist_vec::create(exec, *exec_.get_gko_mpi_device_comm().get()));
+        //
+        // target_vector->move_to(device_vec.get());
+        //
+        // return device_vec;
     }
 };
 
@@ -223,18 +223,18 @@ public:
             gko::dim<2>{local_host_size, 1}, 1);
         scalar *host_buffer = host_device_vector->get_local_values();
 
-        if (partition_.get_ranks_per_gpu() != 1) {
-            auto repartitioner = gko::share(
-                gko::experimental::distributed::repartitioner<label, label>::
-                    create(*comm.get(),
-                           partition_.get_host_partition(),    // from
-                           partition_.get_device_partition(),  // to
-                           true));
-
-            repartitioner->scatter(host_device_vector.get(),
-                                   target_vector.get());
-            host_buffer = target_vector->get_local_values();
-        }
+        // if (partition_.get_ranks_per_gpu() != 1) {
+        //     auto repartitioner = gko::share(
+        //         gko::experimental::distributed::repartitioner<label, label>::
+        //             create(*comm.get(),
+        //                    partition_.get_host_partition(),    // from
+        //                    partition_.get_device_partition(),  // to
+        //                    true));
+        //
+        //     repartitioner->scatter(host_device_vector.get(),
+        //                            target_vector.get());
+        //     host_buffer = target_vector->get_local_values();
+        // }
 
         auto host_buffer_view = gko::array<T>::view(
             exec_.get_ref_exec(), local_host_size, host_buffer);
diff --git a/Preconditioner/Preconditioner.H b/Preconditioner/Preconditioner.H
index 7bcf5bbb3..2f883790e 100644
--- a/Preconditioner/Preconditioner.H
+++ b/Preconditioner/Preconditioner.H
@@ -76,7 +76,7 @@ public:
         if (Pstream::parRun()) {
             return gko::share(
                 ras::build()
-                    .with_generated_inner_solvers(gko::share(precond->generate(
+                    .with_generated_local_solver(gko::share(precond->generate(
                         gko::as<gko::experimental::distributed::Matrix<
                             scalar, label, label>>(gkomatrix)
                             ->get_local_matrix())))
@@ -95,7 +95,7 @@ public:
     {
         if (Pstream::parRun()) {
             return gko::share(ras::build()
-                                  .with_generated_inner_solvers(
+                                  .with_generated_local_solver(
                                       precond->generate(factorization))
                                   .on(device_exec)
                                   ->generate(gkomatrix));
diff --git a/Solver/BiCGStab/GKOBiCGStab.H b/Solver/BiCGStab/GKOBiCGStab.H
index 797cb8958..2cabd288c 100644
--- a/Solver/BiCGStab/GKOBiCGStab.H
+++ b/Solver/BiCGStab/GKOBiCGStab.H
@@ -73,7 +73,7 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto bicgstab = gko::solver::Bicgstab<scalar>::build()
-                            .with_criteria(stoppingCriterionVec_)
+                            // .with_criteria(stoppingCriterionVec_)
                             .on(exec);
         return bicgstab;
     };
@@ -83,7 +83,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto bicgstab = bicgstab::build()
-                            .with_criteria(stoppingCriterionVec_)
+                            // .with_criteria(stoppingCriterionVec_)
                             .with_generated_preconditioner(precond)
                             .on(exec);
         return bicgstab;
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 427a58ca9..66e1636ed 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -68,7 +68,9 @@ public:
     std::unique_ptr<cg::Factory, std::default_delete<cg::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
+        // TODO FIXME
+        // auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
+        auto cg = cg::build().on(exec);
         return cg;
     };
 
@@ -77,7 +79,8 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto cg = cg::build()
-                      .with_criteria(stoppingCriterionVec_)
+                      // TODO FIXME
+                      // .with_criteria(stoppingCriterionVec_)
                       .with_generated_preconditioner(precond)
                       .on(exec);
         return cg;
diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 46382d9f6..28efe1b9f 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -70,7 +70,9 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto gmres =
-            gmres::build().with_criteria(stoppingCriterionVec_).on(exec);
+            gmres::build()
+            // .with_criteria(stoppingCriterionVec_)
+            .on(exec);
         return gmres;
     };
 
@@ -79,7 +81,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto gmres = gmres::build()
-                         .with_criteria(stoppingCriterionVec_)
+                         // .with_criteria(stoppingCriterionVec_)
                          .with_generated_preconditioner(precond)
                          .on(exec);
         return gmres;
diff --git a/common/.common.H.swo b/common/.common.H.swo
new file mode 100644
index 0000000000000000000000000000000000000000..2ecc4277bb64ad17652e80614be1f89770ee053b
GIT binary patch
literal 24576
zcmeI3dyFK<S-_jvgludlcI<@MASmC#dUkJj_I&<>WcESU>wV$d+sp1=oa62odZuf3
z`mU#Y=<eCQ<#O1LP#gybJ3-izk%=Y#ASOzb7zIKi2NMz#L}E)2B$Nj)1)K=tU>k*?
zL^i*w>YnMDo!#5Bg9K5J_BW5}s;{cPdiD49x0Qo$TU-)vDju|WU1?cs&%fI)y*YgC
zL*E^?)<YS1QeD{n<$Lz%db4$?=-Pgxc*c#3(rNW6@@q!mm}AHGST%6u*z){gzBC$v
zpL`S-`jpF)at-7fc$pe##kJ8lU2*KDLpSQBzJK5K;_BBOdznJ>;<*NL4dfcgHIQo{
z*Fdg;Tm!iVat-`{(m>q2)ViGpzanXLA^H4~p3h5@-{Z;WaL@HilHZ4t&q~krA5DI*
zCZG59Tu*-HAGro{4dfcgHIQo{*Fdg;Tm!iVat-7f$Tg5_AlJbEf(C5IvR)jptWUif
z;XMCO?f;Ly*0Meb?}58v4lMZOYb@*i@IIJ@?_FV8AA{e42Vo6f16RQ1@QuqY>uLB3
zJPBWf^H7IXSb?91LHJpC_A=T6x5Fe{16RX$Xy4QD6sWfTAv_BAfCE(+hG*%^C*V=|
z7*yf)@E5PLtSvYIU;8o3x)-M5`Bz%jx8UF5-{2{D7#@HI+yXN&3`1}kyc(Xxq46K^
zIQ%KR7w&;ygyZm5xC*|A<K%8wf}e!H!%^}cu;FLmc^og#!J}{|yaj$7zKjFr5%^&0
zh!O3VKkLaxR4({4((|ISI3Ny*Vc{Q<?s~lvw8E%7bng7P)z05_snD?F&^=r9oYu7C
zgoUAEtJ#!c@;!2YRThSZMzV!XEonK&wbv=MEn`V6X~An-O<5{Mfwv*ULd}()bHr}Q
z5n-ww5$4KF;KyO$MO`)N23G?=iiP7=V>j^aa7zsKl+aCE2%^~3S1<)OgU}I!nfgp}
zr%hroX@QmP+y6yPYBlt}xqd9enq8HPemoP%S}ht8?KHmkD5*1mdmWdyu$zNcQb9&h
zx;c9rnoP@00cIMFHm#pi6XIwbMu%NSd9Yn+)M;>1-jvn(_1lDgr=wNdv%{%OS%mM}
z;JwJ`>GaE)Ud>j;bE_pY%^I!Pu6R;JvBRRM+o5#I%{ZKJowH>-4DGEc5eA!Oenw?H
ztE%sNC{hi)Jr&uoy|fm-wq(1GWMJ%y^hC0oB(-;4$?PcoE}M-ZxzdVUe_gcuB__rC
znV?jfR-K&iR83Q6OEMYTP3z8>jPAJA4BZVomL;p|*-<1q10qVoXt>XQBhFc|Y&1Oz
ziV0J&vjI$TekuK)*YE0E=QS}Q_MvzYo%HvOcarz;?WDD1reZZ{`Eg-r#|+yq>>^G2
zPI>?SBhyQ>VwCA*hjH?AXZii4(NjGatMxGO86T$=+G?jQSM4aC=(^N!t6>mHW`q+>
z6+}mx$rZF>q4aiw7(Q`$dHxhCH0|5R8138Mxvmr=&8>;T+h$j8US6FY66d-yT+|OE
zZBE#&I1m|4O?G!Silyyrm8)J*JyR<A!DeBo-)I|YT2@xk%+xOZ>{;4=N7h_liq*xX
zqlah9Yl};>X|G3w8qpCkxAQ1=E8=ADB;I*27@Vcmc{N5l4{GtolTmyLdOvCPbb}%9
z^n!|Z>H#fImb|Q{i`^W>rP3|ZkK{yq(VKjY_-t>!nJ7$i?RS0ts$R>H>8F@*IbQ5I
z2&b%0S}h1S?a(QEoHW_=hTY8mTzCC5>%o}$DAt=?><HOcET-Y9t8vh5vc=7Wsxnqg
zabaI~#>M3Y8y(AVY;k!t2xBrYI%juD+f~BRP_H~!Elw@6JE2e6`CFIGZr7=@H>ppl
z*|W0BstG%VdQw!GOh-exzPKDzPRnZCEv>dOGk1&0oXmGNhVsnv($eyg@&XcSKIx(>
z@#OI65AHQQX4Zy@Cnk`1m3!f;oO3-Haj)Ns>p@r&^K>8x#j+=T_l%fW58hI5MsB0%
z$n=1)y?~Q!(~aw5xheg*<>{rO5Q{b82Ql{_lCz~A$Ia5%*yiSDaoukfgK&L}Gha>S
zXl$xzsU>?n;5exVEzc2kdqWC_Q@R_{5w@rX%`FksL|m6*{>U*gFBxIm6G!!(gE(v?
zna(haMI#8MaAa(|USx#NEv_AzU0oG(%PV499GzZSTbwy|czQ(~J+^X`&6M`8(#{K1
zpqe>-<hZzPdSzw$$lCF7-D$P2%MI!4(k`c|OQTsyq3y@I*-Nu4GYjOOzIpNR;@WW$
zgqD(&>Pol~x9<@~Mx_-~)HRaoK~puzj)fajkLQVs6s<_sTHc5tr?_o#ZDIM?n(20d
zd|M2k2%2gE3<=xc66#zJ1$ITGmY=Rv^>8kvtE)jRrjh*O&?UZ{93w(9hMf}Prc~*P
zhDXw-Q-W+t78Nw2G~3Q&BkaiBg+z6FSlKcP;2fmyHPxIbX--M?LR4*^@$V=_ser9w
zOpmSO*0xA6{a&jp3Cvg}smUR)+QO14TIsA9`++W*la{jo3)sh(V>>JRKYf1xC2ajC
z;URFK3WwmovH8CXpN0G2UhrTM7T_RU2mg%i|0#GMY{9KC3NK*e{}ns{zYgnAfnS0f
z;YDoyZ-VjxJPhaI9Z-RJ7=fRF>);aj4DEgxZU^N9n1flk0aTy9M&Is&tN8w9KA(U;
zgU8_`@VoFJJOKB@eQ+=QCP*m5BwPho!sGZ4?t*tg1*YJQ@EpE`Pr^IlG_1iCTm#?7
zU+~B98?Xf~NO0g6;0AaDydGY_XYePW{0Zma46K6#r{E+MVH94(pYQ@a0iT0+!ktip
zH^X!I7rqV8z$35$Ef|9${5^h#55XTm1d~vJ8$f`1?dQJ1tuYQljExB&TRv)HhO*A=
zhMx8;{eCZFNSK3SMXsw20bAaw42@gb8W8I;E^Di`+zjPLS)Hm;)|ijBqO!S9D63jW
z%mV3Gm0)L3mW-BTL~3bNwGp@uTlU4(#0IT=5Zl{M8?-=qw#oTk-jLO7gKHgUhY`Sa
zF?@C;5vM&AvD=U|%j(mOOqoi*GTEBZ4TjFHm8-j`;ptRR(iDc#7-9O3aCdtY!^Je|
z9D7&X3faQima0IE?!zu}QX_S{ob*b0d`CHL^KeQS%XPby)S_+6Fq0tT2HRdHb1_qS
zQc(MX`jjy~qE1_r?ArS6lRdGD+s#5%1Lv&)&4Bi`m8)@h-~h=30}@Z3^y5s5>DGzN
zg;U%O>vAXBrB96uN^jQHK~eC6O&KbGK<2uboK(Xl1_w=1Re_l3as&!2E`+eCZO1XW
zgsRn#xm)Vfvi=QI?!Xh&bV_w6k)_h`-FaDYN&^F?r5E3OUG7A?bXV_<PE_wZ*E79u
zC+^yNol7^D!HBE@b=Ore<4oTs4~{EkP8ffyvKjex;DFYj$<(Q3QXE7RCbd0W3}7cU
zGqOp#wlv%QATj1HV06lCe)wAZx@xIKmort<|J};zOvc5VmeciZFQb#`VpjV0Tv}cy
zmm2WxnT)46m1L$5xsr7|I;9Q(^`C%BPQMMY&mNg38J`u7>W-H?5laPnPioj_WZ7(R
z6I@E%U<K`yT9GvwO1~;oOQ%$-hLW4h0`BN`OOj=t@qr;bvw^5)r|n7dnx(I@C7XlE
zaZcyc`**8Jk7c=z`#L?`j0#vCi+(Lo)_OV*&6-u7she)>O53M$DRcgUg)W$(v;9!b
z1ld`__$t(0dnCF0YzC3GK-KEyQj&_>HV|gw;UKA=QTxRJyL!~SZIPyTssR?vwu_`O
zIoK8S;Z0T)r>9?LAIoGQI~jDX*$llKVdh$98!)Tu<!^%hI+^rimoDzIFB&bP=Cp5R
zt<hqITP9zxFr{~2N+%L19h)hXSQyIgKzB;87X`wuDa)uCC<leMV`_N!V*M?3jikA~
zWUH_0hX^M1bgQ9DRv1fVcm%$;rR!AYsupbqUP(74aQ1Q!X!ej~LyMGY+(;T3aeqW%
zGV6_Bza`th=^t$^>qeSB_Bn6*&#26-D<<WO#z(2AO5cgruT0GiqgWT|Iz>(6Qo%Y?
zI*ZJ4sXS-bX9ePKj<?u}(!*389e5VS=@z$;c($;9m9blGg={%-s{A-i#)G3*OG_8{
zMIAnw17Noqq&LK@JIwfbo3<Mo&-{p9f>Tr3HH~h}!t5-aT~D#(z)%ZBO!96XP;~F|
z=EPD9MNIPEFH0iE%&eDi60(eTJUz9d(^Ig$cdI@8g2zJgl%(s#Y4?CSx1Bj0$myBW
zIZ|G7Rc(D<s47)M=D2ZvH#Ub2MjLLEc&psBWA+NR7ISy6?f<W0OMV^OQ`!HV#rpLk
zw*E8ld3YE;05uqe|HQt32EGL!fVaXW@Nw+>b+{Ikjjx{bkHHWW;0Cx1zKY%cH}DAD
z0qR+Q4J0TVe-1`L+4|qdZvP(qC43(4hu?<x!V(O?=djD~hPT5t@C5ewL-5Nm21R%V
zJNz+t2du&y;9uyo?swAX{3F*uu7O+wxdwj7G@y<}^U7nVGO-j{2Vx#OwLK&)kDbb6
zr-;x>TuX_U$~eUnkI$@*#O!w^<2UMeIq?!3+aQmfO1J0qGmo80HPPoYkDbyHV|nb<
z#l}vJTiwrZaTRF$|0?X@E$m@s|EK?-|4-QYe+wUmd*BZ6p$;k*paMg1ExZw43E#m7
z@Em*_z6sxer{UA^3Ai6lgA3QfKjJ6&2T*YW55TX0iW9gR)c*_oIeZvYtUv>f!IkiJ
z`~rUspMg)qWAJ;Rd<g;ga2vc1{uTegm*BJTI6MY_1RsaH;9c-`xD9?9ehR*V4?)EW
z{1&_)ZiNMyhgZN;_!FLlcY}%r_&K-~E`cwg$NS+<xC4}4&q4$0Fa{5zZ}sed8Lor>
z!vFFG_<hJ<xdw6#<QnMFz@EZz_LRxoQ0yV=9<pq|Iq6%dZ({XAF+>+t>Y_61X7?}A
zKef};3tiu?5tCBY1$j#a>`%YsfmKdrBoD0Off4_n(~cU-1FL$Y&yxRk?5HRYtm5gH
zdA@EQNaul7X)GO&NmamA9$3}=?0KidsgwT07g&`%<~E8T4n>8c=x~&FTtoj*OtK{b
zD}+{<EUB7$!aqF1q02f<CJDu)fQq~IMu-qlv292)0WjvhSM0!m5^MrU_&zw8sd#kC
o#Gp(}DuGIp&YT2lsGyJQlbEO^Ziuk3!jR6F#I>mJy9KTM51QLKTmS$7

literal 0
HcmV?d00001

diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 4e3e6dd5f..245eee737 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -312,7 +312,7 @@ public:
         LOG_1(verbose_, "solve solver done")
         TIME_WITH_FIELDNAME(
             verbose_, solve, this->fieldName(),
-            solver->apply(gko::lend(dist_b_v), gko::lend(dist_x_v));)
+            solver->apply(dist_b_v, dist_x_v);)
 
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)

From 169f87b2994dc806454fe2c2a69fd338823d9981 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 27 Oct 2023 14:30:23 +0200
Subject: [PATCH 038/125] Use localized_partition

---
 DevicePersistent/Partition/Partition.H | 91 ++++++++++++++------------
 lduLduBase/lduLduBase.H                |  1 -
 2 files changed, 50 insertions(+), 42 deletions(-)

diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index dad7403c0..2d0e20212 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -39,8 +39,8 @@ namespace Foam {
 
 
 struct PartitionInitFunctor {
-    using vec = gko::matrix::Dense<scalar>;
-    using dist_vec = gko::experimental::distributed::Vector<scalar>;
+    using part_type =
+        gko::experimental::distributed::localized_partition<label, label>;
 
     const ExecutorHandler &exec_;
 
@@ -63,47 +63,53 @@ struct PartitionInitFunctor {
           ranks_per_gpu_(ranks_per_gpu)
     {}
 
-    void update(
-        std::shared_ptr<gko::experimental::distributed::Partition<label, label>>
-            persistent_partition) const
+    void update(std::shared_ptr<part_type> persistent_partition) const
     {
         UNUSED(persistent_partition);
     }
 
-    std::shared_ptr<gko::experimental::distributed::Partition<label, label>>
-    init() const
+    std::shared_ptr<part_type> init() const
     {
         auto exec = exec_.get_ref_exec();
         auto comm = exec_.get_gko_mpi_host_comm();
 
+        const label local_size = rows_end_ - rows_start_;
         word msg{"initialising partition of size " +
-                 std::to_string(rows_end_ - rows_start_)};
+                 std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
-        auto host_partition = gko::share(
-            gko::experimental::distributed::build_partition_from_local_range<
-                label, label>(exec, *comm.get(),
-                              gko::span(rows_start_, rows_end_)));
+        std::vector<std::pair<index_set<index_type>, comm_index_type>>
+            send_idxs;
+        std::vector<std::pair<index_set<index_type>, comm_index_type>>
+            recv_idxs;
 
-        if (ranks_per_gpu_ == 1) return host_partition;
-
-        auto local_rank = comm->rank();
-        auto local_owning_rank = ranks_per_gpu_ * (local_rank / ranks_per_gpu_);
-        auto next_owning_rank = ranks_per_gpu_ + local_owning_rank;
+        auto host_partition = gko::experimental::distributed::
+            localized_partition::build_from_blocked_recv(
+                exec, rows_end_ - rows_start_, send_idxs, recv_idxs);
 
-        // The new partition ranges up to next owning rank elements
-        auto target_rows_end =
-            host_partition->get_range_bounds()[next_owning_rank];
-
-        // if we are not on an owning rank start rows are set to be the end
-        // row else the rows start are unchanged
-        auto target_rows_start =
-            (local_rank % ranks_per_gpu_ != 0) ? target_rows_end : rows_start_;
+        if (ranks_per_gpu_ == 1) return host_partition;
 
-        return gko::share(
-            gko::experimental::distributed::build_partition_from_local_range<
-                label, label>(exec, *comm.get(),
-                              gko::span(target_rows_start, target_rows_end)));
+        // TODO FIXME
+        // auto local_rank = comm->rank();
+        // auto local_owning_rank = ranks_per_gpu_ * (local_rank /
+        // ranks_per_gpu_); auto next_owning_rank = ranks_per_gpu_ +
+        // local_owning_rank;
+
+        // // The new partition ranges up to next owning rank elements
+        // auto target_rows_end =
+        //     host_partition->get_range_bounds()[next_owning_rank];
+
+        // // if we are not on an owning rank start rows are set to be the end
+        // // row else the rows start are unchanged
+        // auto target_rows_start =
+        //     (local_rank % ranks_per_gpu_ != 0) ? target_rows_end :
+        //     rows_start_;
+
+        // return gko::share(
+        //     gko::experimental::distributed::build_partition_from_local_range<
+        //         label, label>(exec, *comm.get(),
+        //                       gko::span(target_rows_start,
+        //                       target_rows_end)));
     }
 };
 
@@ -132,6 +138,8 @@ class PersistentPartition
 
 
 public:
+    using part_type =
+        gko::experimental::distributed::localized_partition<label, label>;
     /* PersistentPartition constructor using existing memory
      *
      * @param objectRegistry reference to registry for storage
@@ -144,9 +152,7 @@ public:
     PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
                         const label verbose, const int ranks_per_gpu,
                         const label offset, const label elements)
-        : PersistentBase<
-              gko::experimental::distributed::Partition<label, label>,
-              PartitionInitFunctor>(
+        : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, offset, offset + elements, verbose,
                                    ranks_per_gpu),
@@ -157,21 +163,24 @@ public:
           exec_(exec)
     {}
 
-    std::shared_ptr<gko::experimental::distributed::Partition<label, label>>
-    get_device_partition() const
+    std::shared_ptr<part_type> get_device_partition() const
     {
         return this->get_persistent_object();
     }
 
-    std::shared_ptr<
-        const gko::experimental::distributed::Partition<label, label>>
-    get_host_partition() const
+    std::shared_ptr<const part_type> get_host_partition() const
     {
         auto comm = exec_.get_gko_mpi_host_comm();
-        return gko::share(
-            gko::experimental::distributed::build_partition_from_local_range<
-                label, label>(exec_.get_ref_exec(), *comm.get(),
-                              gko::span(offset_, offset_ + elements_)));
+        const label local_size = rows_end_ - rows_start_;
+        std::vector<std::pair<index_set<index_type>, comm_index_type>>
+            send_idxs;
+        std::vector<std::pair<index_set<index_type>, comm_index_type>>
+            recv_idxs;
+        return gko::experimental::distributed::localized_partition::
+            build_from_blocked_recv(
+                exec, rows_end_ - rows_start_, send_idxs, recv_idxs
+
+            );
     }
 
     // number of elements on this rank on the host
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 245eee737..2853e61cf 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -147,7 +147,6 @@ private:
     using dist_mtx =
         gko::experimental::distributed::Matrix<scalar, label, label>;
     using GlobalIndexType = gko::int64;
-    using part_type = gko::experimental::distributed::Partition<label, label>;
     using mtx = gko::matrix::Csr<scalar>;
     using coo_mtx = gko::matrix::Coo<scalar>;
     using csr_mtx = gko::matrix::Csr<scalar>;

From f0206117786017e3a621b8093cd9d7bcc750d86d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 27 Oct 2023 14:33:04 +0200
Subject: [PATCH 039/125] Fixup

---
 DevicePersistent/Partition/Partition.H | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 2d0e20212..77018214b 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -40,7 +40,7 @@ namespace Foam {
 
 struct PartitionInitFunctor {
     using part_type =
-        gko::experimental::distributed::localized_partition<label, label>;
+        gko::experimental::distributed::localized_partition;
 
     const ExecutorHandler &exec_;
 
@@ -126,7 +126,7 @@ struct PartitionInitFunctor {
  * */
 class PersistentPartition
     : public PersistentBase<
-          gko::experimental::distributed::Partition<label, label>,
+          gko::experimental::distributed::localized_partition,
           PartitionInitFunctor> {
     const int ranks_per_gpu_;
 
@@ -139,7 +139,7 @@ class PersistentPartition
 
 public:
     using part_type =
-        gko::experimental::distributed::localized_partition<label, label>;
+        gko::experimental::distributed::localized_partition;
     /* PersistentPartition constructor using existing memory
      *
      * @param objectRegistry reference to registry for storage

From 9451d539752ac41dd723a6c197b2e67be09a4ab2 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 11:10:00 +0100
Subject: [PATCH 040/125] construct sparse communicator

---
 DevicePersistent/Base/Base.H           |  7 ++-
 DevicePersistent/Partition/Partition.H | 72 +++++++++++++++-----------
 HostMatrix/HostMatrix.C                | 55 +++++++++++++++++++-
 HostMatrix/HostMatrix.H                | 24 +++++++++
 lduLduBase/lduLduBase.H                |  5 +-
 5 files changed, 131 insertions(+), 32 deletions(-)

diff --git a/DevicePersistent/Base/Base.H b/DevicePersistent/Base/Base.H
index e1bb611c6..5f8c8b9c2 100644
--- a/DevicePersistent/Base/Base.H
+++ b/DevicePersistent/Base/Base.H
@@ -86,6 +86,8 @@ private:
 
     const label verbose_;
 
+    const InitFunctor init_functor_;
+
     // This needs to be a pointer so that it can be assigned
     // during the constructor, since regIOobject needs arguments
     // to be constructed
@@ -99,7 +101,8 @@ public:
         : name_(name),
           db_(db),
           from_registry_{db.foundObject<regIOobject>(name_)},
-          verbose_(verbose)
+          verbose_(verbose),
+          init_functor_(f)
     {
         if (from_registry_) {
             word msg = "reading " + name_ + " from registry";
@@ -149,6 +152,8 @@ public:
 
     word get_name() const { return name_; }
 
+    const InitFunctor &get_init_functor() const { return init_functor_; }
+
     const objectRegistry &get_db() const { return db_; }
 
     bool get_stored() const { return from_registry_; }
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 77018214b..1c12bdf28 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -39,8 +39,7 @@ namespace Foam {
 
 
 struct PartitionInitFunctor {
-    using part_type =
-        gko::experimental::distributed::localized_partition;
+    using part_type = gko::experimental::distributed::localized_partition<>;
 
     const ExecutorHandler &exec_;
 
@@ -52,15 +51,25 @@ struct PartitionInitFunctor {
 
     const int ranks_per_gpu_;
 
+    // Array storing the ranks of the neighbouring processors
+    const gko::array<label> target_ids_;
+
+    // Array storing the ranks of the neighbouring processors
+    const gko::array<label> target_sizes_;
+
 
     PartitionInitFunctor(const ExecutorHandler &exec, const label rows_start,
                          const label rows_end, const label verbose,
-                         const int ranks_per_gpu)
+                         const int ranks_per_gpu,
+                         const gko::array<label> &target_ids,
+                         const gko::array<label> target_sizes)
         : exec_(exec),
           rows_end_(rows_end),
           rows_start_(rows_start),
           verbose_(verbose),
-          ranks_per_gpu_(ranks_per_gpu)
+          ranks_per_gpu_(ranks_per_gpu),
+          target_ids_(target_ids),
+          target_sizes_(target_sizes)
     {}
 
     void update(std::shared_ptr<part_type> persistent_partition) const
@@ -78,14 +87,11 @@ struct PartitionInitFunctor {
                  std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
-        std::vector<std::pair<index_set<index_type>, comm_index_type>>
-            send_idxs;
-        std::vector<std::pair<index_set<index_type>, comm_index_type>>
-            recv_idxs;
+        std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
 
         auto host_partition = gko::experimental::distributed::
-            localized_partition::build_from_blocked_recv(
-                exec, rows_end_ - rows_start_, send_idxs, recv_idxs);
+            localized_partition<>::build_from_blocked_recv(
+                exec, local_size, send_idxs, target_ids_, target_sizes_);
 
         if (ranks_per_gpu_ == 1) return host_partition;
 
@@ -126,7 +132,7 @@ struct PartitionInitFunctor {
  * */
 class PersistentPartition
     : public PersistentBase<
-          gko::experimental::distributed::localized_partition,
+          gko::experimental::distributed::localized_partition<>,
           PartitionInitFunctor> {
     const int ranks_per_gpu_;
 
@@ -138,8 +144,7 @@ class PersistentPartition
 
 
 public:
-    using part_type =
-        gko::experimental::distributed::localized_partition;
+    using part_type = gko::experimental::distributed::localized_partition<>;
     /* PersistentPartition constructor using existing memory
      *
      * @param objectRegistry reference to registry for storage
@@ -151,11 +156,13 @@ public:
      */
     PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
                         const label verbose, const int ranks_per_gpu,
-                        const label offset, const label elements)
+                        const label offset, const label elements,
+                         const gko::array<label> &target_ids,
+                         const gko::array<label> target_sizes)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, offset, offset + elements, verbose,
-                                   ranks_per_gpu),
+                                   ranks_per_gpu, target_ids, target_sizes),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           offset_(offset),
@@ -170,17 +177,20 @@ public:
 
     std::shared_ptr<const part_type> get_host_partition() const
     {
-        auto comm = exec_.get_gko_mpi_host_comm();
-        const label local_size = rows_end_ - rows_start_;
-        std::vector<std::pair<index_set<index_type>, comm_index_type>>
-            send_idxs;
-        std::vector<std::pair<index_set<index_type>, comm_index_type>>
-            recv_idxs;
-        return gko::experimental::distributed::localized_partition::
-            build_from_blocked_recv(
-                exec, rows_end_ - rows_start_, send_idxs, recv_idxs
-
-            );
+        // TODO FIXME can this be avoided? Why do we have this twice?
+        // auto comm = exec_.get_gko_mpi_host_comm();
+        // auto exec = exec_.get_ref_exec();
+        // const label local_size = rows_end_ - rows_start_;
+        // std::vector<std::pair<gko::index_set<label>, label>>
+        //     send_idxs;
+        // gko::array<label> target_ids;
+        // gko::array<label> target_sizes;
+        return {};
+        // gko::experimental::distributed::localized_partition<>::
+        //     build_from_blocked_recv(
+        //         exec, rows_end_ - rows_start_, send_idxs, recv_idxs
+        //
+        //     );
     }
 
     // number of elements on this rank on the host
@@ -189,14 +199,18 @@ public:
     // number of elements on this rank on the device
     label get_local_device_size() const
     {
-        auto comm = exec_.get_gko_mpi_device_comm();
-        return this->get_persistent_object()->get_part_size(comm->rank());
+        // TODO FIXME can this be avoided? Why do we have this twice?
+        // auto comm = exec_.get_gko_mpi_device_comm();
+        // return this->get_persistent_object()->get_part_size(comm->rank());
+        return 0;
     }
 
 
     label get_total_size() const
     {
-        return this->get_persistent_object()->get_size();
+        // TODO FIXME can this be avoided? Why do we have this twice?
+        // return this->get_persistent_object()->get_size();
+        return 0;
     }
 
     label get_ranks_per_gpu() const { return ranks_per_gpu_; }
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 731b20936..fc6cb2b14 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -39,7 +39,7 @@ const lduInterfaceField *interface_getter(
 #else
     return interfaces.operator()(i);
 #endif
-};
+}
 
 template <class MatrixType>
 label HostMatrixWrapper<MatrixType>::count_interface_nnz(
@@ -149,6 +149,59 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
     return local_interface_idxs;
 }
 
+template <class MatrixType>
+std::pair<gko::array<label>, gko::array<label>> HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    // temp vector to store neighbour proc number and number of cells to send
+    std::vector<std::pair<label, label>> neighbour_procs{};
+    // TODO FIXME use a lambda to avoid repeating
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+
+        const auto &iface{interface_getter(interfaces, i)};
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        if (isA<processorLduInterface>(iface->interface())) {
+            const processorLduInterface &pldui =
+                refCast<const processorLduInterface>(iface->interface());
+            const label neighbProcNo = pldui.neighbProcNo();
+
+            neighbour_procs.push_back(
+                std::pair<label, label>{neighbProcNo, interface_size});
+        }
+    }
+
+    // reduce vector
+    label n_procs = 0;
+    std::map<label, label> reduce_map{};
+    for (auto [proc, n_faces] : neighbour_procs) {
+        auto search = reduce_map.find(proc);
+        if (search == reduce_map.end()) {
+            n_procs+=1;
+            reduce_map.insert(proc, n_faces);
+        } else {
+            reduce_map[proc] = reduce_map[proc] + n_faces;
+        }
+    }
+
+    // convert to gko::array
+    gko::array<label> target_ids {exec_.get_ref_exec(), n_procs};
+    gko::array<label> target_sizes {exec_.get_ref_exec(), n_procs};
+
+    label iter = 0;
+    for (const auto& [proc, size] : reduce_map) {
+        target_ids.get_data()[iter] = proc;
+        target_sizes.get_data()[iter] = size;
+        iter++;
+    }
+
+    return std::pair(target_ids, target_sizes);
+}
+
 template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 6449d587c..93912b7ca 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -138,6 +138,10 @@ private:
     // non-local indices
     const label non_local_matrix_nnz_;
 
+    // Array storing the ranks of the neighbouring processors
+    const std::pair<gko::array<label>, gko::array<label>>
+        proc_target_id_and_sizes_;
+
     mutable PersistentSparsityPattern non_local_sparsity_;
 
     mutable PersistentArray<scalar> non_local_coeffs_;
@@ -196,6 +200,7 @@ public:
                      // matrix it will be put on the device
           },
           non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
+          proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
           non_local_sparsity_{
               fieldName + "_non_local", db,       exec_,
               non_local_matrix_nnz_,    verbose_,
@@ -269,6 +274,7 @@ public:
                      // matrix it will be put on the device
           },
           non_local_matrix_nnz_(),
+          // proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
           non_local_sparsity_{
               fieldName + "_non_local", db,       exec_,
               non_local_matrix_nnz_,    verbose_,
@@ -323,6 +329,24 @@ public:
     label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
                               bool proc_interfaces) const;
 
+    /* Iterates all interfaces and counts the number of unique neighbour
+     *processors and number of interfaces in total for this processor
+     **
+     ** @param interfaces The list of interfaces for the search
+     */
+    std::pair<gko::array<label>, gko::array<label>> assemble_proc_id_and_sizes(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /* Getter for proc_id and sizes arrays
+     *
+     */
+    const std::pair<gko::array<label>, gko::array<label>> &
+    get_proc_target_id_and_sizes() const
+    {
+        return proc_target_id_and_sizes_;
+    };
+
+
     /* Iterates all interfaces and collects non-local global column indices
     **
     ** Non-local global column indices are computed from the global cell-id
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 2853e61cf..58db9edaf 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -216,13 +216,16 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
+        const auto& [target_ids, target_sizes] = this->get_proc_target_id_and_sizes();
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
             verbose_,
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
             this->get_global_row_index().offset(Pstream::myProcNo()),  // offset
-            psi.size()  // number of elements
+            psi.size(),  // number of elements
+            target_ids,
+            target_sizes
         };
 
         // TODO this is now the local matrix part for dist_A

From b8fca337bed7c1c954c086e9952f206fa9ebabbb Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 11:15:12 +0100
Subject: [PATCH 041/125] reformat

---
 DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H | 10 ++++++----
 DevicePersistent/Partition/Partition.H               |  4 ++--
 DevicePersistent/Vector/Vector.H                     | 12 +++++++-----
 HostMatrix/HostMatrix.C                              | 11 ++++++-----
 HostMatrix/HostMatrix.H                              |  4 ++--
 lduLduBase/lduLduBase.H                              | 11 +++++------
 6 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 2eafe3661..5938d187d 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -231,8 +231,8 @@ struct MatrixInitFunctor {
             //         *sorting_idx_pers.get_array().get(), send_sizes,
             //         send_offsets, recv_sizes, recv_offsets,
             //         *local_scatter_map_pers.get_array().get(),
-            //         *non_local_scatter_map_pers.get_array().get(), local_values,
-            //         non_local_values);)
+            //         *non_local_scatter_map_pers.get_array().get(),
+            //         local_values, non_local_values);)
 
             auto value_view =
                 val_array::view(device_exec, local_nnz, value_ptr);
@@ -372,13 +372,15 @@ struct MatrixInitFunctor {
 
         // TODO test if this needs to be persistent
         // auto repartitioner = gko::share(
-        //     gko::experimental::distributed::repartitioner<label, label>::create(
+        //     gko::experimental::distributed::repartitioner<label,
+        //     label>::create(
         //         *comm.get(), partition_.get_host_partition(),
         //         partition_.get_device_partition(), false));
         // auto to_mat = gko::share(
         //     dist_mtx::create(exec, repartitioner->get_to_communicator()));
         //
-        // auto [local_scatter_map, non_local_scatter_map, sorting_idx, send_sizes,
+        // auto [local_scatter_map, non_local_scatter_map, sorting_idx,
+        // send_sizes,
         //       send_offsets, recv_sizes, recv_offsets] =
         //     repartitioner->gather(dist_A.get(), to_mat.get());
         //
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 1c12bdf28..178e10d09 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -157,8 +157,8 @@ public:
     PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
                         const label verbose, const int ranks_per_gpu,
                         const label offset, const label elements,
-                         const gko::array<label> &target_ids,
-                         const gko::array<label> target_sizes)
+                        const gko::array<label> &target_ids,
+                        const gko::array<label> target_sizes)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, offset, offset + elements, verbose,
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index a6e38986b..33bc807e3 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -101,23 +101,25 @@ struct VectorInitFunctor {
         if (partition_.get_ranks_per_gpu() == 1) {
             return gko::share(dist_vec::create(
                 exec, *exec_.get_gko_mpi_device_comm().get(),
-                vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)
-                    ));
+                vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)));
         }
 
         // when having more ranks than GPUs we must first repartition and
         // gather the vector
         // auto host_vector = gko::share(dist_vec::create(
         //     exec, *exec_.get_gko_mpi_host_comm().get(),
-        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
+        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view,
+        //     1).get()));
         //
         // auto target_vector = gko::share(dist_vec::create(
         //     exec, *exec_.get_gko_mpi_host_comm().get(),
-        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1).get()));
+        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view,
+        //     1).get()));
         //
         // // TODO test if this needs to be persistent
         // auto repartitioner = gko::share(
-        //     gko::experimental::distributed::repartitioner<label, label>::create(
+        //     gko::experimental::distributed::repartitioner<label,
+        //     label>::create(
         //         *comm.get(), partition_.get_host_partition(),
         //         partition_.get_device_partition(), true));
         //
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index fc6cb2b14..4e272b1b3 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -150,7 +150,8 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
 }
 
 template <class MatrixType>
-std::pair<gko::array<label>, gko::array<label>> HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
+std::pair<gko::array<label>, gko::array<label>>
+HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // temp vector to store neighbour proc number and number of cells to send
@@ -181,7 +182,7 @@ std::pair<gko::array<label>, gko::array<label>> HostMatrixWrapper<MatrixType>::a
     for (auto [proc, n_faces] : neighbour_procs) {
         auto search = reduce_map.find(proc);
         if (search == reduce_map.end()) {
-            n_procs+=1;
+            n_procs += 1;
             reduce_map.insert(proc, n_faces);
         } else {
             reduce_map[proc] = reduce_map[proc] + n_faces;
@@ -189,11 +190,11 @@ std::pair<gko::array<label>, gko::array<label>> HostMatrixWrapper<MatrixType>::a
     }
 
     // convert to gko::array
-    gko::array<label> target_ids {exec_.get_ref_exec(), n_procs};
-    gko::array<label> target_sizes {exec_.get_ref_exec(), n_procs};
+    gko::array<label> target_ids{exec_.get_ref_exec(), n_procs};
+    gko::array<label> target_sizes{exec_.get_ref_exec(), n_procs};
 
     label iter = 0;
-    for (const auto& [proc, size] : reduce_map) {
+    for (const auto &[proc, size] : reduce_map) {
         target_ids.get_data()[iter] = proc;
         target_sizes.get_data()[iter] = size;
         iter++;
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 93912b7ca..8657519a9 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -340,8 +340,8 @@ public:
     /* Getter for proc_id and sizes arrays
      *
      */
-    const std::pair<gko::array<label>, gko::array<label>> &
-    get_proc_target_id_and_sizes() const
+    const std::pair<gko::array<label>, gko::array<label>>
+        &get_proc_target_id_and_sizes() const
     {
         return proc_target_id_and_sizes_;
     };
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 58db9edaf..0313002ef 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -216,7 +216,8 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
-        const auto& [target_ids, target_sizes] = this->get_proc_target_id_and_sizes();
+        const auto &[target_ids, target_sizes] =
+            this->get_proc_target_id_and_sizes();
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
@@ -225,8 +226,7 @@ public:
             this->get_global_row_index().offset(Pstream::myProcNo()),  // offset
             psi.size(),  // number of elements
             target_ids,
-            target_sizes
-        };
+            target_sizes};
 
         // TODO this is now the local matrix part for dist_A
         MatrixWrapper dist_A{db_,
@@ -312,9 +312,8 @@ public:
         TIME_WITH_FIELDNAME(verbose_, generate_solver, this->fieldName(),
                             auto solver = solver_gen->generate(dist_A_v);)
         LOG_1(verbose_, "solve solver done")
-        TIME_WITH_FIELDNAME(
-            verbose_, solve, this->fieldName(),
-            solver->apply(dist_b_v, dist_x_v);)
+        TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
+                            solver->apply(dist_b_v, dist_x_v);)
 
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)

From c51a5e607c219ff6eb242b8eed3efd4a2fe42b5a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 11:18:04 +0100
Subject: [PATCH 042/125] reformat

---
 Solver/GMRES/GKOGMRES.H | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 28efe1b9f..7e6497627 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -69,10 +69,9 @@ public:
     std::unique_ptr<gmres::Factory, std::default_delete<gmres::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        auto gmres =
-            gmres::build()
-            // .with_criteria(stoppingCriterionVec_)
-            .on(exec);
+        auto gmres = gmres::build()
+                         // .with_criteria(stoppingCriterionVec_)
+                         .on(exec);
         return gmres;
     };
 

From 42b162099e608503bf0d7386a2687a36c6babf12 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 15:54:16 +0100
Subject: [PATCH 043/125] add a todo fixme notes check

---
 .github/workflows/pr-formatting.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/pr-formatting.yml b/.github/workflows/pr-formatting.yml
index bb3734396..0c834e5c6 100644
--- a/.github/workflows/pr-formatting.yml
+++ b/.github/workflows/pr-formatting.yml
@@ -19,3 +19,8 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
         run: cp --preserve .github/format.sh /tmp && /tmp/format.sh
+      - name: check for todo fixme note
+        run: |
+          NTODOS="$(grep -r 'TODO FIXME' | wc -l)"
+          echo "Found $NTODOS TODO FIXME notes" 
+          ! grep -q -r "TODO FIXME"  

From 463199fd22253cbe7d0dc2ab204bfcfe7b2fea91 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 13:04:27 +0100
Subject: [PATCH 044/125] Allow - as separator

---
 .github/workflows/integration-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 203157856..815925cb3 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -91,7 +91,7 @@ jobs:
     - name: Get Ginkgo checkout version
       shell: bash
       run: |
-        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z]*" > GINKGO_CHECKOUT_VERSION
+        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
         export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
         echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
         echo "Ginkgo_DIR=${{inputs.foam_user_libbin}}/cmake/Ginkgo" >> $GITHUB_ENV

From 011107641ce3e48acea8dc626db53c35f2f29131 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 13:05:10 +0100
Subject: [PATCH 045/125] Allow - as separator

---
 .github/workflows/build-foam.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index e082fbb6a..09c56769e 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -60,7 +60,7 @@ jobs:
     - name: Get Ginkgo checkout version
       shell: bash
       run: |
-        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z]*" > GINKGO_CHECKOUT_VERSION
+        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
         export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
         echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
 

From e0463967e92a86e00939c1a563814746c1c54188 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 14:27:59 +0100
Subject: [PATCH 046/125] switch gko version to include try-run patch

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 48033155e..9e3d69238 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT OGL_ALLOW_REFERENCE_ONLY)
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "sparse-communicator"
+    "sparse-communicator-ogl"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)

From 622e8da4ddba91254cffc5ded6e3e8c525f33060 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 15:58:01 +0100
Subject: [PATCH 047/125] add a todo fixme notes check

---
 third_party/ginkgo/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/third_party/ginkgo/CMakeLists.txt b/third_party/ginkgo/CMakeLists.txt
index f63d5dda6..56195154f 100644
--- a/third_party/ginkgo/CMakeLists.txt
+++ b/third_party/ginkgo/CMakeLists.txt
@@ -15,4 +15,6 @@ ginkgo_load_git_package(
   "-DGINKGO_BUILD_HWLOC=OFF"
   "-DGINKGO_FORCE_GPU_AWARE_MPI=${GINKGO_FORCE_GPU_AWARE_MPI}"
   "-DCMAKE_INSTALL_PREFIX=$ENV{FOAM_USER_LIBBIN}/.."
-  "-DBUILD_SHARED_LIBS=ON")
+  "-DBUILD_SHARED_LIBS=ON"
+  "-DCMAKE_CXX_STANDARD=17"
+  )

From 74c757ccf7b07850aa914dbac24e3a68f9ca181f Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 31 Oct 2023 16:06:04 +0100
Subject: [PATCH 048/125] add a todo fixme notes check

---
 .github/workflows/format-check.yml  | 5 +++++
 .github/workflows/pr-formatting.yml | 5 -----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml
index a7d84810b..3d2519756 100644
--- a/.github/workflows/format-check.yml
+++ b/.github/workflows/format-check.yml
@@ -14,3 +14,8 @@ jobs:
         clang-format-version: '13'
         check-path: '.'
         fallback-style: 'Mozilla' # optional
+    - name: check for todo fixme note
+      run: |
+        NTODOS="$(grep -r 'TODO FIXME' | wc -l)"
+        echo "Found $NTODOS TODO FIXME notes" 
+        ! grep -q -r "TODO FIXME"  
diff --git a/.github/workflows/pr-formatting.yml b/.github/workflows/pr-formatting.yml
index 0c834e5c6..bb3734396 100644
--- a/.github/workflows/pr-formatting.yml
+++ b/.github/workflows/pr-formatting.yml
@@ -19,8 +19,3 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
         run: cp --preserve .github/format.sh /tmp && /tmp/format.sh
-      - name: check for todo fixme note
-        run: |
-          NTODOS="$(grep -r 'TODO FIXME' | wc -l)"
-          echo "Found $NTODOS TODO FIXME notes" 
-          ! grep -q -r "TODO FIXME"  

From 9c5732f7eb98c874bd9e3d8c5613e57412c240a6 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 1 Nov 2023 17:43:38 +0100
Subject: [PATCH 049/125] add read_distributed with sparse comm

- removed repartitioning for now
- moved sorted validation to python script (not implemented yet)
---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 147 +-----------------
 DevicePersistent/Partition/Partition.H        |  82 +++-------
 DevicePersistent/Vector/Vector.H              |  54 +------
 common/common.H                               |   2 +
 lduLduBase/lduLduBase.H                       |   2 +-
 test/data_validation.py                       |  48 ++++++
 6 files changed, 84 insertions(+), 251 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 5938d187d..49aa7f45d 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -293,8 +293,8 @@ struct MatrixInitFunctor {
 
     std::shared_ptr<dist_mtx> init() const
     {
-        label nCells = partition_.get_local_host_size();
-        word msg{"init global csr matrix of size " + std::to_string(nCells)};
+        const label local_size = partition_.get_local_size();
+        word msg{"init global csr matrix of size " + std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
         auto coeffs = coeffs_.get_array();
@@ -305,45 +305,6 @@ struct MatrixInitFunctor {
         auto non_local_cols = non_local_col_idxs_.get_array();
         auto non_local_rows = non_local_row_idxs_.get_array();
 
-#ifdef DATA_VALIDATION
-        bool debug = true;
-#else
-        bool debug = false;
-#endif
-        // check if sorted
-        if (debug) {
-            bool is_sorted_rows = true;
-            bool is_sorted_cols = true;
-            auto rows_data = rows->get_const_data();
-            auto cols_data = cols->get_const_data();
-            for (size_t i = 1; i < cols->get_num_elems(); i++) {
-                if (rows_data[i] < rows_data[i - 1]) {
-                    is_sorted_rows = false;
-                    Info << "rows sorting error element " << i << " row[i] "
-                         << rows_data[i] << " row[i-1] " << rows_data[i - 1]
-                         << endl;
-                }
-                // same row but subsequent column is smaller
-                if (cols_data[i] < cols_data[i - 1] &&
-                    rows_data[i] == rows_data[i - 1]) {
-                    is_sorted_cols = false;
-                    Info << "cols sorting error element " << i << " row[i] "
-                         << rows_data[i] << " row[i-1] " << rows_data[i - 1]
-                         << " col[i] " << cols_data[i] << " col[i-1] "
-                         << cols_data[i - 1] << endl;
-                }
-            }
-
-            Info << "is_sorted rows " << is_sorted_rows << endl;
-            Info << "is_sorted cols " << is_sorted_cols << endl;
-
-            if (!is_sorted_cols || !is_sorted_rows) {
-                for (size_t i = 1; i < cols->get_num_elems(); i++) {
-                    Info << i << "sparsity (" << rows_data[i] << ","
-                         << cols_data[i] << ")\n";
-                }
-            }
-        }
         auto exec = exec_.get_ref_exec();
 
         auto num_rows = partition_.get_total_size();
@@ -359,9 +320,11 @@ struct MatrixInitFunctor {
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
             exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm());
-        // TODO FIXME implement me
-        // dist_A->read_distributed(A_data, non_local_A_data,
-        //                          partition_.get_host_partition().get());
+        dist_A->read_distributed(A_data, non_local_A_data,
+                gko::experimental::distributed::sparse_communicator::create(
+                    *comm.get(),
+                                 partition_.get_localized_partition()
+                                 ));
 
         if (partition_.get_ranks_per_gpu() == 1) {
             auto device_mat = generate_dist_mtx_with_inner_type(
@@ -370,102 +333,6 @@ struct MatrixInitFunctor {
             return device_mat;
         }
 
-        // TODO test if this needs to be persistent
-        // auto repartitioner = gko::share(
-        //     gko::experimental::distributed::repartitioner<label,
-        //     label>::create(
-        //         *comm.get(), partition_.get_host_partition(),
-        //         partition_.get_device_partition(), false));
-        // auto to_mat = gko::share(
-        //     dist_mtx::create(exec, repartitioner->get_to_communicator()));
-        //
-        // auto [local_scatter_map, non_local_scatter_map, sorting_idx,
-        // send_sizes,
-        //       send_offsets, recv_sizes, recv_offsets] =
-        //     repartitioner->gather(dist_A.get(), to_mat.get());
-        //
-        // // TODO add field name
-        // PersistentArray<label> local_scatter_map_pers{
-        //     local_scatter_map.get_data(),
-        //     "local_scatter_map",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(local_scatter_map.get_num_elems()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> non_local_scatter_map_pers{
-        //     non_local_scatter_map.get_data(),
-        //     "non_local_scatter_map",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(non_local_scatter_map.get_num_elems()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> sorting_idx_pers{
-        //     sorting_idx.get_data(),
-        //     "sorting_idx",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(sorting_idx.get_num_elems()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> send_sizes_pers{
-        //     send_sizes.data(),
-        //     "send_sizes",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(send_sizes.size()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> send_offs_pers{
-        //     send_offsets.data(),
-        //     "send_offs",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(send_offsets.size()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // // TODO add field name
-        // PersistentArray<label> recv_sizes_pers{
-        //     recv_sizes.data(),
-        //     "recv_sizes",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(recv_sizes.size()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // PersistentArray<label> recv_offs_pers{
-        //     recv_offsets.data(),
-        //     "recv_offs",
-        //     db_,
-        //     exec_,
-        //     static_cast<label>(recv_offsets.size()),
-        //     verbose_,
-        //     false,
-        //     false};
-        //
-        // auto device_mat = generate_dist_mtx_with_inner_type(
-        //     exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
-        //
-        // to_mat->move_to(device_mat.get());
-        //
-        // return device_mat;
     }
 };
 
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 178e10d09..13743ef50 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -47,6 +47,8 @@ struct PartitionInitFunctor {
 
     const label rows_start_;
 
+    const label local_size_;
+
     const label verbose_;
 
     const int ranks_per_gpu_;
@@ -66,11 +68,13 @@ struct PartitionInitFunctor {
         : exec_(exec),
           rows_end_(rows_end),
           rows_start_(rows_start),
+          local_size_(rows_end_ - rows_start_),
           verbose_(verbose),
           ranks_per_gpu_(ranks_per_gpu),
           target_ids_(target_ids),
           target_sizes_(target_sizes)
-    {}
+    {
+    }
 
     void update(std::shared_ptr<part_type> persistent_partition) const
     {
@@ -82,40 +86,17 @@ struct PartitionInitFunctor {
         auto exec = exec_.get_ref_exec();
         auto comm = exec_.get_gko_mpi_host_comm();
 
-        const label local_size = rows_end_ - rows_start_;
         word msg{"initialising partition of size " +
-                 std::to_string(local_size)};
+                 std::to_string(local_size_)};
         LOG_1(verbose_, msg)
 
         std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
 
         auto host_partition = gko::experimental::distributed::
             localized_partition<>::build_from_blocked_recv(
-                exec, local_size, send_idxs, target_ids_, target_sizes_);
+                exec, local_size_, send_idxs, target_ids_, target_sizes_);
 
         if (ranks_per_gpu_ == 1) return host_partition;
-
-        // TODO FIXME
-        // auto local_rank = comm->rank();
-        // auto local_owning_rank = ranks_per_gpu_ * (local_rank /
-        // ranks_per_gpu_); auto next_owning_rank = ranks_per_gpu_ +
-        // local_owning_rank;
-
-        // // The new partition ranges up to next owning rank elements
-        // auto target_rows_end =
-        //     host_partition->get_range_bounds()[next_owning_rank];
-
-        // // if we are not on an owning rank start rows are set to be the end
-        // // row else the rows start are unchanged
-        // auto target_rows_start =
-        //     (local_rank % ranks_per_gpu_ != 0) ? target_rows_end :
-        //     rows_start_;
-
-        // return gko::share(
-        //     gko::experimental::distributed::build_partition_from_local_range<
-        //         label, label>(exec, *comm.get(),
-        //                       gko::span(target_rows_start,
-        //                       target_rows_end)));
     }
 };
 
@@ -138,7 +119,9 @@ class PersistentPartition
 
     const label offset_;
 
-    const label elements_;
+    const label local_elements_;
+
+    mutable label global_elements_;
 
     const ExecutorHandler &exec_;
 
@@ -166,51 +149,26 @@ public:
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           offset_(offset),
-          elements_(elements),
+          local_elements_(elements),
           exec_(exec)
-    {}
-
-    std::shared_ptr<part_type> get_device_partition() const
     {
-        return this->get_persistent_object();
-    }
+        auto comm = exec.get_gko_mpi_host_comm();
+        label local_elements = local_elements_;
+        comm->all_reduce(exec.get_ref_exec(), &local_elements, 1, MPI_SUM);
+        global_elements_ = local_elements;
+          }
 
-    std::shared_ptr<const part_type> get_host_partition() const
+    std::shared_ptr<part_type> get_localized_partition() const
     {
-        // TODO FIXME can this be avoided? Why do we have this twice?
-        // auto comm = exec_.get_gko_mpi_host_comm();
-        // auto exec = exec_.get_ref_exec();
-        // const label local_size = rows_end_ - rows_start_;
-        // std::vector<std::pair<gko::index_set<label>, label>>
-        //     send_idxs;
-        // gko::array<label> target_ids;
-        // gko::array<label> target_sizes;
-        return {};
-        // gko::experimental::distributed::localized_partition<>::
-        //     build_from_blocked_recv(
-        //         exec, rows_end_ - rows_start_, send_idxs, recv_idxs
-        //
-        //     );
+        return this->get_persistent_object();
     }
 
     // number of elements on this rank on the host
-    label get_local_host_size() const { return elements_; }
-
-    // number of elements on this rank on the device
-    label get_local_device_size() const
-    {
-        // TODO FIXME can this be avoided? Why do we have this twice?
-        // auto comm = exec_.get_gko_mpi_device_comm();
-        // return this->get_persistent_object()->get_part_size(comm->rank());
-        return 0;
-    }
-
+    label get_local_size() const { return local_elements_; }
 
     label get_total_size() const
     {
-        // TODO FIXME can this be avoided? Why do we have this twice?
-        // return this->get_persistent_object()->get_size();
-        return 0;
+        return global_elements_;
     }
 
     label get_ranks_per_gpu() const { return ranks_per_gpu_; }
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 33bc807e3..640c60c40 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -75,7 +75,7 @@ struct VectorInitFunctor {
     void update(std::shared_ptr<gko::experimental::distributed::Vector<T>>
                     persistent_vector) const
     {
-        const auto local_size = partition_.get_local_host_size();
+        const auto local_size = partition_.get_local_size();
         word msg{"updating array " + name_ + " of size " +
                  std::to_string(local_size)};
         LOG_1(verbose_, msg)
@@ -89,7 +89,7 @@ struct VectorInitFunctor {
             (on_device_) ? exec_.get_device_exec() : exec_.get_ref_exec();
         auto comm = exec_.get_gko_mpi_host_comm();
 
-        const auto local_size = partition_.get_local_host_size();
+        const auto local_size = partition_.get_local_size();
         word msg{"initialising vector " + name_ + " of size " +
                  std::to_string(local_size)};
         msg += (on_device_) ? " on device" : " on host";
@@ -103,34 +103,6 @@ struct VectorInitFunctor {
                 exec, *exec_.get_gko_mpi_device_comm().get(),
                 vec::create(exec, gko::dim<2>{local_size, 1}, host_view, 1)));
         }
-
-        // when having more ranks than GPUs we must first repartition and
-        // gather the vector
-        // auto host_vector = gko::share(dist_vec::create(
-        //     exec, *exec_.get_gko_mpi_host_comm().get(),
-        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view,
-        //     1).get()));
-        //
-        // auto target_vector = gko::share(dist_vec::create(
-        //     exec, *exec_.get_gko_mpi_host_comm().get(),
-        //     vec::create(exec, gko::dim<2>{local_size, 1}, host_view,
-        //     1).get()));
-        //
-        // // TODO test if this needs to be persistent
-        // auto repartitioner = gko::share(
-        //     gko::experimental::distributed::repartitioner<label,
-        //     label>::create(
-        //         *comm.get(), partition_.get_host_partition(),
-        //         partition_.get_device_partition(), true));
-        //
-        // repartitioner->gather(host_vector.get(), target_vector.get());
-        //
-        // auto device_vec = gko::share(
-        //     dist_vec::create(exec, *exec_.get_gko_mpi_device_comm().get()));
-        //
-        // target_vector->move_to(device_vec.get());
-        //
-        // return device_vec;
     }
 };
 
@@ -209,8 +181,7 @@ public:
 
     void copy_back()
     {
-        const auto local_host_size = partition_.get_local_host_size();
-        const auto local_device_size = partition_.get_local_device_size();
+        const auto local_size = partition_.get_local_size();
         auto global_size = partition_.get_total_size();
 
         auto comm = exec_.get_gko_mpi_host_comm();
@@ -222,27 +193,14 @@ public:
         // repartitioner size should be sufficient
         auto target_vector = dist_vec::create(
             exec_.get_ref_exec(), *comm.get(), gko::dim<2>{global_size, 1},
-            gko::dim<2>{local_host_size, 1}, 1);
+            gko::dim<2>{local_size, 1}, 1);
         scalar *host_buffer = host_device_vector->get_local_values();
 
-        // if (partition_.get_ranks_per_gpu() != 1) {
-        //     auto repartitioner = gko::share(
-        //         gko::experimental::distributed::repartitioner<label, label>::
-        //             create(*comm.get(),
-        //                    partition_.get_host_partition(),    // from
-        //                    partition_.get_device_partition(),  // to
-        //                    true));
-        //
-        //     repartitioner->scatter(host_device_vector.get(),
-        //                            target_vector.get());
-        //     host_buffer = target_vector->get_local_values();
-        // }
-
         auto host_buffer_view = gko::array<T>::view(
-            exec_.get_ref_exec(), local_host_size, host_buffer);
+            exec_.get_ref_exec(), local_size, host_buffer);
 
         auto to_view = gko::array<T>::view(
-            exec_.get_ref_exec(), local_host_size, const_cast<T *>(memory_));
+            exec_.get_ref_exec(), local_size, const_cast<T *>(memory_));
         to_view = host_buffer_view;
     }
 
diff --git a/common/common.H b/common/common.H
index f3b3bf194..1c5998d94 100644
--- a/common/common.H
+++ b/common/common.H
@@ -106,6 +106,8 @@ namespace Foam {
 
 #define UNUSED(x) (void)(x)
 
+#define OGL_NOT_IMPLEMENTED GKO_NOT_IMPLEMENTED
+
 std::ostream &operator<<(
     std::ostream &os,
     const std::shared_ptr<gko::matrix::Dense<scalar>> array_in);
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 0313002ef..ff1eee075 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -318,7 +318,7 @@ public:
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)
         auto bandwidth_copy_back = sizeof(scalar) *
-                                   partition.get_local_device_size() /
+                                   partition.get_local_size() /
                                    delta_t_copy_x_back / 1000.0;
 
         solverPerf.initialResidual() = this->get_init_res_norm();
diff --git a/test/data_validation.py b/test/data_validation.py
index d3905a45d..fa31ce15a 100644
--- a/test/data_validation.py
+++ b/test/data_validation.py
@@ -106,6 +106,54 @@ def verify_local_matrix(lines, file_path):
                 )
                 sys.exit(1)
 
+def verify_matrix_is_in_row_major_order(lines, file_path):
+    """This function checks if matrix is sorted in row major order
+    """
+    from scipy.io import mmread
+
+    # bool is_sorted_rows = true;
+    # bool is_sorted_cols = true;
+    # auto rows_data = rows->get_const_data();
+    # auto cols_data = cols->get_const_data();
+    # for (size_t i = 1; i < cols->get_num_elems(); i++) {
+    #     if (rows_data[i] < rows_data[i - 1]) {
+    #         is_sorted_rows = false;
+    #         Info << "rows sorting error element " << i << " row[i] "
+    #              << rows_data[i] << " row[i-1] " << rows_data[i - 1]
+    #              << endl;
+    #     }
+    #     // same row but subsequent column is smaller
+    #     if (cols_data[i] < cols_data[i - 1] &&
+    #         rows_data[i] == rows_data[i - 1]) {
+    #         is_sorted_cols = false;
+    #         Info << "cols sorting error element " << i << " row[i] "
+    #              << rows_data[i] << " row[i-1] " << rows_data[i - 1]
+    #              << " col[i] " << cols_data[i] << " col[i-1] "
+    #              << cols_data[i - 1] << endl;
+    #     }
+    # }
+    # bool is_sorted_rows = true;
+    # bool is_sorted_cols = true;
+    # auto rows_data = rows->get_const_data();
+    # auto cols_data = cols->get_const_data();
+    # for (size_t i = 1; i < cols->get_num_elems(); i++) {
+    #     if (rows_data[i] < rows_data[i - 1]) {
+    #         is_sorted_rows = false;
+    #         Info << "rows sorting error element " << i << " row[i] "
+    #              << rows_data[i] << " row[i-1] " << rows_data[i - 1]
+    #              << endl;
+    #     }
+    #     // same row but subsequent column is smaller
+    #     if (cols_data[i] < cols_data[i - 1] &&
+    #         rows_data[i] == rows_data[i - 1]) {
+    #         is_sorted_cols = false;
+    #         Info << "cols sorting error element " << i << " row[i] "
+    #              << rows_data[i] << " row[i-1] " << rows_data[i - 1]
+    #              << " col[i] " << cols_data[i] << " col[i-1] "
+    #              << cols_data[i - 1] << endl;
+    #     }
+    # }
+
 
 def test_matrix_value_bounds(workspace):
     """This function checks if matrices are coeffs are within bounds"""

From c7f32c6f5a6cab7fb144f78a7b637ca9e927ee10 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 3 Nov 2023 15:42:32 +0100
Subject: [PATCH 050/125] use local indices

---
 DevicePersistent/Base/Base.C                  |  2 +-
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       |  4 +-
 HostMatrix/HostMatrix.C                       | 48 +++----------------
 HostMatrix/HostMatrix.H                       |  1 +
 Solver/CG/GKOCG.H                             | 14 ++++--
 StoppingCriterion/StoppingCriterion.H         |  2 +
 common/common.C                               |  3 +-
 7 files changed, 24 insertions(+), 50 deletions(-)

diff --git a/DevicePersistent/Base/Base.C b/DevicePersistent/Base/Base.C
index 123dfbeb7..487e9ccc8 100644
--- a/DevicePersistent/Base/Base.C
+++ b/DevicePersistent/Base/Base.C
@@ -23,7 +23,7 @@ defineTemplateTypeNameWithName(DevicePersistentBase<gko::LinOp>,
 
 // typedef needed  to avoid confusion with the comma separated template
 // arguments as macro arguments
-typedef gko::experimental::distributed::Partition<label, label> Partition;
+typedef gko::experimental::distributed::localized_partition<label> Partition;
 defineTemplateTypeNameWithName(DevicePersistentBase<Partition>,
                                "PersistentPartition");
 
diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 49aa7f45d..383776a7c 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -307,13 +307,13 @@ struct MatrixInitFunctor {
 
         auto exec = exec_.get_ref_exec();
 
-        auto num_rows = partition_.get_total_size();
+        auto num_rows = partition_.get_local_size();
         gko::device_matrix_data<scalar, label> A_data(
             exec, gko::dim<2>(num_rows, num_rows), *rows.get(), *cols.get(),
             *coeffs.get());
 
         gko::device_matrix_data<scalar, label> non_local_A_data(
-            exec, gko::dim<2>(num_rows, num_rows), *non_local_rows.get(),
+            exec, gko::dim<2>(num_rows, non_local_cols->get_num_elems()), *non_local_rows.get(),
             *non_local_cols.get(), *non_local_coeffs.get());
 
 
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 4e272b1b3..2255967c5 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -183,7 +183,7 @@ HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
         auto search = reduce_map.find(proc);
         if (search == reduce_map.end()) {
             n_procs += 1;
-            reduce_map.insert(proc, n_faces);
+            reduce_map.insert(std::pair<label,label>{proc, n_faces});
         } else {
             reduce_map[proc] = reduce_map[proc] + n_faces;
         }
@@ -203,6 +203,11 @@ HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
     return std::pair(target_ids, target_sizes);
 }
 
+template
+std::pair<gko::array<label>, gko::array<label>>
+HostMatrixWrapper<lduMatrix>::assemble_proc_id_and_sizes(
+    const lduInterfaceFieldPtrsList &interfaces) const;
+
 template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
@@ -212,31 +217,6 @@ HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
     std::vector<std::tuple<label, label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
-    label startOfRequests = Pstream::nRequests();
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-
-        if (isA<processorLduInterface>(iface->interface())) {
-            const processorLduInterface &pldui =
-                refCast<const processorLduInterface>(iface->interface());
-            const label neighbProcNo = pldui.neighbProcNo();
-
-            word msg = "send face cells interface " + std::to_string(i) +
-                       " from proc " + std::to_string(Pstream::myProcNo()) +
-                       " to neighbour proc " + std::to_string(neighbProcNo);
-
-            LOG_2(verbose_, msg)
-            pldui.send(Pstream::commsTypes::nonBlocking, face_cells);
-        }
-    }
-    Pstream::waitRequests(startOfRequests);
-    LOG_2(verbose_, "send face cells done")
-
     label interface_ctr = 0;
     for (int i = 0; i < interfaces.size(); i++) {
         if (interface_getter(interfaces, i) == nullptr) {
@@ -248,23 +228,9 @@ HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
         const label interface_size = face_cells.size();
 
         if (isA<processorLduInterface>(iface->interface())) {
-            const processorLduInterface &pldui =
-                refCast<const processorLduInterface>(iface->interface());
-            const label neighbProcNo = pldui.neighbProcNo();
-
-            word msg_2 = "receive face cells interface " + std::to_string(i) +
-                         " from proc " + std::to_string(neighbProcNo);
-            LOG_2(verbose_, msg_2)
-
-            auto otherSide_tmp = pldui.receive<label>(
-                Pstream::commsTypes::nonBlocking, interface_size);
-            LOG_2(verbose_, "receive face cells done")
-
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto global_row = global_row_index_.toGlobal(
-                    neighbProcNo, otherSide_tmp()[cellI]);
                 non_local_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], global_row});
+                    {interface_ctr, face_cells[cellI], interface_ctr});
                 interface_ctr += 1;
             }
         }
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 8657519a9..9fb99043b 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -349,6 +349,7 @@ public:
 
     /* Iterates all interfaces and collects non-local global column indices
     **
+    ** TODO FIXME this is not uptodate
     ** Non-local global column indices are computed from the global cell-id
     ** (row) on the other side of an interface
     **
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 66e1636ed..7ad2ea3b5 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -68,9 +68,10 @@ public:
     std::unique_ptr<cg::Factory, std::default_delete<cg::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        // TODO FIXME
-        // auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
-        auto cg = cg::build().on(exec);
+        auto cg = cg::build().with_criteria(
+              gko::stop::Iteration::build().with_max_iters(20u)
+                // stoppingCriterionVec_
+                ).on(exec);
         return cg;
     };
 
@@ -79,8 +80,11 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto cg = cg::build()
-                      // TODO FIXME
-                      // .with_criteria(stoppingCriterionVec_)
+                      .with_criteria(
+                              gko::stop::Iteration::build().with_max_iters(20u)
+                              //
+                              // stoppingCriterionVec_
+                              )
                       .with_generated_preconditioner(precond)
                       .on(exec);
         return cg;
diff --git a/StoppingCriterion/StoppingCriterion.H b/StoppingCriterion/StoppingCriterion.H
index 64c8a7178..eda67ccbe 100644
--- a/StoppingCriterion/StoppingCriterion.H
+++ b/StoppingCriterion/StoppingCriterion.H
@@ -187,6 +187,8 @@ class StoppingCriterion {
                 *(parameters_.iter) += 1;
                 return false;
             }
+            // TODO FIXME
+            std::cout << "iteration\n ";
 
             const auto exec = this->get_executor();
 
diff --git a/common/common.C b/common/common.C
index 644485c28..ed9846fe6 100644
--- a/common/common.C
+++ b/common/common.C
@@ -62,8 +62,9 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
                 const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
-    std::filesystem::create_directories(folder);
+    //std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
+    std::cout  << "exporting " << fn <<  std::endl;
     std::ofstream stream{fn};
     stream << std::setprecision(15);
     if (matrixFormat == "Coo") {

From 3a5a687e2f66f02d6d907b8d84a1be4c6452a226 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 3 Nov 2023 15:59:18 +0100
Subject: [PATCH 051/125] fixup

---
 .github/workflows/integration-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 815925cb3..d5ad28029 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -56,7 +56,7 @@ jobs:
     - name: Get Ginkgo checkout version
       shell: bash
       run: |
-        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z]*" > GINKGO_CHECKOUT_VERSION
+        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
         export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
         echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
         echo "Ginkgo_DIR=${{inputs.foam_user_libbin}}/cmake/Ginkgo" >> $GITHUB_ENV

From 726420cc68d434ecafadfadddd799a1666a6eb41 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 07:47:46 +0100
Subject: [PATCH 052/125] compute send_idxs

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 187 +++++-------------
 DevicePersistent/Partition/Partition.H        |  45 +++--
 HostMatrix/HostMatrix.C                       | 145 +++++++++-----
 HostMatrix/HostMatrix.H                       |  63 +++---
 common/common.H                               |   4 +
 lduLduBase/lduLduBase.H                       |  55 +++---
 6 files changed, 237 insertions(+), 262 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 383776a7c..344948570 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -143,127 +143,19 @@ struct MatrixInitFunctor {
             non_local_value_ptr = non_local_coo_ptr->get_values();
         }
 
-        if (partition_.get_ranks_per_gpu() != 1) {
-            // TODO add field name
-            PersistentArray<label> local_scatter_map_pers{
-                "local_scatter_map",  //
-                db_,
-                exec_,
-                0,
-                verbose_,
-                false,
-                false};
-
-            // TODO add field name
-            PersistentArray<label> non_local_scatter_map_pers{
-                "non_local_scatter_map",  //
-                db_,
-                exec_,
-                0,
-                verbose_,
-                false,
-                false};
-            // TODO add field name
-            PersistentArray<label> sorting_idx_pers{
-                "sorting_idx", db_, exec_, 0, verbose_, false, false};
-
-            // TODO add field name
-            PersistentArray<label> send_sizes_pers{
-                "send_sizes", db_, exec_, 0, verbose_, false, false};
-
-            // TODO add field name
-            PersistentArray<label> send_offs_pers{"send_offs", db_,   exec_, 0,
-                                                  verbose_,    false, false};
-
-            // TODO add field name
-            PersistentArray<label> recv_sizes_pers{
-                "recv_sizes", db_, exec_, 0, verbose_, false, false};
-
-            // TODO add field name
-            PersistentArray<label> recv_offs_pers{"recv_offs", db_,   exec_, 0,
-                                                  verbose_,    false, false};
-
-
-            auto comm = exec_.get_gko_mpi_host_comm();
-            // TIME_WITH_FIELDNAME(
-            //     verbose_, update_build_repartitioner, field_name_,
-            //     auto repartitioner = gko::share(
-            //         gko::experimental::distributed::repartitioner<
-            //             label, label>::create(*comm.get(),
-            //                                   partition_.get_host_partition(),
-            //                                   partition_.get_device_partition(),
-            //                                   false));)
-
-            auto local_nnz =
-                local_scatter_map_pers.get_array()->get_num_elems();
-            auto non_local_nnz =
-                non_local_scatter_map_pers.get_array()->get_num_elems();
-
-            gko::array<scalar> local_values{exec, local_nnz};
-            gko::array<scalar> non_local_values{exec, non_local_nnz};
-
-            std::vector<int> send_sizes;
-            std::vector<int> send_offsets;
-            std::vector<int> recv_sizes;
-            std::vector<int> recv_offsets;
-
-            send_sizes.assign(send_sizes_pers.get_array()->get_data(),
-                              send_sizes_pers.get_array()->get_data() +
-                                  send_sizes_pers.get_array()->get_num_elems());
-            send_offsets.assign(
-                send_offs_pers.get_array()->get_data(),
-                send_offs_pers.get_array()->get_data() +
-                    send_offs_pers.get_array()->get_num_elems());
-            recv_sizes.assign(recv_sizes_pers.get_array()->get_data(),
-                              recv_sizes_pers.get_array()->get_data() +
-                                  recv_sizes_pers.get_array()->get_num_elems());
-            recv_offsets.assign(
-                recv_offs_pers.get_array()->get_data(),
-                recv_offs_pers.get_array()->get_data() +
-                    recv_offs_pers.get_array()->get_num_elems());
-
-
-            // TIME_WITH_FIELDNAME(
-            //     verbose_, update_repartitioned_existing, field_name_,
-            //     repartitioner->update_existing(
-            //         *row_idxs_.get_array(), *non_local_row_idxs_.get_array(),
-            //         *coeffs.get(), *non_local_coeffs.get(),
-            //         *sorting_idx_pers.get_array().get(), send_sizes,
-            //         send_offsets, recv_sizes, recv_offsets,
-            //         *local_scatter_map_pers.get_array().get(),
-            //         *non_local_scatter_map_pers.get_array().get(),
-            //         local_values, non_local_values);)
-
-            auto value_view =
-                val_array::view(device_exec, local_nnz, value_ptr);
-
-            auto non_local_view = val_array::view(device_exec, non_local_nnz,
-                                                  non_local_value_ptr);
-
-            TIME_WITH_FIELDNAME(verbose_, update_offload_local_values,
-                                field_name_,
-                                local_values.set_executor(device_exec);
-                                value_view = local_values;)
-
-            TIME_WITH_FIELDNAME(verbose_, update_offload_non_local_values,
-                                field_name_,
-                                non_local_values.set_executor(device_exec);
-                                non_local_view = non_local_values;)
-        } else {
-            gko::array<scalar> device_values{device_exec, *coeffs.get()};
-            gko::array<scalar> device_non_local_values{device_exec,
-                                                       *non_local_coeffs.get()};
-
-            auto value_view = val_array::view(
-                device_exec, device_values.get_num_elems(), value_ptr);
-
-            auto non_local_view = val_array::view(
-                device_exec, device_non_local_values.get_num_elems(),
-                non_local_value_ptr);
-
-            value_view = device_values;
-            non_local_view = device_non_local_values;
-        }
+        gko::array<scalar> device_values{device_exec, *coeffs.get()};
+        gko::array<scalar> device_non_local_values{device_exec,
+                                                   *non_local_coeffs.get()};
+
+        auto value_view = val_array::view(
+            device_exec, device_values.get_num_elems(), value_ptr);
+
+        auto non_local_view = val_array::view(
+            device_exec, device_non_local_values.get_num_elems(),
+            non_local_value_ptr);
+
+        value_view = device_values;
+        non_local_view = device_non_local_values;
     }
 
     std::shared_ptr<dist_mtx> generate_dist_mtx_with_inner_type(
@@ -294,7 +186,8 @@ struct MatrixInitFunctor {
     std::shared_ptr<dist_mtx> init() const
     {
         const label local_size = partition_.get_local_size();
-        word msg{"init global csr matrix of size " + std::to_string(local_size)};
+        word msg{"init global csr matrix of size " +
+                 std::to_string(local_size)};
         LOG_1(verbose_, msg)
 
         auto coeffs = coeffs_.get_array();
@@ -313,26 +206,50 @@ struct MatrixInitFunctor {
             *coeffs.get());
 
         gko::device_matrix_data<scalar, label> non_local_A_data(
-            exec, gko::dim<2>(num_rows, non_local_cols->get_num_elems()), *non_local_rows.get(),
-            *non_local_cols.get(), *non_local_coeffs.get());
+            exec, gko::dim<2>(num_rows, non_local_cols->get_num_elems()),
+            *non_local_rows.get(), *non_local_cols.get(),
+            *non_local_coeffs.get());
 
 
         auto comm = exec_.get_gko_mpi_host_comm();
         auto dist_A = generate_dist_mtx_with_inner_type(
             exec_.get_ref_exec(), exec_.get_gko_mpi_host_comm());
-        dist_A->read_distributed(A_data, non_local_A_data,
-                gko::experimental::distributed::sparse_communicator::create(
-                    *comm.get(),
-                                 partition_.get_localized_partition()
-                                 ));
-
-        if (partition_.get_ranks_per_gpu() == 1) {
-            auto device_mat = generate_dist_mtx_with_inner_type(
-                exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
-            dist_A->move_to(device_mat.get());
-            return device_mat;
+
+        auto sparse_comm =
+            gko::experimental::distributed::sparse_communicator::create(
+                *comm.get(), partition_.get_localized_partition());
+
+        // TODO FIXME
+        auto recv_sizes = sparse_comm->get_recv_sizes();
+        auto recv_offs = sparse_comm->get_recv_offsets();
+        auto send_sizes = sparse_comm->get_send_sizes();
+        auto send_offs = sparse_comm->get_send_offsets();
+
+        if (Pstream::master() ) {
+        for (auto e: recv_sizes) {
+            std::cout << "recv_size: " << e << std::endl;
+        }
+
+        for (auto e: recv_offs) {
+            std::cout << "recv_offs: " << e << std::endl;
+        }
+
+        for (auto e: send_sizes) {
+            std::cout << "send_sizes: " << e << std::endl;
+        }
+
+        for (auto e: send_offs) {
+            std::cout << "send_offs: " << e << std::endl;
+        }
         }
 
+
+        dist_A->read_distributed(A_data, non_local_A_data, sparse_comm);
+
+        auto device_mat = generate_dist_mtx_with_inner_type(
+            exec_.get_device_exec(), exec_.get_gko_mpi_device_comm());
+        dist_A->move_to(device_mat.get());
+        return device_mat;
     }
 };
 
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 13743ef50..eb8c8f9df 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -59,12 +59,14 @@ struct PartitionInitFunctor {
     // Array storing the ranks of the neighbouring processors
     const gko::array<label> target_sizes_;
 
-
-    PartitionInitFunctor(const ExecutorHandler &exec, const label rows_start,
-                         const label rows_end, const label verbose,
-                         const int ranks_per_gpu,
-                         const gko::array<label> &target_ids,
-                         const gko::array<label> target_sizes)
+    const std::vector<std::pair<gko::index_set<label>, label>> send_idxs_;
+
+    PartitionInitFunctor(
+        const ExecutorHandler &exec, const label rows_start,
+        const label rows_end, const label verbose, const int ranks_per_gpu,
+        const gko::array<label> &target_ids,
+        const gko::array<label> &target_sizes,
+        const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : exec_(exec),
           rows_end_(rows_end),
           rows_start_(rows_start),
@@ -72,9 +74,9 @@ struct PartitionInitFunctor {
           verbose_(verbose),
           ranks_per_gpu_(ranks_per_gpu),
           target_ids_(target_ids),
-          target_sizes_(target_sizes)
-    {
-    }
+          target_sizes_(target_sizes),
+          send_idxs_(send_idxs)
+    {}
 
     void update(std::shared_ptr<part_type> persistent_partition) const
     {
@@ -90,11 +92,9 @@ struct PartitionInitFunctor {
                  std::to_string(local_size_)};
         LOG_1(verbose_, msg)
 
-        std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
-
         auto host_partition = gko::experimental::distributed::
             localized_partition<>::build_from_blocked_recv(
-                exec, local_size_, send_idxs, target_ids_, target_sizes_);
+                exec, local_size_, send_idxs_, target_ids_, target_sizes_);
 
         if (ranks_per_gpu_ == 1) return host_partition;
     }
@@ -137,15 +137,17 @@ public:
      * @param offset
      * @param elements
      */
-    PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
-                        const label verbose, const int ranks_per_gpu,
-                        const label offset, const label elements,
-                        const gko::array<label> &target_ids,
-                        const gko::array<label> target_sizes)
+    PersistentPartition(
+        const objectRegistry &db, const ExecutorHandler &exec,
+        const label verbose, const int ranks_per_gpu, const label offset,
+        const label elements, const gko::array<label> &target_ids,
+        const gko::array<label> &target_sizes,
+        const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, offset, offset + elements, verbose,
-                                   ranks_per_gpu, target_ids, target_sizes),
+                                   ranks_per_gpu, target_ids, target_sizes,
+                                   send_idxs),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           offset_(offset),
@@ -156,7 +158,7 @@ public:
         label local_elements = local_elements_;
         comm->all_reduce(exec.get_ref_exec(), &local_elements, 1, MPI_SUM);
         global_elements_ = local_elements;
-          }
+    }
 
     std::shared_ptr<part_type> get_localized_partition() const
     {
@@ -166,10 +168,7 @@ public:
     // number of elements on this rank on the host
     label get_local_size() const { return local_elements_; }
 
-    label get_total_size() const
-    {
-        return global_elements_;
-    }
+    label get_total_size() const { return global_elements_; }
 
     label get_ranks_per_gpu() const { return ranks_per_gpu_; }
 };
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 2255967c5..196832952 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -102,60 +102,18 @@ HostMatrixWrapper<lduMatrix>::collect_interface_coeffs(
     const lduInterfaceFieldPtrsList &, const FieldField<Field, scalar> &,
     const bool) const;
 
-template <class MatrixType>
-std::vector<std::tuple<label, label, label>>
-HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
-    const lduInterfaceFieldPtrsList &interfaces) const
-{
-    std::vector<std::tuple<label, label, label>> local_interface_idxs{};
-    local_interface_idxs.reserve(local_interface_nnz_);
-
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        // TODO make this a separate specialized function
-        label interface_ctr = 0;
-        if (isA<cyclicLduInterface>(iface->interface())) {
-            const cyclicLduInterface &pldui =
-                refCast<const cyclicLduInterface>(iface->interface());
-            // const labelUList &rows = this->matrix().lduAddr().patchAddr(i);
-
-#ifdef WITH_ESI_VERSION
-            const label neighbPatchId = pldui.neighbPatchID();
-#else
-            const label neighbPatchId = pldui.nbrPatchID();
-#endif
-            const labelUList &cols =
-                this->matrix().lduAddr().patchAddr(neighbPatchId);
-            for (label cellI = 0; cellI < interface_size; cellI++) {
-                // DEBUG
-                // cout << "patch_ctr " << cellI << " ictr " << interface_ctr
-                //     << " patchId " << i << " neighPatchId " << neighbPatchId
-                //      << " row " << rows[cellI] << " col " << cols[cellI]
-                //      << "\n" << endl;
-                local_interface_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], cols[cellI]});
-                interface_ctr += 1;
-            }
-        }
-    }
-    return local_interface_idxs;
-}
 
 template <class MatrixType>
-std::pair<gko::array<label>, gko::array<label>>
-HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
+CommunicationPattern
+HostMatrixWrapper<MatrixType>::create_communication_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // temp vector to store neighbour proc number and number of cells to send
     std::vector<std::pair<label, label>> neighbour_procs{};
+
+    // temp map, mapping from neighbour rank interface cells
+    std::map<label, std::vector<label>> interface_cell_map{};
+
     // TODO FIXME use a lambda to avoid repeating
     for (int i = 0; i < interfaces.size(); i++) {
         if (interface_getter(interfaces, i) == nullptr) {
@@ -173,22 +131,54 @@ HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
 
             neighbour_procs.push_back(
                 std::pair<label, label>{neighbProcNo, interface_size});
+
+            //
+            auto search = interface_cell_map.find(neighbProcNo);
+            if (search == interface_cell_map.end()) {
+                interface_cell_map.insert(std::pair{
+                    neighbProcNo,
+                    std::vector<label>(face_cells.begin(), face_cells.end())});
+            } else {
+                // TODO FIXME reserve size
+                auto cur_face_cells = interface_cell_map[neighbProcNo];
+                for (auto face_cell : face_cells) {
+                    cur_face_cells.push_back(face_cell);
+                }
+                interface_cell_map[neighbProcNo] = cur_face_cells;
+            }
         }
     }
 
-    // reduce vector
+    // reduce vector of sizes
+    // this assumes that a rank can be connected to the same neighbour through
+    // different interfaces this migth not be necessary
     label n_procs = 0;
     std::map<label, label> reduce_map{};
     for (auto [proc, n_faces] : neighbour_procs) {
         auto search = reduce_map.find(proc);
         if (search == reduce_map.end()) {
             n_procs += 1;
-            reduce_map.insert(std::pair<label,label>{proc, n_faces});
+            reduce_map.insert(std::pair<label, label>{proc, n_faces});
         } else {
             reduce_map[proc] = reduce_map[proc] + n_faces;
         }
     }
 
+    // create index_sets
+    // currently this assumes that there is only one interface to a given
+    // neighbour rank
+    std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
+    for (auto [proc, interface_cells] : interface_cell_map) {
+        label interface_size = interface_cells.size();
+        auto exec = exec_.get_ref_exec();
+        send_idxs.push_back(std::pair<gko::index_set<label>, label>(
+            gko::index_set(
+                exec, interface_size,
+                gko::array<label>::view(exec, interface_size, interface_cells.data()),
+                true),
+            proc));
+    }
+
     // convert to gko::array
     gko::array<label> target_ids{exec_.get_ref_exec(), n_procs};
     gko::array<label> target_sizes{exec_.get_ref_exec(), n_procs};
@@ -200,14 +190,61 @@ HostMatrixWrapper<MatrixType>::assemble_proc_id_and_sizes(
         iter++;
     }
 
-    return std::pair(target_ids, target_sizes);
+    return CommunicationPattern{target_ids, target_sizes, send_idxs};
 }
 
-template
-std::pair<gko::array<label>, gko::array<label>>
-HostMatrixWrapper<lduMatrix>::assemble_proc_id_and_sizes(
+template CommunicationPattern
+HostMatrixWrapper<lduMatrix>::create_communication_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const;
 
+
+template <class MatrixType>
+std::vector<std::tuple<label, label, label>>
+HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
+    const lduInterfaceFieldPtrsList &interfaces) const
+{
+    std::vector<std::tuple<label, label, label>> local_interface_idxs{};
+    local_interface_idxs.reserve(local_interface_nnz_);
+
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+
+        const auto iface{interface_getter(interfaces, i)};
+
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        // TODO make this a separate specialized function
+        label interface_ctr = 0;
+        if (isA<cyclicLduInterface>(iface->interface())) {
+            const cyclicLduInterface &pldui =
+                refCast<const cyclicLduInterface>(iface->interface());
+            // const labelUList &rows = this->matrix().lduAddr().patchAddr(i);
+
+#ifdef WITH_ESI_VERSION
+            const label neighbPatchId = pldui.neighbPatchID();
+#else
+            const label neighbPatchId = pldui.nbrPatchID();
+#endif
+            const labelUList &cols =
+                this->matrix().lduAddr().patchAddr(neighbPatchId);
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                // DEBUG
+                // cout << "patch_ctr " << cellI << " ictr " << interface_ctr
+                //     << " patchId " << i << " neighPatchId " << neighbPatchId
+                //      << " row " << rows[cellI] << " col " << cols[cellI]
+                //      << "\n" << endl;
+                local_interface_idxs.push_back(
+                    {interface_ctr, face_cells[cellI], cols[cellI]});
+                interface_ctr += 1;
+            }
+        }
+    }
+    return local_interface_idxs;
+}
+
 template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 9fb99043b..fffb62684 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -39,6 +39,7 @@ SourceFiles
 namespace Foam {
 
 
+    // TODO Move this somewhere else
 struct PersistentSparsityPattern {
     PersistentSparsityPattern(
       const word &fieldName,
@@ -84,6 +85,20 @@ struct PersistentSparsityPattern {
     mutable PersistentArray<label> ldu_mapping_;
 };
 
+struct CommunicationPattern {
+    // an array storing to which rank to communicate
+    gko::array<label> target_ids;
+
+    // an array storing how many elements to communicate
+    // to the corresponding target_id
+    gko::array<label> target_sizes;
+
+    // send_idx stores the index_set ie which cells
+    // are owned by the interface and the corresponding target rank
+    std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
+};
+
+
 /* The HostMatrixWrapper class handles the conversion from OpenFOAMs lduMatrix
  * format into Ginkgo array data structures
  *
@@ -138,9 +153,7 @@ private:
     // non-local indices
     const label non_local_matrix_nnz_;
 
-    // Array storing the ranks of the neighbouring processors
-    const std::pair<gko::array<label>, gko::array<label>>
-        proc_target_id_and_sizes_;
+    const CommunicationPattern communication_pattern_;
 
     mutable PersistentSparsityPattern non_local_sparsity_;
 
@@ -200,7 +213,7 @@ public:
                      // matrix it will be put on the device
           },
           non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
-          proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
+          communication_pattern_(create_communication_pattern(interfaces)),
           non_local_sparsity_{
               fieldName + "_non_local", db,       exec_,
               non_local_matrix_nnz_,    verbose_,
@@ -311,42 +324,46 @@ public:
         const bool local) const;
 
 
-    /* Iterates all local interfaces and returns the relative order and
-     *corresponding row and column indices
-     **
-     */
-    std::vector<std::tuple<label, label, label>>
-    collect_local_interface_indices(
-        const lduInterfaceFieldPtrsList &interfaces_) const;
-
-
     /* Iterates all interfaces and counts the number of elements
     **
     ** @param interfaces The list of interfaces for the search
     ** @param proc_interfaces Count only elements on (true)
     *processorLduInterfaces or exclude processorLduInterfaces (false)
+     ** TODO make private and no discard
     */
     label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
                               bool proc_interfaces) const;
 
-    /* Iterates all interfaces and counts the number of unique neighbour
-     *processors and number of interfaces in total for this processor
+     /** Iterates all local interfaces and returns the relative order and
+     **corresponding row and column indices
+     **
+     ** @return vector of tuples containing the interface number, the local row, the local column
+     **/
+    std::vector<std::tuple<label, label, label>>
+    collect_local_interface_indices(
+        const lduInterfaceFieldPtrsList &interfaces_) const;
+
+
+    /** Iterates all interfaces and counts the number of unique neighbour
+     **processors and number of interfaces in total for this processor
+     ** and collects all interface cells of this rank
      **
      ** @param interfaces The list of interfaces for the search
+     ** @return the CommunicationPattern 
+     ** TODO make private and no discard
      */
-    std::pair<gko::array<label>, gko::array<label>> assemble_proc_id_and_sizes(
+    CommunicationPattern create_communication_pattern(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
-    /* Getter for proc_id and sizes arrays
-     *
-     */
-    const std::pair<gko::array<label>, gko::array<label>>
-        &get_proc_target_id_and_sizes() const
+    /** Getter for the CommunicationPattern
+     **
+     ** @return the CommunicationPattern 
+     **/
+    const CommunicationPattern &get_communication_pattern() const
     {
-        return proc_target_id_and_sizes_;
+        return communication_pattern_;
     };
 
-
     /* Iterates all interfaces and collects non-local global column indices
     **
     ** TODO FIXME this is not uptodate
diff --git a/common/common.H b/common/common.H
index 1c5998d94..5100477e6 100644
--- a/common/common.H
+++ b/common/common.H
@@ -60,6 +60,10 @@ namespace Foam {
         }                                                                   \
     }
 
+
+#define IN_USE_DO_NOT_REMOVE(FUNC) \
+std::cout << FUNC <<  " in "   << __FILENAME__ << ":" <<__LINE__ << " is in use do not remove"  << std::endl
+
 #define LOG_0(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 0, MSG, 0)
 #define LOG_1(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 1, MSG, 0)
 #define LOG_2(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 2, MSG, 0)
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index ff1eee075..8548f7f7e 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -123,7 +123,8 @@ public:
             int rank;
             MPI_Comm_rank(MPI_COMM_WORLD, &rank);
             word fn = "/tmp/mpi_debug_" + std::to_string(rank) + ".pid";
-            word msg = "writing pid to" + fn;
+            word msg =
+                "writing pid to" + fn + " waiting 20s for debugger to attach";
             LOG_0(verbose_, msg)
             std::ofstream os(fn);
             os << getpid() << std::endl;
@@ -216,8 +217,8 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
-        const auto &[target_ids, target_sizes] =
-            this->get_proc_target_id_and_sizes();
+        const auto communication_pattern = this->get_communication_pattern();
+
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
@@ -225,8 +226,9 @@ public:
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
             this->get_global_row_index().offset(Pstream::myProcNo()),  // offset
             psi.size(),  // number of elements
-            target_ids,
-            target_sizes};
+            communication_pattern.target_ids,
+            communication_pattern.target_sizes,
+            communication_pattern.send_idxs};
 
         // TODO this is now the local matrix part for dist_A
         MatrixWrapper dist_A{db_,
@@ -317,33 +319,32 @@ public:
 
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)
-        auto bandwidth_copy_back = sizeof(scalar) *
-                                   partition.get_local_size() /
+        auto bandwidth_copy_back = sizeof(scalar) * partition.get_local_size() /
                                    delta_t_copy_x_back / 1000.0;
 
         solverPerf.initialResidual() = this->get_init_res_norm();
         solverPerf.finalResidual() = this->get_res_norm();
         solverPerf.nIterations() = this->get_number_of_iterations();
-        this->store_number_of_iterations();
-        auto time_for_res_norm_eval = this->get_res_norm_time();
-        auto time_per_iter =
-            delta_t_solve / max(this->get_number_of_iterations(), 1);
-        scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
-        this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
-            this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1, 0);
-        this->set_prev_rel_res_cost(prev_rel_res_cost);
-        auto time_per_iter_and_dof =
-            time_per_iter * 1000.0 / partition.get_total_size();
-        word msg =
-            "\nStatistics:\n\tTime per iteration: " +
-            std::to_string(time_per_iter) +
-            std::string(" [mu s]\n\tTime per residual norm calculation: ") +
-            std::to_string(time_for_res_norm_eval) +
-            std::string(" [mu s]\n\tTime per iteration and DOF: ") +
-            std::to_string(time_per_iter_and_dof) + std::string(" [ns]") +
-            std::string("\n\tRetrieve results bandwidth ") +
-            std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
-        MLOG_0(verbose_, msg)
+        // this->store_number_of_iterations();
+        // auto time_for_res_norm_eval = this->get_res_norm_time();
+        // auto time_per_iter =
+        //     delta_t_solve / max(this->get_number_of_iterations(), 1);
+        // scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
+        // this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
+        //     this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1, 0);
+        // this->set_prev_rel_res_cost(prev_rel_res_cost);
+        // auto time_per_iter_and_dof =
+        //     time_per_iter * 1000.0 / partition.get_total_size();
+        // word msg =
+        //     "\nStatistics:\n\tTime per iteration: " +
+        //     std::to_string(time_per_iter) +
+        //     std::string(" [mu s]\n\tTime per residual norm calculation: ") +
+        //     std::to_string(time_for_res_norm_eval) +
+        //     std::string(" [mu s]\n\tTime per iteration and DOF: ") +
+        //     std::to_string(time_per_iter_and_dof) + std::string(" [ns]") +
+        //     std::string("\n\tRetrieve results bandwidth ") +
+        //     std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
+        // MLOG_0(verbose_, msg)
 
         return solverPerf;
     }

From d38eca9944a9c2ba3fca5d66522fa2745f246235 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 08:02:24 +0100
Subject: [PATCH 053/125] reorder header

---
 HostMatrix/HostMatrix.H | 151 ++++++++++++++++++++--------------------
 1 file changed, 76 insertions(+), 75 deletions(-)

diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index fffb62684..29bcf6811 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -159,6 +159,7 @@ private:
 
     mutable PersistentArray<scalar> non_local_coeffs_;
 
+    // FIXME TODO unused
     const word permutation_matrix_name_;
 
     const bool permutation_stored_;
@@ -166,6 +167,70 @@ private:
     mutable std::shared_ptr<gko::LinOp> P_;
 
 
+    /* Iterates all interfaces and collects the coefficients into a vector
+    **
+    ** @param local whether local or non local coefficients should be collected
+    */
+    std::vector<scalar> collect_interface_coeffs(
+        const lduInterfaceFieldPtrsList &interfaces_,
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        const bool local) const;
+
+
+    /* Iterates all interfaces and counts the number of elements
+    **
+    ** @param interfaces The list of interfaces for the search
+    ** @param proc_interfaces Count only elements on (true)
+    *processorLduInterfaces or exclude processorLduInterfaces (false)
+     ** TODO make private and no discard
+    */
+    label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
+                              bool proc_interfaces) const;
+
+     /** Iterates all local interfaces and returns the relative order and
+     **corresponding row and column indices
+     **
+     ** @return vector of tuples containing the interface number, the local row, the local column
+     **/
+    std::vector<std::tuple<label, label, label>>
+    collect_local_interface_indices(
+        const lduInterfaceFieldPtrsList &interfaces_) const;
+
+
+    /** Iterates all interfaces and counts the number of unique neighbour
+     **processors and number of interfaces in total for this processor
+     ** and collects all interface cells of this rank
+     **
+     ** @param interfaces The list of interfaces for the search
+     ** @return the CommunicationPattern
+     ** TODO make private and no discard
+     */
+    CommunicationPattern create_communication_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /* Iterates all interfaces and collects non-local global column indices
+    **
+    ** TODO FIXME this is not uptodate
+    ** Non-local global column indices are computed from the global cell-id
+    ** (row) on the other side of an interface
+    **
+    ** @return vector off size nnz_non_local, with sections for each interface
+    ** ret = [(1,2,3),(10,20,30), ...]
+    **         i0      i1,       i...
+    */
+    std::vector<std::tuple<label, label, label>> collect_non_local_col_indices(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    /* Based on OpenFOAMs ldu matrix format this function computes two
+     *consecutive index arrays in row major ordering and scattering indices
+     **
+     */
+    void init_local_sparsity_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
+    void init_non_local_sparsity_pattern(
+        const lduInterfaceFieldPtrsList &interfaces) const;
+
 public:
     // segregated wrapper constructor
     // TODO - compute local_interface_nnzs these are matrix coefficient
@@ -314,94 +379,31 @@ public:
         // TODO reimplement this
     }
 
-    /* Iterates all interfaces and collects the coefficients into a vector
-    **
-    ** @param local whether local or non local coefficients should be collected
-    */
-    std::vector<scalar> collect_interface_coeffs(
-        const lduInterfaceFieldPtrsList &interfaces_,
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        const bool local) const;
 
+    // updater
 
-    /* Iterates all interfaces and counts the number of elements
-    **
-    ** @param interfaces The list of interfaces for the search
-    ** @param proc_interfaces Count only elements on (true)
-    *processorLduInterfaces or exclude processorLduInterfaces (false)
-     ** TODO make private and no discard
-    */
-    label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
-                              bool proc_interfaces) const;
+    void update_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
-     /** Iterates all local interfaces and returns the relative order and
-     **corresponding row and column indices
-     **
-     ** @return vector of tuples containing the interface number, the local row, the local column
-     **/
-    std::vector<std::tuple<label, label, label>>
-    collect_local_interface_indices(
-        const lduInterfaceFieldPtrsList &interfaces_) const;
+    void update_non_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
+    // getter
+    public:
 
-    /** Iterates all interfaces and counts the number of unique neighbour
-     **processors and number of interfaces in total for this processor
-     ** and collects all interface cells of this rank
-     **
-     ** @param interfaces The list of interfaces for the search
-     ** @return the CommunicationPattern 
-     ** TODO make private and no discard
-     */
-    CommunicationPattern create_communication_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
+    bool get_verbose() const { return verbose_; }
 
     /** Getter for the CommunicationPattern
      **
-     ** @return the CommunicationPattern 
+     ** @return the CommunicationPattern
      **/
     const CommunicationPattern &get_communication_pattern() const
     {
         return communication_pattern_;
     };
 
-    /* Iterates all interfaces and collects non-local global column indices
-    **
-    ** TODO FIXME this is not uptodate
-    ** Non-local global column indices are computed from the global cell-id
-    ** (row) on the other side of an interface
-    **
-    ** @return vector off size nnz_non_local, with sections for each interface
-    ** ret = [(1,2,3),(10,20,30), ...]
-    **         i0      i1,       i...
-    */
-    std::vector<std::tuple<label, label, label>> collect_non_local_col_indices(
-        const lduInterfaceFieldPtrsList &interfaces) const;
-
-    /* Based on OpenFOAMs ldu matrix format this function computes two
-     *consecutive index arrays in row major ordering and scattering indices
-     **
-     */
-    void init_local_sparsity_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
-
-    void init_non_local_sparsity_pattern(
-        const lduInterfaceFieldPtrsList &interfaces) const;
-
-
-    bool get_verbose() const { return verbose_; }
-
-    // updater
-
-    void update_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
-
-    void update_non_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
-
-    // getter
-
     label get_local_nrows() const { return nrows_; }
 
     label get_global_nrows() const { return global_row_index_.size(); }
@@ -413,7 +415,6 @@ public:
         return global_row_index_;
     }
 
-
     PersistentArray<scalar> &get_local_coeffs() const  //
     {
         return local_coeffs_;

From 794957961000470862657821f895f7112e4cb7a5 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 08:42:41 +0100
Subject: [PATCH 054/125] move ctr to body

---
 HostMatrix/HostMatrix.C | 123 +++++++++++++++++++++++++++++++---------
 1 file changed, 95 insertions(+), 28 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 196832952..a0dabdea6 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -62,8 +62,89 @@ label HostMatrixWrapper<MatrixType>::count_interface_nnz(
     return ctr;
 }
 
-template label HostMatrixWrapper<lduMatrix>::count_interface_nnz(
-    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const;
+
+template <class MatrixType>
+HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
+        const objectRegistry &db, const MatrixType &matrix,
+        // coeffs for cells on boundaries
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        // coeffs for internal cells
+        const FieldField<Field, scalar> &interfaceIntCoeffs,
+        // pointers to interfaces can be used to access concrete
+        // functions such as transferring indices, patch neighbours etc
+        const lduInterfaceFieldPtrsList &interfaces,
+        const dictionary &solverControls, const word &fieldName)
+        : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
+                             interfaceIntCoeffs, interfaces, solverControls),
+          exec_{db, solverControls, fieldName},
+          device_id_guard_{db, fieldName, exec_.get_device_exec()},
+          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+          scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
+          nrows_(matrix.diag().size()),
+          local_interface_nnz_(count_interface_nnz(interfaces, false)),
+          upper_nnz_(matrix.lduAddr().upperAddr().size()),
+          non_diag_nnz_(2 * upper_nnz_),
+          local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
+          local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
+                                         local_interface_nnz_),
+          global_row_index_{nrows_},
+          local_sparsity_{
+              fieldName + "_local",           db,       exec_,
+              local_matrix_w_interfaces_nnz_, verbose_,
+          },
+          local_coeffs_{
+              fieldName + "_local_coeffs",
+              db,
+              exec_,
+              local_matrix_w_interfaces_nnz_,
+              verbose_,
+              true,  // needs to be updated
+              false  // leave it on host once it is turned into a distributed
+                     // matrix it will be put on the device
+          },
+          non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
+          communication_pattern_(create_communication_pattern(interfaces)),
+          non_local_sparsity_{
+              fieldName + "_non_local", db,       exec_,
+              non_local_matrix_nnz_,    verbose_,
+          },
+          non_local_coeffs_{
+              fieldName + "_non_local_coeffs",
+              db,
+              exec_,
+              non_local_matrix_nnz_,
+              verbose_,
+              true,  // needs to be updated
+              false  // leave it on host once it is turned into a distributed
+                     // matrix it will be put on the device
+          },
+          permutation_matrix_name_{"PermutationMatrix"},
+          permutation_stored_{
+              db.template foundObject<regIOobject>(permutation_matrix_name_)},
+          P_{(permutation_stored_) ? db.template lookupObjectRef<
+                                           DevicePersistentBase<gko::LinOp>>(
+                                           permutation_matrix_name_)
+                                         .get_ptr()
+                                   : nullptr}
+    {
+        if (!local_sparsity_.col_idxs_.get_stored() ||
+            local_sparsity_.col_idxs_.get_update()) {
+            TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
+                                this->fieldName(),
+                                init_local_sparsity_pattern(interfaces);)
+            TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
+                                this->fieldName(),
+                                init_non_local_sparsity_pattern(interfaces);)
+        }
+        if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
+            TIME_WITH_FIELDNAME(
+                verbose_, update_local_matrix_data, this->fieldName(),
+                update_local_matrix_data(interfaces, interfaceBouCoeffs);)
+            TIME_WITH_FIELDNAME(
+                verbose_, update_non_local_matrix_data, this->fieldName(),
+                update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
+        }
+    }
 
 
 template <class MatrixType>
@@ -97,12 +178,6 @@ std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
 }
 
 
-template std::vector<scalar>
-HostMatrixWrapper<lduMatrix>::collect_interface_coeffs(
-    const lduInterfaceFieldPtrsList &, const FieldField<Field, scalar> &,
-    const bool) const;
-
-
 template <class MatrixType>
 CommunicationPattern
 HostMatrixWrapper<MatrixType>::create_communication_pattern(
@@ -193,10 +268,6 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     return CommunicationPattern{target_ids, target_sizes, send_idxs};
 }
 
-template CommunicationPattern
-HostMatrixWrapper<lduMatrix>::create_communication_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const;
-
 
 template <class MatrixType>
 std::vector<std::tuple<label, label, label>>
@@ -278,10 +349,6 @@ HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
     return non_local_idxs;
 }
 
-template std::vector<std::tuple<label, label, label>>
-HostMatrixWrapper<lduMatrix>::collect_non_local_col_indices(
-    const lduInterfaceFieldPtrsList &interfaces) const;
-
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
@@ -326,9 +393,6 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     }
 }
 
-template void HostMatrixWrapper<lduMatrix>::init_non_local_sparsity_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const;
-
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
@@ -492,9 +556,6 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     LOG_1(verbose_, "done init host matrix")
 }
 
-template void HostMatrixWrapper<lduMatrix>::init_local_sparsity_pattern(
-    const lduInterfaceFieldPtrsList &interfaces) const;
-
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
@@ -590,10 +651,6 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     }
 }
 
-template void HostMatrixWrapper<lduMatrix>::update_local_matrix_data(
-    const lduInterfaceFieldPtrsList &interfaces,
-    const FieldField<Field, scalar> &interfaceBouCoeffs) const;
-
 
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
@@ -636,6 +693,16 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     i_device_view = tmp_contiguous_iface;
 }
 
-template void HostMatrixWrapper<lduMatrix>::update_non_local_matrix_data(
-    const lduInterfaceFieldPtrsList &, const FieldField<Field, scalar> &) const;
+template 
+HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
+        const objectRegistry &db, const lduMatrix &matrix,
+        // coeffs for cells on boundaries
+        const FieldField<Field, scalar> &interfaceBouCoeffs,
+        // coeffs for internal cells
+        const FieldField<Field, scalar> &interfaceIntCoeffs,
+        // pointers to interfaces can be used to access concrete
+        // functions such as transferring indices, patch neighbours etc
+        const lduInterfaceFieldPtrsList &interfaces,
+        const dictionary &solverControls, const word &fieldName);
+
 }  // namespace Foam

From f840fe24d283a6b53e393ccd6b060f6afe0fed78 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 08:53:31 +0100
Subject: [PATCH 055/125] remove gko globalIndex

---
 DevicePersistent/Partition/Partition.H | 23 ++-----
 HostMatrix/HostMatrix.C                |  1 -
 HostMatrix/HostMatrix.H                | 86 +-------------------------
 lduLduBase/lduLduBase.H                |  1 -
 4 files changed, 8 insertions(+), 103 deletions(-)

diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index eb8c8f9df..beabad5c6 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -43,10 +43,6 @@ struct PartitionInitFunctor {
 
     const ExecutorHandler &exec_;
 
-    const label rows_end_;
-
-    const label rows_start_;
-
     const label local_size_;
 
     const label verbose_;
@@ -62,15 +58,13 @@ struct PartitionInitFunctor {
     const std::vector<std::pair<gko::index_set<label>, label>> send_idxs_;
 
     PartitionInitFunctor(
-        const ExecutorHandler &exec, const label rows_start,
-        const label rows_end, const label verbose, const int ranks_per_gpu,
+        const ExecutorHandler &exec, const label local_size
+        , const label verbose, const int ranks_per_gpu,
         const gko::array<label> &target_ids,
         const gko::array<label> &target_sizes,
         const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : exec_(exec),
-          rows_end_(rows_end),
-          rows_start_(rows_start),
-          local_size_(rows_end_ - rows_start_),
+          local_size_(local_size),
           verbose_(verbose),
           ranks_per_gpu_(ranks_per_gpu),
           target_ids_(target_ids),
@@ -107,9 +101,6 @@ struct PartitionInitFunctor {
  * Here device partitioning refers to the partitioning as used on for the Ginkgo
  * data structures and can also reside on the host if the executor is either
  * reference or omp.
- *
- *
- *
  * */
 class PersistentPartition
     : public PersistentBase<
@@ -117,15 +108,12 @@ class PersistentPartition
           PartitionInitFunctor> {
     const int ranks_per_gpu_;
 
-    const label offset_;
-
     const label local_elements_;
 
     mutable label global_elements_;
 
     const ExecutorHandler &exec_;
 
-
 public:
     using part_type = gko::experimental::distributed::localized_partition<>;
     /* PersistentPartition constructor using existing memory
@@ -139,18 +127,17 @@ public:
      */
     PersistentPartition(
         const objectRegistry &db, const ExecutorHandler &exec,
-        const label verbose, const int ranks_per_gpu, const label offset,
+        const label verbose, const int ranks_per_gpu,
         const label elements, const gko::array<label> &target_ids,
         const gko::array<label> &target_sizes,
         const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
-              PartitionInitFunctor(exec, offset, offset + elements, verbose,
+              PartitionInitFunctor(exec, elements, verbose,
                                    ranks_per_gpu, target_ids, target_sizes,
                                    send_idxs),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
-          offset_(offset),
           local_elements_(elements),
           exec_(exec)
     {
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index a0dabdea6..403a3d40a 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -87,7 +87,6 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
           local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
           local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
                                          local_interface_nnz_),
-          global_row_index_{nrows_},
           local_sparsity_{
               fieldName + "_local",           db,       exec_,
               local_matrix_w_interfaces_nnz_, verbose_,
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 29bcf6811..34e781cc7 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -104,7 +104,8 @@ struct CommunicationPattern {
  *
  * It mainly generates:
  * - sparsity pattern/device_matrix_data of the local_matrix and
- * non_local_matrix (with global columns)
+ * - non_local_matrix (with global columns)
+ * - CommuncationPattern
  * */
 template <class MatrixType>
 class HostMatrixWrapper : public MatrixType::solver {
@@ -144,8 +145,6 @@ private:
     // nnzs of local matrix including local interfaces
     const label local_matrix_w_interfaces_nnz_;
 
-    const gkoGlobalIndex global_row_index_;
-
     mutable PersistentSparsityPattern local_sparsity_;
 
     mutable PersistentArray<scalar> local_coeffs_;
@@ -248,78 +247,7 @@ public:
         // pointers to interfaces can be used to access concrete
         // functions such as transferring indices, patch neighbours etc
         const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls, const word &fieldName)
-        : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
-                             interfaceIntCoeffs, interfaces, solverControls),
-          exec_{db, solverControls, fieldName},
-          device_id_guard_{db, fieldName, exec_.get_device_exec()},
-          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-          scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
-          nrows_(matrix.diag().size()),
-          local_interface_nnz_(count_interface_nnz(interfaces, false)),
-          upper_nnz_(matrix.lduAddr().upperAddr().size()),
-          non_diag_nnz_(2 * upper_nnz_),
-          local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
-          local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
-                                         local_interface_nnz_),
-          global_row_index_{nrows_},
-          local_sparsity_{
-              fieldName + "_local",           db,       exec_,
-              local_matrix_w_interfaces_nnz_, verbose_,
-          },
-          local_coeffs_{
-              fieldName + "_local_coeffs",
-              db,
-              exec_,
-              local_matrix_w_interfaces_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
-          communication_pattern_(create_communication_pattern(interfaces)),
-          non_local_sparsity_{
-              fieldName + "_non_local", db,       exec_,
-              non_local_matrix_nnz_,    verbose_,
-          },
-          non_local_coeffs_{
-              fieldName + "_non_local_coeffs",
-              db,
-              exec_,
-              non_local_matrix_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          permutation_matrix_name_{"PermutationMatrix"},
-          permutation_stored_{
-              db.template foundObject<regIOobject>(permutation_matrix_name_)},
-          P_{(permutation_stored_) ? db.template lookupObjectRef<
-                                           DevicePersistentBase<gko::LinOp>>(
-                                           permutation_matrix_name_)
-                                         .get_ptr()
-                                   : nullptr}
-    {
-        if (!local_sparsity_.col_idxs_.get_stored() ||
-            local_sparsity_.col_idxs_.get_update()) {
-            TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
-                                this->fieldName(),
-                                init_local_sparsity_pattern(interfaces);)
-            TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
-                                this->fieldName(),
-                                init_non_local_sparsity_pattern(interfaces);)
-        }
-        if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
-            TIME_WITH_FIELDNAME(
-                verbose_, update_local_matrix_data, this->fieldName(),
-                update_local_matrix_data(interfaces, interfaceBouCoeffs);)
-            TIME_WITH_FIELDNAME(
-                verbose_, update_non_local_matrix_data, this->fieldName(),
-                update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
-        }
-    }
+        const dictionary &solverControls, const word &fieldName);
 
     // TODO reimplement this at some point
     // coupled  wrapper constructor
@@ -337,7 +265,6 @@ public:
           local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
           local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
                                          local_interface_nnz_),
-          global_row_index_{nrows_},
           local_sparsity_{
               fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
           },
@@ -406,15 +333,8 @@ public:
 
     label get_local_nrows() const { return nrows_; }
 
-    label get_global_nrows() const { return global_row_index_.size(); }
-
     const ExecutorHandler &get_exec_handler() const { return exec_; }
 
-    const gkoGlobalIndex &get_global_row_index() const
-    {
-        return global_row_index_;
-    }
-
     PersistentArray<scalar> &get_local_coeffs() const  //
     {
         return local_coeffs_;
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 8548f7f7e..813c84f7a 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -224,7 +224,6 @@ public:
             this->get_exec_handler(),
             verbose_,
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
-            this->get_global_row_index().offset(Pstream::myProcNo()),  // offset
             psi.size(),  // number of elements
             communication_pattern.target_ids,
             communication_pattern.target_sizes,

From adabcfd9a8ac5f2cbaf8161be0c585dbaf4d894e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 09:11:55 +0100
Subject: [PATCH 056/125] add stopping criterion

---
 Solver/CG/GKOCG.H                     | 7 ++-----
 StoppingCriterion/StoppingCriterion.H | 2 --
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 7ad2ea3b5..3da54f949 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -69,8 +69,7 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto cg = cg::build().with_criteria(
-              gko::stop::Iteration::build().with_max_iters(20u)
-                // stoppingCriterionVec_
+                stoppingCriterionVec_
                 ).on(exec);
         return cg;
     };
@@ -81,9 +80,7 @@ public:
     {
         auto cg = cg::build()
                       .with_criteria(
-                              gko::stop::Iteration::build().with_max_iters(20u)
-                              //
-                              // stoppingCriterionVec_
+                              stoppingCriterionVec_
                               )
                       .with_generated_preconditioner(precond)
                       .on(exec);
diff --git a/StoppingCriterion/StoppingCriterion.H b/StoppingCriterion/StoppingCriterion.H
index eda67ccbe..64c8a7178 100644
--- a/StoppingCriterion/StoppingCriterion.H
+++ b/StoppingCriterion/StoppingCriterion.H
@@ -187,8 +187,6 @@ class StoppingCriterion {
                 *(parameters_.iter) += 1;
                 return false;
             }
-            // TODO FIXME
-            std::cout << "iteration\n ";
 
             const auto exec = this->get_executor();
 

From ea101149c5f04d8713e9ba31f520873b2e145eb9 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 10:21:17 +0100
Subject: [PATCH 057/125] switch gko branch

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9e3d69238..48033155e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT OGL_ALLOW_REFERENCE_ONLY)
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
-    "sparse-communicator-ogl"
+    "sparse-communicator"
     CACHE STRING "Use specific version of ginkgo")
 
 include(CheckIncludeFileCXX)

From efa2a61f6a8c13a36133b627d1dbaf6fc74c45d1 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 10:24:22 +0100
Subject: [PATCH 058/125] reformat

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       |  32 +--
 DevicePersistent/Partition/Partition.H        |  13 +-
 DevicePersistent/Vector/Vector.H              |  14 +-
 HostMatrix/HostMatrix.C                       | 183 +++++++++---------
 HostMatrix/HostMatrix.H                       |  18 +-
 Solver/CG/GKOCG.H                             |   8 +-
 common/common.C                               |   4 +-
 lduLduBase/lduLduBase.H                       |   3 +-
 8 files changed, 134 insertions(+), 141 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 344948570..91fc541e6 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -225,22 +225,22 @@ struct MatrixInitFunctor {
         auto send_sizes = sparse_comm->get_send_sizes();
         auto send_offs = sparse_comm->get_send_offsets();
 
-        if (Pstream::master() ) {
-        for (auto e: recv_sizes) {
-            std::cout << "recv_size: " << e << std::endl;
-        }
-
-        for (auto e: recv_offs) {
-            std::cout << "recv_offs: " << e << std::endl;
-        }
-
-        for (auto e: send_sizes) {
-            std::cout << "send_sizes: " << e << std::endl;
-        }
-
-        for (auto e: send_offs) {
-            std::cout << "send_offs: " << e << std::endl;
-        }
+        if (Pstream::master()) {
+            for (auto e : recv_sizes) {
+                std::cout << "recv_size: " << e << std::endl;
+            }
+
+            for (auto e : recv_offs) {
+                std::cout << "recv_offs: " << e << std::endl;
+            }
+
+            for (auto e : send_sizes) {
+                std::cout << "send_sizes: " << e << std::endl;
+            }
+
+            for (auto e : send_offs) {
+                std::cout << "send_offs: " << e << std::endl;
+            }
         }
 
 
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index beabad5c6..64cba12d4 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -58,8 +58,8 @@ struct PartitionInitFunctor {
     const std::vector<std::pair<gko::index_set<label>, label>> send_idxs_;
 
     PartitionInitFunctor(
-        const ExecutorHandler &exec, const label local_size
-        , const label verbose, const int ranks_per_gpu,
+        const ExecutorHandler &exec, const label local_size,
+        const label verbose, const int ranks_per_gpu,
         const gko::array<label> &target_ids,
         const gko::array<label> &target_sizes,
         const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
@@ -127,15 +127,14 @@ public:
      */
     PersistentPartition(
         const objectRegistry &db, const ExecutorHandler &exec,
-        const label verbose, const int ranks_per_gpu,
-        const label elements, const gko::array<label> &target_ids,
+        const label verbose, const int ranks_per_gpu, const label elements,
+        const gko::array<label> &target_ids,
         const gko::array<label> &target_sizes,
         const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
-              PartitionInitFunctor(exec, elements, verbose,
-                                   ranks_per_gpu, target_ids, target_sizes,
-                                   send_idxs),
+              PartitionInitFunctor(exec, elements, verbose, ranks_per_gpu,
+                                   target_ids, target_sizes, send_idxs),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           local_elements_(elements),
diff --git a/DevicePersistent/Vector/Vector.H b/DevicePersistent/Vector/Vector.H
index 640c60c40..86c93daa4 100644
--- a/DevicePersistent/Vector/Vector.H
+++ b/DevicePersistent/Vector/Vector.H
@@ -191,16 +191,16 @@ public:
 
         // TODO why does it need its size
         // repartitioner size should be sufficient
-        auto target_vector = dist_vec::create(
-            exec_.get_ref_exec(), *comm.get(), gko::dim<2>{global_size, 1},
-            gko::dim<2>{local_size, 1}, 1);
+        auto target_vector = dist_vec::create(exec_.get_ref_exec(), *comm.get(),
+                                              gko::dim<2>{global_size, 1},
+                                              gko::dim<2>{local_size, 1}, 1);
         scalar *host_buffer = host_device_vector->get_local_values();
 
-        auto host_buffer_view = gko::array<T>::view(
-            exec_.get_ref_exec(), local_size, host_buffer);
+        auto host_buffer_view =
+            gko::array<T>::view(exec_.get_ref_exec(), local_size, host_buffer);
 
-        auto to_view = gko::array<T>::view(
-            exec_.get_ref_exec(), local_size, const_cast<T *>(memory_));
+        auto to_view = gko::array<T>::view(exec_.get_ref_exec(), local_size,
+                                           const_cast<T *>(memory_));
         to_view = host_buffer_view;
     }
 
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 403a3d40a..940fdd557 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -65,85 +65,83 @@ label HostMatrixWrapper<MatrixType>::count_interface_nnz(
 
 template <class MatrixType>
 HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
-        const objectRegistry &db, const MatrixType &matrix,
-        // coeffs for cells on boundaries
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        // coeffs for internal cells
-        const FieldField<Field, scalar> &interfaceIntCoeffs,
-        // pointers to interfaces can be used to access concrete
-        // functions such as transferring indices, patch neighbours etc
-        const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls, const word &fieldName)
-        : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
-                             interfaceIntCoeffs, interfaces, solverControls),
-          exec_{db, solverControls, fieldName},
-          device_id_guard_{db, fieldName, exec_.get_device_exec()},
-          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-          scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
-          nrows_(matrix.diag().size()),
-          local_interface_nnz_(count_interface_nnz(interfaces, false)),
-          upper_nnz_(matrix.lduAddr().upperAddr().size()),
-          non_diag_nnz_(2 * upper_nnz_),
-          local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
-          local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
-                                         local_interface_nnz_),
-          local_sparsity_{
-              fieldName + "_local",           db,       exec_,
-              local_matrix_w_interfaces_nnz_, verbose_,
-          },
-          local_coeffs_{
-              fieldName + "_local_coeffs",
-              db,
-              exec_,
-              local_matrix_w_interfaces_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
-          communication_pattern_(create_communication_pattern(interfaces)),
-          non_local_sparsity_{
-              fieldName + "_non_local", db,       exec_,
-              non_local_matrix_nnz_,    verbose_,
-          },
-          non_local_coeffs_{
-              fieldName + "_non_local_coeffs",
-              db,
-              exec_,
-              non_local_matrix_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          permutation_matrix_name_{"PermutationMatrix"},
-          permutation_stored_{
-              db.template foundObject<regIOobject>(permutation_matrix_name_)},
-          P_{(permutation_stored_) ? db.template lookupObjectRef<
-                                           DevicePersistentBase<gko::LinOp>>(
-                                           permutation_matrix_name_)
-                                         .get_ptr()
-                                   : nullptr}
-    {
-        if (!local_sparsity_.col_idxs_.get_stored() ||
-            local_sparsity_.col_idxs_.get_update()) {
-            TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
-                                this->fieldName(),
-                                init_local_sparsity_pattern(interfaces);)
-            TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
-                                this->fieldName(),
-                                init_non_local_sparsity_pattern(interfaces);)
-        }
-        if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
-            TIME_WITH_FIELDNAME(
-                verbose_, update_local_matrix_data, this->fieldName(),
-                update_local_matrix_data(interfaces, interfaceBouCoeffs);)
-            TIME_WITH_FIELDNAME(
-                verbose_, update_non_local_matrix_data, this->fieldName(),
-                update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
-        }
+    const objectRegistry &db, const MatrixType &matrix,
+    // coeffs for cells on boundaries
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    // coeffs for internal cells
+    const FieldField<Field, scalar> &interfaceIntCoeffs,
+    // pointers to interfaces can be used to access concrete
+    // functions such as transferring indices, patch neighbours etc
+    const lduInterfaceFieldPtrsList &interfaces,
+    const dictionary &solverControls, const word &fieldName)
+    : MatrixType::solver(fieldName, matrix, interfaceBouCoeffs,
+                         interfaceIntCoeffs, interfaces, solverControls),
+      exec_{db, solverControls, fieldName},
+      device_id_guard_{db, fieldName, exec_.get_device_exec()},
+      verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
+      nrows_(matrix.diag().size()),
+      local_interface_nnz_(count_interface_nnz(interfaces, false)),
+      upper_nnz_(matrix.lduAddr().upperAddr().size()),
+      non_diag_nnz_(2 * upper_nnz_),
+      local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
+      local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
+      local_sparsity_{
+          fieldName + "_local",           db,       exec_,
+          local_matrix_w_interfaces_nnz_, verbose_,
+      },
+      local_coeffs_{
+          fieldName + "_local_coeffs",
+          db,
+          exec_,
+          local_matrix_w_interfaces_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      non_local_matrix_nnz_(count_interface_nnz(interfaces, true)),
+      communication_pattern_(create_communication_pattern(interfaces)),
+      non_local_sparsity_{
+          fieldName + "_non_local", db, exec_, non_local_matrix_nnz_, verbose_,
+      },
+      non_local_coeffs_{
+          fieldName + "_non_local_coeffs",
+          db,
+          exec_,
+          non_local_matrix_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      permutation_matrix_name_{"PermutationMatrix"},
+      permutation_stored_{
+          db.template foundObject<regIOobject>(permutation_matrix_name_)},
+      P_{(permutation_stored_)
+             ? db.template lookupObjectRef<DevicePersistentBase<gko::LinOp>>(
+                     permutation_matrix_name_)
+                   .get_ptr()
+             : nullptr}
+{
+    if (!local_sparsity_.col_idxs_.get_stored() ||
+        local_sparsity_.col_idxs_.get_update()) {
+        TIME_WITH_FIELDNAME(verbose_, init_local_sparsity_pattern,
+                            this->fieldName(),
+                            init_local_sparsity_pattern(interfaces);)
+        TIME_WITH_FIELDNAME(verbose_, init_non_local_sparsity_pattern,
+                            this->fieldName(),
+                            init_non_local_sparsity_pattern(interfaces);)
     }
+    if (!local_coeffs_.get_stored() || local_coeffs_.get_update()) {
+        TIME_WITH_FIELDNAME(
+            verbose_, update_local_matrix_data, this->fieldName(),
+            update_local_matrix_data(interfaces, interfaceBouCoeffs);)
+        TIME_WITH_FIELDNAME(
+            verbose_, update_non_local_matrix_data, this->fieldName(),
+            update_non_local_matrix_data(interfaces, interfaceBouCoeffs);)
+    }
+}
 
 
 template <class MatrixType>
@@ -246,10 +244,10 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
         label interface_size = interface_cells.size();
         auto exec = exec_.get_ref_exec();
         send_idxs.push_back(std::pair<gko::index_set<label>, label>(
-            gko::index_set(
-                exec, interface_size,
-                gko::array<label>::view(exec, interface_size, interface_cells.data()),
-                true),
+            gko::index_set(exec, interface_size,
+                           gko::array<label>::view(exec, interface_size,
+                                                   interface_cells.data()),
+                           true),
             proc));
     }
 
@@ -692,16 +690,15 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     i_device_view = tmp_contiguous_iface;
 }
 
-template 
-HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
-        const objectRegistry &db, const lduMatrix &matrix,
-        // coeffs for cells on boundaries
-        const FieldField<Field, scalar> &interfaceBouCoeffs,
-        // coeffs for internal cells
-        const FieldField<Field, scalar> &interfaceIntCoeffs,
-        // pointers to interfaces can be used to access concrete
-        // functions such as transferring indices, patch neighbours etc
-        const lduInterfaceFieldPtrsList &interfaces,
-        const dictionary &solverControls, const word &fieldName);
+template HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(
+    const objectRegistry &db, const lduMatrix &matrix,
+    // coeffs for cells on boundaries
+    const FieldField<Field, scalar> &interfaceBouCoeffs,
+    // coeffs for internal cells
+    const FieldField<Field, scalar> &interfaceIntCoeffs,
+    // pointers to interfaces can be used to access concrete
+    // functions such as transferring indices, patch neighbours etc
+    const lduInterfaceFieldPtrsList &interfaces,
+    const dictionary &solverControls, const word &fieldName);
 
 }  // namespace Foam
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 34e781cc7..91d914107 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -39,7 +39,7 @@ SourceFiles
 namespace Foam {
 
 
-    // TODO Move this somewhere else
+// TODO Move this somewhere else
 struct PersistentSparsityPattern {
     PersistentSparsityPattern(
       const word &fieldName,
@@ -181,16 +181,17 @@ private:
     ** @param interfaces The list of interfaces for the search
     ** @param proc_interfaces Count only elements on (true)
     *processorLduInterfaces or exclude processorLduInterfaces (false)
-     ** TODO make private and no discard
+    ** TODO make private and no discard
     */
     label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
                               bool proc_interfaces) const;
 
-     /** Iterates all local interfaces and returns the relative order and
-     **corresponding row and column indices
-     **
-     ** @return vector of tuples containing the interface number, the local row, the local column
-     **/
+    /** Iterates all local interfaces and returns the relative order and
+    **corresponding row and column indices
+    **
+    ** @return vector of tuples containing the interface number, the local row,
+    *the local column
+    **/
     std::vector<std::tuple<label, label, label>>
     collect_local_interface_indices(
         const lduInterfaceFieldPtrsList &interfaces_) const;
@@ -318,8 +319,7 @@ public:
         const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
     // getter
-    public:
-
+public:
     bool get_verbose() const { return verbose_; }
 
     /** Getter for the CommunicationPattern
diff --git a/Solver/CG/GKOCG.H b/Solver/CG/GKOCG.H
index 3da54f949..427a58ca9 100644
--- a/Solver/CG/GKOCG.H
+++ b/Solver/CG/GKOCG.H
@@ -68,9 +68,7 @@ public:
     std::unique_ptr<cg::Factory, std::default_delete<cg::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        auto cg = cg::build().with_criteria(
-                stoppingCriterionVec_
-                ).on(exec);
+        auto cg = cg::build().with_criteria(stoppingCriterionVec_).on(exec);
         return cg;
     };
 
@@ -79,9 +77,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto cg = cg::build()
-                      .with_criteria(
-                              stoppingCriterionVec_
-                              )
+                      .with_criteria(stoppingCriterionVec_)
                       .with_generated_preconditioner(precond)
                       .on(exec);
         return cg;
diff --git a/common/common.C b/common/common.C
index ed9846fe6..d7ace7281 100644
--- a/common/common.C
+++ b/common/common.C
@@ -62,9 +62,9 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
                 const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
-    //std::filesystem::create_directories(folder);
+    // std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
-    std::cout  << "exporting " << fn <<  std::endl;
+    std::cout << "exporting " << fn << std::endl;
     std::ofstream stream{fn};
     stream << std::setprecision(15);
     if (matrixFormat == "Coo") {
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 813c84f7a..9114a532f 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -330,7 +330,8 @@ public:
         //     delta_t_solve / max(this->get_number_of_iterations(), 1);
         // scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
         // this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
-        //     this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1, 0);
+        //     this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1,
+        //     0);
         // this->set_prev_rel_res_cost(prev_rel_res_cost);
         // auto time_per_iter_and_dof =
         //     time_per_iter * 1000.0 / partition.get_total_size();

From e5843461e441f61f6aa309af361315d211fb07dc Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 10:25:52 +0100
Subject: [PATCH 059/125] format

---
 common/common.H | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/common/common.H b/common/common.H
index 5100477e6..5efbe9430 100644
--- a/common/common.H
+++ b/common/common.H
@@ -61,8 +61,9 @@ namespace Foam {
     }
 
 
-#define IN_USE_DO_NOT_REMOVE(FUNC) \
-std::cout << FUNC <<  " in "   << __FILENAME__ << ":" <<__LINE__ << " is in use do not remove"  << std::endl
+#define IN_USE_DO_NOT_REMOVE(FUNC)                                 \
+    std::cout << FUNC << " in " << __FILENAME__ << ":" << __LINE__ \
+              << " is in use do not remove" << std::endl
 
 #define LOG_0(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 0, MSG, 0)
 #define LOG_1(VERBOSE, MSG) SIMPLE_LOG(VERBOSE, 1, MSG, 0)

From a604389f7589b0f76334cd728b801486b1235ff5 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 6 Nov 2023 10:27:15 +0100
Subject: [PATCH 060/125] fix typo

---
 HostMatrix/HostMatrix.C | 2 +-
 HostMatrix/HostMatrix.H | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 940fdd557..5b69527a5 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -223,7 +223,7 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
 
     // reduce vector of sizes
     // this assumes that a rank can be connected to the same neighbour through
-    // different interfaces this migth not be necessary
+    // different interfaces this might not be necessary
     label n_procs = 0;
     std::map<label, label> reduce_map{};
     for (auto [proc, n_faces] : neighbour_procs) {
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 91d914107..12ff6bb18 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -105,7 +105,7 @@ struct CommunicationPattern {
  * It mainly generates:
  * - sparsity pattern/device_matrix_data of the local_matrix and
  * - non_local_matrix (with global columns)
- * - CommuncationPattern
+ * - CommunicationPattern
  * */
 template <class MatrixType>
 class HostMatrixWrapper : public MatrixType::solver {

From c8cdbbb63c4b37a96eea9950992032e6a0331316 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 7 Nov 2023 14:48:18 +0100
Subject: [PATCH 061/125] Move ctr to .C file

---
 HostMatrix/HostMatrix.C | 241 ++++++++++++++++++++++------------------
 HostMatrix/HostMatrix.H |  83 ++------------
 2 files changed, 142 insertions(+), 182 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 5b69527a5..77799a638 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -27,6 +27,7 @@ SourceFiles
 
 #include "cyclicFvPatchField.H"
 #include "lduMatrix.H"
+#include "processorFvPatch.H"
 
 
 namespace Foam {
@@ -144,6 +145,65 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
 }
 
 
+template <class MatrixType>
+HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
+    const objectRegistry &db, const MatrixType &matrix,
+    const dictionary &solverControls, const word &fieldName)
+    : MatrixType::solver(fieldName, matrix, solverControls),
+      exec_{db, solverControls, fieldName},
+      device_id_guard_{db, fieldName, exec_.get_device_exec()},
+      verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
+      nrows_(matrix.diag().size()),
+      local_interface_nnz_(0),
+      upper_nnz_(matrix.lduAddr().upperAddr().size()),
+      non_diag_nnz_(2 * upper_nnz_),
+      local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
+      local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
+      local_sparsity_{
+          fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
+      },
+      local_coeffs_{
+          fieldName + "_coeffs",
+          db,
+          exec_,
+          local_matrix_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      non_local_matrix_nnz_(),
+      // proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
+      non_local_sparsity_{
+          fieldName + "_non_local", db, exec_, non_local_matrix_nnz_, verbose_,
+      },
+      non_local_coeffs_{
+          fieldName + "_non_local_coeffs",
+          db,
+          exec_,
+          non_local_matrix_nnz_,
+          verbose_,
+          true,  // needs to be updated
+          false  // leave it on host once it is turned into a distributed
+                 // matrix it will be put on the device
+      },
+      permutation_matrix_name_{"PermutationMatrix"},
+      permutation_stored_{
+          db.template foundObject<regIOobject>(permutation_matrix_name_)},
+      P_{(permutation_stored_)
+             ? db.template lookupObjectRef<DevicePersistentBase<gko::LinOp>>(
+                     permutation_matrix_name_)
+                   .get_ptr()
+             : nullptr}
+{
+        FatalErrorInFunction
+           << "This constructor is currently not implemented" 
+           << abort(FatalError);
+
+}
+
+
 template <class MatrixType>
 std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
     const lduInterfaceFieldPtrsList &interfaces,
@@ -174,6 +234,24 @@ std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
     return ret;
 }
 
+template <class Sel, class Func>
+void interface_iterator(const lduInterfaceFieldPtrsList &interfaces, Func func)
+{
+    label element_ctr = 0;
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        if (isA<Sel>(iface->interface())) {
+            func(element_ctr, interface_size, iface);
+        }
+    }
+}
+
 
 template <class MatrixType>
 CommunicationPattern
@@ -186,17 +264,11 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     // temp map, mapping from neighbour rank interface cells
     std::map<label, std::vector<label>> interface_cell_map{};
 
-    // TODO FIXME use a lambda to avoid repeating
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto &iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        if (isA<processorLduInterface>(iface->interface())) {
+    //
+    interface_iterator<processorFvPatch>(
+        interfaces,
+        [&](label, const label interface_size, const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
             const processorLduInterface &pldui =
                 refCast<const processorLduInterface>(iface->interface());
             const label neighbProcNo = pldui.neighbProcNo();
@@ -204,22 +276,20 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
             neighbour_procs.push_back(
                 std::pair<label, label>{neighbProcNo, interface_size});
 
-            //
             auto search = interface_cell_map.find(neighbProcNo);
             if (search == interface_cell_map.end()) {
                 interface_cell_map.insert(std::pair{
                     neighbProcNo,
                     std::vector<label>(face_cells.begin(), face_cells.end())});
             } else {
-                // TODO FIXME reserve size
                 auto cur_face_cells = interface_cell_map[neighbProcNo];
+                cur_face_cells.reserve(interface_size);
                 for (auto face_cell : face_cells) {
                     cur_face_cells.push_back(face_cell);
                 }
                 interface_cell_map[neighbProcNo] = cur_face_cells;
             }
-        }
-    }
+        });
 
     // reduce vector of sizes
     // this assumes that a rank can be connected to the same neighbour through
@@ -236,6 +306,7 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
         }
     }
 
+
     // create index_sets
     // currently this assumes that there is only one interface to a given
     // neighbour rank
@@ -262,6 +333,12 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
         iter++;
     }
 
+    for (int n = 0; n < n_procs; n++) {
+        std::cout << Pstream::myProcNo()
+                  << "target_ids:" << target_ids.get_data()[n] << "target_size "
+                  << target_sizes.get_data()[n] << std::endl;
+    }
+
     return CommunicationPattern{target_ids, target_sizes, send_idxs};
 }
 
@@ -274,22 +351,13 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
     std::vector<std::tuple<label, label, label>> local_interface_idxs{};
     local_interface_idxs.reserve(local_interface_nnz_);
 
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
 
-        // TODO make this a separate specialized function
-        label interface_ctr = 0;
-        if (isA<cyclicLduInterface>(iface->interface())) {
+    interface_iterator<cyclicLduInterface>(
+        interfaces, [&](label &element_ctr, label interface_size,
+                        const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
             const cyclicLduInterface &pldui =
                 refCast<const cyclicLduInterface>(iface->interface());
-            // const labelUList &rows = this->matrix().lduAddr().patchAddr(i);
 
 #ifdef WITH_ESI_VERSION
             const label neighbPatchId = pldui.neighbPatchID();
@@ -299,47 +367,33 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
             const labelUList &cols =
                 this->matrix().lduAddr().patchAddr(neighbPatchId);
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                // DEBUG
-                // cout << "patch_ctr " << cellI << " ictr " << interface_ctr
-                //     << " patchId " << i << " neighPatchId " << neighbPatchId
-                //      << " row " << rows[cellI] << " col " << cols[cellI]
-                //      << "\n" << endl;
                 local_interface_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], cols[cellI]});
-                interface_ctr += 1;
+                    {element_ctr, face_cells[cellI], cols[cellI]});
+                element_ctr += 1;
             }
-        }
-    }
+        });
     return local_interface_idxs;
 }
 
 template <class MatrixType>
-std::vector<std::tuple<label, label, label>>
-HostMatrixWrapper<MatrixType>::collect_non_local_col_indices(
+std::vector<std::tuple<label, label>>
+HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
     // vector of local cell ids on other side
-    std::vector<std::tuple<label, label, label>> non_local_idxs{};
+    std::vector<std::tuple<label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
-    label interface_ctr = 0;
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        if (isA<processorLduInterface>(iface->interface())) {
+    interface_iterator<processorLduInterface>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const lduInterfaceField *iface) {
+            const auto &face_cells{iface->interface().faceCells()};
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                non_local_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], interface_ctr});
-                interface_ctr += 1;
+                non_local_idxs.push_back({element_ctr, face_cells[cellI]});
+                element_ctr += 1;
             }
-        }
-    }
+        });
+
     word msg = "done collecting neighbouring processor cell id";
 
     LOG_2(verbose_, msg)
@@ -350,44 +404,30 @@ template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
-    auto non_local_row_indices = collect_non_local_col_indices(interfaces);
+    auto non_local_row_indices = collect_cells_on_interface(interfaces);
     auto rows = non_local_sparsity_.row_idxs_.get_data();
     auto cols = non_local_sparsity_.col_idxs_.get_data();
     auto permute = non_local_sparsity_.ldu_mapping_.get_data();
 
     std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
               [&](const auto &a, const auto &b) {
-                  auto [interface_idx_a, row_a, col_a] = a;
-                  auto [interface_idx_b, row_b, col_b] = b;
-                  return std::tie(row_a, col_a) < std::tie(row_b, col_b);
+                  auto [interface_idx_a, row_a] = a;
+                  auto [interface_idx_b, row_b] = b;
+                  return std::tie(row_a, interface_idx_a) <
+                         std::tie(row_b, interface_idx_b);
               });
 
-    label interface_ctr = 0;
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto &iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        if (isA<processorLduInterface>(iface->interface())) {
-            // const processorLduInterface &pldui =
-            //     refCast<const processorLduInterface>(iface->interface());
-
-            // check if current cell is on the current patch
-            // NOTE cells can be several times on same patch
+    interface_iterator<processorLduInterface>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const lduInterfaceField *) {
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto [interface_idx, row, col] =
-                    non_local_row_indices[interface_ctr];
-                rows[interface_ctr] = row;
-                cols[interface_ctr] = col;
-                permute[interface_ctr] = interface_idx;
-                interface_ctr += 1;
+                auto [interface_idx, row] = non_local_row_indices[element_ctr];
+                rows[element_ctr] = row;
+                cols[element_ctr] = interface_idx;
+                permute[element_ctr] = interface_idx;
+                element_ctr += 1;
             }
-        }
-    }
+        });
 }
 
 template <class MatrixType>
@@ -542,7 +582,6 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
         // post insert if local interfaces were consumed but stuff remains in
         // rows_copy and cols_copy
         for (int i = total_ctr; i < local_matrix_w_interfaces_nnz_; i++) {
-            // std::cout << "insert missing values " << i << endl;
             rows[i] = rows_copy[current_idx_ctr];
             cols[i] = cols_copy[current_idx_ctr];
             permute[i] = permute_copy[current_idx_ctr];
@@ -593,10 +632,6 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
                     value = diag[pos - upper_nnz_];
                 }
                 if (pos >= upper_nnz_ + diag_nnz) {
-                    // std::cout <<  "idx ( "
-                    // << local_sparsity_.row_idxs_.get_data()[i] << ", "
-                    // << local_sparsity_.col_idxs_.get_data()[i] << "): "
-                    // << value << "\n" ;
                     value = -couple_coeffs[pos - upper_nnz_ - diag_nnz];
                 }
 
@@ -655,7 +690,6 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     const FieldField<Field, scalar> &interfaceBouCoeffs) const
 {
     auto ref_exec = exec_.get_ref_exec();
-
     auto interface_coeffs =
         collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
     auto permute = non_local_sparsity_.ldu_mapping_.get_data();
@@ -665,24 +699,15 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
         gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
     auto contiguous_iface = tmp_contiguous_iface.get_data();
 
-    label interface_ctr{0};
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-        const auto iface{interface_getter(interfaces, i)};
-        const label patch_size = iface->interface().faceCells().size();
-
-        if (!isA<processorLduInterface>(iface->interface())) {
-            continue;
-        }
-
-        for (label cellI = 0; cellI < patch_size; cellI++) {
-            contiguous_iface[interface_ctr] =
-                -interface_coeffs[permute[interface_ctr]];
-            interface_ctr += 1;
-        }
-    }
+    interface_iterator<processorLduInterface>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const lduInterfaceField *) {
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                contiguous_iface[element_ctr] =
+                    -interface_coeffs[permute[element_ctr]];
+                element_ctr++;
+            }
+        });
 
     // copy to persistent
     auto i_device_view = gko::array<scalar>::view(
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 12ff6bb18..9997268e1 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -39,7 +39,6 @@ SourceFiles
 namespace Foam {
 
 
-// TODO Move this somewhere else
 struct PersistentSparsityPattern {
     PersistentSparsityPattern(
       const word &fieldName,
@@ -181,7 +180,6 @@ private:
     ** @param interfaces The list of interfaces for the search
     ** @param proc_interfaces Count only elements on (true)
     *processorLduInterfaces or exclude processorLduInterfaces (false)
-    ** TODO make private and no discard
     */
     label count_interface_nnz(const lduInterfaceFieldPtrsList &interfaces,
                               bool proc_interfaces) const;
@@ -203,22 +201,18 @@ private:
      **
      ** @param interfaces The list of interfaces for the search
      ** @return the CommunicationPattern
-     ** TODO make private and no discard
      */
     CommunicationPattern create_communication_pattern(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
-    /* Iterates all interfaces and collects non-local global column indices
-    **
-    ** TODO FIXME this is not uptodate
-    ** Non-local global column indices are computed from the global cell-id
-    ** (row) on the other side of an interface
-    **
+    /** Iterates all interfaces and collect the corresponding cell id (row)
+    ** and a unique counter
+    ** 
     ** @return vector off size nnz_non_local, with sections for each interface
-    ** ret = [(1,2,3),(10,20,30), ...]
-    **         i0      i1,       i...
+    ** ret = [(1,2),(2,20), ...]
+    **         i0   i1,   i...
     */
-    std::vector<std::tuple<label, label, label>> collect_non_local_col_indices(
+    std::vector<std::tuple<label, label>> collect_cells_on_interface(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
     /* Based on OpenFOAMs ldu matrix format this function computes two
@@ -253,60 +247,7 @@ public:
     // TODO reimplement this at some point
     // coupled  wrapper constructor
     HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
-                      const dictionary &solverControls, const word &fieldName)
-        : MatrixType::solver(fieldName, matrix, solverControls),
-          exec_{db, solverControls, fieldName},
-          device_id_guard_{db, fieldName, exec_.get_device_exec()},
-          verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
-          scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
-          nrows_(matrix.diag().size()),
-          local_interface_nnz_(0),
-          upper_nnz_(matrix.lduAddr().upperAddr().size()),
-          non_diag_nnz_(2 * upper_nnz_),
-          local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
-          local_matrix_w_interfaces_nnz_(local_matrix_nnz_ +
-                                         local_interface_nnz_),
-          local_sparsity_{
-              fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
-          },
-          local_coeffs_{
-              fieldName + "_coeffs",
-              db,
-              exec_,
-              local_matrix_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          non_local_matrix_nnz_(),
-          // proc_target_id_and_sizes_(assemble_proc_id_and_sizes(interfaces)),
-          non_local_sparsity_{
-              fieldName + "_non_local", db,       exec_,
-              non_local_matrix_nnz_,    verbose_,
-          },
-          non_local_coeffs_{
-              fieldName + "_non_local_coeffs",
-              db,
-              exec_,
-              non_local_matrix_nnz_,
-              verbose_,
-              true,  // needs to be updated
-              false  // leave it on host once it is turned into a distributed
-                     // matrix it will be put on the device
-          },
-          permutation_matrix_name_{"PermutationMatrix"},
-          permutation_stored_{
-              db.template foundObject<regIOobject>(permutation_matrix_name_)},
-          P_{(permutation_stored_) ? db.template lookupObjectRef<
-                                           DevicePersistentBase<gko::LinOp>>(
-                                           permutation_matrix_name_)
-                                         .get_ptr()
-                                   : nullptr}
-    {
-        // TODO reimplement this
-    }
-
+                      const dictionary &solverControls, const word &fieldName);
 
     // updater
 
@@ -319,13 +260,10 @@ public:
         const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
     // getter
+
 public:
     bool get_verbose() const { return verbose_; }
 
-    /** Getter for the CommunicationPattern
-     **
-     ** @return the CommunicationPattern
-     **/
     const CommunicationPattern &get_communication_pattern() const
     {
         return communication_pattern_;
@@ -335,10 +273,7 @@ public:
 
     const ExecutorHandler &get_exec_handler() const { return exec_; }
 
-    PersistentArray<scalar> &get_local_coeffs() const  //
-    {
-        return local_coeffs_;
-    };
+    PersistentArray<scalar> &get_local_coeffs() const { return local_coeffs_; };
 
     PersistentArray<label> &get_local_col_idxs() const
     {

From 7b79a724d0c380d9bcfd5b36aa732bed0c7e6d25 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 7 Nov 2023 15:00:08 +0100
Subject: [PATCH 062/125] use concrete types for interfaces

---
 HostMatrix/HostMatrix.C | 50 +++++++++++++++++------------------------
 1 file changed, 21 insertions(+), 29 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 77799a638..8a5bf21d1 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -197,10 +197,8 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
                    .get_ptr()
              : nullptr}
 {
-        FatalErrorInFunction
-           << "This constructor is currently not implemented" 
-           << abort(FatalError);
-
+    FatalErrorInFunction << "This constructor is currently not implemented"
+                         << abort(FatalError);
 }
 
 
@@ -245,9 +243,10 @@ void interface_iterator(const lduInterfaceFieldPtrsList &interfaces, Func func)
         const auto iface{interface_getter(interfaces, i)};
         const auto &face_cells{iface->interface().faceCells()};
         const label interface_size = face_cells.size();
+        const Sel &patch = refCast<const Sel>(iface->interface());
 
         if (isA<Sel>(iface->interface())) {
-            func(element_ctr, interface_size, iface);
+            func(element_ctr, interface_size, patch, iface);
         }
     }
 }
@@ -267,11 +266,10 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     //
     interface_iterator<processorFvPatch>(
         interfaces,
-        [&](label, const label interface_size, const lduInterfaceField *iface) {
+        [&](label, const label interface_size, const processorFvPatch &patch,
+            const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
-            const processorLduInterface &pldui =
-                refCast<const processorLduInterface>(iface->interface());
-            const label neighbProcNo = pldui.neighbProcNo();
+            const label neighbProcNo = patch.neighbProcNo();
 
             neighbour_procs.push_back(
                 std::pair<label, label>{neighbProcNo, interface_size});
@@ -333,12 +331,6 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
         iter++;
     }
 
-    for (int n = 0; n < n_procs; n++) {
-        std::cout << Pstream::myProcNo()
-                  << "target_ids:" << target_ids.get_data()[n] << "target_size "
-                  << target_sizes.get_data()[n] << std::endl;
-    }
-
     return CommunicationPattern{target_ids, target_sizes, send_idxs};
 }
 
@@ -352,17 +344,16 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
     local_interface_idxs.reserve(local_interface_nnz_);
 
 
-    interface_iterator<cyclicLduInterface>(
-        interfaces, [&](label &element_ctr, label interface_size,
-                        const lduInterfaceField *iface) {
+    interface_iterator<cyclicFvPatch>(
+        interfaces,
+        [&](label &element_ctr, label interface_size,
+            const cyclicFvPatch &patch, const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
-            const cyclicLduInterface &pldui =
-                refCast<const cyclicLduInterface>(iface->interface());
 
 #ifdef WITH_ESI_VERSION
-            const label neighbPatchId = pldui.neighbPatchID();
+            const label neighbPatchId = patch.neighbPatchID();
 #else
-            const label neighbPatchId = pldui.nbrPatchID();
+            const label neighbPatchId = patch.nbrPatchID();
 #endif
             const labelUList &cols =
                 this->matrix().lduAddr().patchAddr(neighbPatchId);
@@ -384,9 +375,10 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     std::vector<std::tuple<label, label>> non_local_idxs{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
-    interface_iterator<processorLduInterface>(
-        interfaces, [&](label &element_ctr, const label interface_size,
-                        const lduInterfaceField *iface) {
+    interface_iterator<processorFvPatch>(
+        interfaces,
+        [&](label &element_ctr, const label interface_size,
+            const processorFvPatch &, const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
             for (label cellI = 0; cellI < interface_size; cellI++) {
                 non_local_idxs.push_back({element_ctr, face_cells[cellI]});
@@ -417,9 +409,9 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
                          std::tie(row_b, interface_idx_b);
               });
 
-    interface_iterator<processorLduInterface>(
+    interface_iterator<processorFvPatch>(
         interfaces, [&](label &element_ctr, const label interface_size,
-                        const lduInterfaceField *) {
+                        const processorFvPatch &, const lduInterfaceField *) {
             for (label cellI = 0; cellI < interface_size; cellI++) {
                 auto [interface_idx, row] = non_local_row_indices[element_ctr];
                 rows[element_ctr] = row;
@@ -699,9 +691,9 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
         gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
     auto contiguous_iface = tmp_contiguous_iface.get_data();
 
-    interface_iterator<processorLduInterface>(
+    interface_iterator<processorFvPatch>(
         interfaces, [&](label &element_ctr, const label interface_size,
-                        const lduInterfaceField *) {
+                        const processorFvPatch &, const lduInterfaceField *) {
             for (label cellI = 0; cellI < interface_size; cellI++) {
                 contiguous_iface[element_ctr] =
                     -interface_coeffs[permute[element_ctr]];

From d41ef19c47508219b8bd5c833e0ba210eb73a5e7 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 8 Nov 2023 07:11:40 +0100
Subject: [PATCH 063/125] clean-up

---
 .../CsrMatrixWrapper/CsrMatrixWrapper.H       | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index 91fc541e6..e3fd5feb3 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -225,25 +225,6 @@ struct MatrixInitFunctor {
         auto send_sizes = sparse_comm->get_send_sizes();
         auto send_offs = sparse_comm->get_send_offsets();
 
-        if (Pstream::master()) {
-            for (auto e : recv_sizes) {
-                std::cout << "recv_size: " << e << std::endl;
-            }
-
-            for (auto e : recv_offs) {
-                std::cout << "recv_offs: " << e << std::endl;
-            }
-
-            for (auto e : send_sizes) {
-                std::cout << "send_sizes: " << e << std::endl;
-            }
-
-            for (auto e : send_offs) {
-                std::cout << "send_offs: " << e << std::endl;
-            }
-        }
-
-
         dist_A->read_distributed(A_data, non_local_A_data, sparse_comm);
 
         auto device_mat = generate_dist_mtx_with_inner_type(

From 7a9c400590a7dfe8d91188afe31f3f1468e19478 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 8 Nov 2023 07:12:51 +0100
Subject: [PATCH 064/125] add fs creation

---
 common/common.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/common.C b/common/common.C
index d7ace7281..a79cecc8a 100644
--- a/common/common.C
+++ b/common/common.C
@@ -62,7 +62,7 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
                 const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
-    // std::filesystem::create_directories(folder);
+    std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
     std::cout << "exporting " << fn << std::endl;
     std::ofstream stream{fn};

From 5ce85693a28ddfa8291c4ec44dcc1720f53b1e63 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 8 Nov 2023 08:15:19 +0100
Subject: [PATCH 065/125] Fixup pass stopping vector

---
 Solver/BiCGStab/GKOBiCGStab.H | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Solver/BiCGStab/GKOBiCGStab.H b/Solver/BiCGStab/GKOBiCGStab.H
index 2cabd288c..797cb8958 100644
--- a/Solver/BiCGStab/GKOBiCGStab.H
+++ b/Solver/BiCGStab/GKOBiCGStab.H
@@ -73,7 +73,7 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto bicgstab = gko::solver::Bicgstab<scalar>::build()
-                            // .with_criteria(stoppingCriterionVec_)
+                            .with_criteria(stoppingCriterionVec_)
                             .on(exec);
         return bicgstab;
     };
@@ -83,7 +83,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto bicgstab = bicgstab::build()
-                            // .with_criteria(stoppingCriterionVec_)
+                            .with_criteria(stoppingCriterionVec_)
                             .with_generated_preconditioner(precond)
                             .on(exec);
         return bicgstab;

From 96092bd0dd6ceeccfeab8b1fff5177154fbf5517 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 8 Nov 2023 10:44:00 +0100
Subject: [PATCH 066/125] add stopping crit vect to GMRES, format

---
 HostMatrix/HostMatrix.H | 2 +-
 Solver/GMRES/GKOGMRES.H | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 9997268e1..96bb5de58 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -207,7 +207,7 @@ private:
 
     /** Iterates all interfaces and collect the corresponding cell id (row)
     ** and a unique counter
-    ** 
+    **
     ** @return vector off size nnz_non_local, with sections for each interface
     ** ret = [(1,2),(2,20), ...]
     **         i0   i1,   i...
diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 7e6497627..4d3b253af 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -70,7 +70,7 @@ public:
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
         auto gmres = gmres::build()
-                         // .with_criteria(stoppingCriterionVec_)
+                         .with_criteria(stoppingCriterionVec_)
                          .on(exec);
         return gmres;
     };
@@ -80,7 +80,7 @@ public:
                    std::shared_ptr<gko::LinOp> precond) const
     {
         auto gmres = gmres::build()
-                         // .with_criteria(stoppingCriterionVec_)
+                         .with_criteria(stoppingCriterionVec_)
                          .with_generated_preconditioner(precond)
                          .on(exec);
         return gmres;

From b80141e06e6c85bdd992990ff32f46e55a2bceea Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 13:55:59 +0100
Subject: [PATCH 067/125] pass CommunicationPattern, move free functions to
 separate TU

---
 CMakeLists.txt                         |   1 +
 DevicePersistent/Partition/Partition.H |  21 ++--
 HostMatrix/HostMatrix.C                | 137 ++++++++-----------------
 HostMatrix/HostMatrix.H                |  81 ++++++++++++---
 HostMatrix/HostMatrixFreeFunctions.C   | 117 +++++++++++++++++++++
 lduLduBase/lduLduBase.H                |   7 +-
 6 files changed, 236 insertions(+), 128 deletions(-)
 create mode 100644 HostMatrix/HostMatrixFreeFunctions.C

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 48033155e..4c4564bc1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -151,6 +151,7 @@ target_sources(
           BaseWrapper/lduBase/GKOlduBase.C
           BaseWrapper/CoupledLduBase/GKOCoupledLduBase.C
           HostMatrix/HostMatrix.C
+          HostMatrix/HostMatrixFreeFunctions.C
           Solver/CG/GKOCG.C
           Solver/BiCGStab/GKOBiCGStab.C
 #Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C
diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 64cba12d4..d83de9894 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -31,6 +31,7 @@ SourceFiles
 
 #include <ginkgo/ginkgo.hpp>
 
+#include "HostMatrix/HostMatrix.H"
 #include "DevicePersistent/Base/Base.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "common/common.H"
@@ -55,21 +56,20 @@ struct PartitionInitFunctor {
     // Array storing the ranks of the neighbouring processors
     const gko::array<label> target_sizes_;
 
-    const std::vector<std::pair<gko::index_set<label>, label>> send_idxs_;
+    const std::vector<std::pair<gko::array<label>, label>> send_idxs_;
 
     PartitionInitFunctor(
         const ExecutorHandler &exec, const label local_size,
         const label verbose, const int ranks_per_gpu,
-        const gko::array<label> &target_ids,
-        const gko::array<label> &target_sizes,
-        const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
+        const CommunicationPattern& communication_pattern
+        )
         : exec_(exec),
           local_size_(local_size),
           verbose_(verbose),
           ranks_per_gpu_(ranks_per_gpu),
-          target_ids_(target_ids),
-          target_sizes_(target_sizes),
-          send_idxs_(send_idxs)
+          target_ids_(communication_pattern.target_ids),
+          target_sizes_(communication_pattern.target_sizes),
+          send_idxs_(communication_pattern.send_idxs)
     {}
 
     void update(std::shared_ptr<part_type> persistent_partition) const
@@ -128,13 +128,12 @@ public:
     PersistentPartition(
         const objectRegistry &db, const ExecutorHandler &exec,
         const label verbose, const int ranks_per_gpu, const label elements,
-        const gko::array<label> &target_ids,
-        const gko::array<label> &target_sizes,
-        const std::vector<std::pair<gko::index_set<label>, label>> send_idxs)
+        const CommunicationPattern &communication_pattern
+        )
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, elements, verbose, ranks_per_gpu,
-                                   target_ids, target_sizes, send_idxs),
+                                   communication_pattern),
               false, verbose),
           ranks_per_gpu_(ranks_per_gpu),
           local_elements_(elements),
diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 8a5bf21d1..d5c16ffd7 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -32,38 +32,6 @@ SourceFiles
 
 namespace Foam {
 
-const lduInterfaceField *interface_getter(
-    const lduInterfaceFieldPtrsList &interfaces, const label i)
-{
-#ifdef WITH_ESI_VERSION
-    return interfaces.get(i);
-#else
-    return interfaces.operator()(i);
-#endif
-}
-
-template <class MatrixType>
-label HostMatrixWrapper<MatrixType>::count_interface_nnz(
-    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
-{
-    label ctr{0};
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-        const auto iface{interface_getter(interfaces, i)};
-
-        bool count = (proc_interfaces)
-                         ? !!isA<processorLduInterface>(iface->interface())
-                         : !isA<processorLduInterface>(iface->interface());
-        if (count) {
-            ctr += iface->interface().faceCells().size();
-        }
-    }
-    return ctr;
-}
-
-
 template <class MatrixType>
 HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
     const objectRegistry &db, const MatrixType &matrix,
@@ -80,6 +48,8 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
       exec_{db, solverControls, fieldName},
       device_id_guard_{db, fieldName, exec_.get_device_exec()},
       verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      reorder_on_copy_(
+          solverControls.lookupOrDefault<Switch>("reorderOnHost", true)),
       scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
       nrows_(matrix.diag().size()),
       local_interface_nnz_(count_interface_nnz(interfaces, false)),
@@ -153,6 +123,8 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
       exec_{db, solverControls, fieldName},
       device_id_guard_{db, fieldName, exec_.get_device_exec()},
       verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
+      reorder_on_copy_(
+          solverControls.lookupOrDefault<Switch>("reorderOnHost", true)),
       scaling_(solverControls.lookupOrDefault<scalar>("scaling", 1)),
       nrows_(matrix.diag().size()),
       local_interface_nnz_(0),
@@ -308,21 +280,20 @@ HostMatrixWrapper<MatrixType>::create_communication_pattern(
     // create index_sets
     // currently this assumes that there is only one interface to a given
     // neighbour rank
-    std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
+    std::vector<std::pair<gko::array<label>, label>> send_idxs;
     for (auto [proc, interface_cells] : interface_cell_map) {
-        label interface_size = interface_cells.size();
         auto exec = exec_.get_ref_exec();
-        send_idxs.push_back(std::pair<gko::index_set<label>, label>(
-            gko::index_set(exec, interface_size,
-                           gko::array<label>::view(exec, interface_size,
-                                                   interface_cells.data()),
-                           true),
+        send_idxs.push_back(std::pair<gko::array<label>, label>(
+            gko::array<label>(exec, interface_cells.begin(),
+                              interface_cells.end()),
             proc));
     }
 
     // convert to gko::array
-    gko::array<label> target_ids{exec_.get_ref_exec(), n_procs};
-    gko::array<label> target_sizes{exec_.get_ref_exec(), n_procs};
+    gko::array<label> target_ids{exec_.get_ref_exec(),
+                                 static_cast<size_t>(n_procs)};
+    gko::array<label> target_sizes{exec_.get_ref_exec(),
+                                   static_cast<size_t>(n_procs)};
 
     label iter = 0;
     for (const auto &[proc, size] : reduce_map) {
@@ -386,7 +357,7 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
             }
         });
 
-    word msg = "done collecting neighbouring processor cell id";
+    word msg = "done collecting neighbouring processor cell ids";
 
     LOG_2(verbose_, msg)
     return non_local_idxs;
@@ -584,6 +555,19 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     LOG_1(verbose_, "done init host matrix")
 }
 
+
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        dense[i] =
+            scale * (pos >= upper_nnz) ? diag[pos - upper_nnz] : upper[pos];
+    }
+}
+
+
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
@@ -611,66 +595,25 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
         auto couple_coeffs =
             collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
         if (is_symmetric) {
-            for (label i = 0; i < local_matrix_w_interfaces_nnz_; ++i) {
-                // where the element is stored in a combined array
-                const label pos{permute[i]};
-                scalar value;
-                // all values up to upper_nnz_ are upper_nnz_ values
-                if (pos < upper_nnz_) {
-                    value = upper[pos];
-                }
-                // all values up to upper_nnz_ are upper_nnz_ values
-                if (pos >= upper_nnz_ && pos < upper_nnz_ + diag_nnz) {
-                    value = diag[pos - upper_nnz_];
-                }
-                if (pos >= upper_nnz_ + diag_nnz) {
-                    value = -couple_coeffs[pos - upper_nnz_ - diag_nnz];
-                }
-
-                dense[i] = scaling_ * value;
-            }
+            symmetric_update_w_interface(
+                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute, scaling_,
+                diag.data(), upper.data(), lower.data(), couple_coeffs.data(),
+                dense);
         } else {
-            for (label i = 0; i < local_matrix_w_interfaces_nnz_; ++i) {
-                const label pos{permute[i]};
-                scalar value;
-                if (pos < upper_nnz_) {
-                    value = upper[pos];
-                }
-                if (pos >= upper_nnz_ && pos < 2 * upper_nnz_) {
-                    value = lower[pos - upper_nnz_];
-                }
-                if (pos >= 2 * upper_nnz_ && pos < 2 * upper_nnz_ + diag_nnz) {
-                    value = diag[pos - 2 * upper_nnz_];
-                }
-                if (pos >= 2 * upper_nnz_ + diag_nnz) {
-                    value = -couple_coeffs[pos - 2 * upper_nnz_ - diag_nnz];
-                }
-                dense[i] = scaling_ * value;
-            }
+            non_symmetric_update_w_interface(
+                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute, scaling_,
+                diag.data(), upper.data(), lower.data(), couple_coeffs.data(),
+                dense);
         }
     } else {
-        // TODO move this to a function fill_non_interfaces
         if (is_symmetric) {
-            for (label i = 0; i < local_matrix_nnz_; ++i) {
-                const label pos{permute[i]};
-                dense[i] = scaling_ * (pos >= upper_nnz_)
-                               ? diag[pos - upper_nnz_]
-                               : upper[pos];
-            }
-            return;
+            symmetric_update(local_matrix_nnz_, upper_nnz_,
+                                          permute, scaling_, diag.data(),
+                                          upper.data(), dense);
         } else {
-            for (label i = 0; i < local_matrix_nnz_; ++i) {
-                const label pos{permute[i]};
-                if (pos < upper_nnz_) {
-                    dense[i] = scaling_ * upper[pos];
-                    continue;
-                }
-                if (pos >= upper_nnz_ && pos < 2 * upper_nnz_) {
-                    dense[i] = scaling_ * lower[pos - upper_nnz_];
-                    continue;
-                }
-                dense[i] = scaling_ * diag[pos - 2 * upper_nnz_];
-            }
+            non_symmetric_update(local_matrix_nnz_, upper_nnz_,
+                                          permute, scaling_, diag.data(),
+                                          upper.data(), lower.data(), dense);
         }
     }
 }
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 96bb5de58..424975e65 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -94,10 +94,40 @@ struct CommunicationPattern {
 
     // send_idx stores the index_set ie which cells
     // are owned by the interface and the corresponding target rank
-    std::vector<std::pair<gko::index_set<label>, label>> send_idxs;
+    std::vector<std::pair<gko::array<label>, label>> send_idxs;
 };
 
 
+// Free functions
+const lduInterfaceField *interface_getter(
+    const lduInterfaceFieldPtrsList &interfaces, const label i);
+
+
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper,
+                      const scalar *lower, scalar *dense);
+
+
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, const label upper_nnz,
+                                  const label *permute, const scalar scale,
+                                  const scalar *diag, const scalar *upper,
+                                  const scalar *lower,
+                                  const scalar *couple_coeffs, scalar *dense);
+
+
+void non_symmetric_update(const label total_nnz, const label upper_nnz,
+                          const label *permute, const scalar scale,
+                          const scalar *diag, const scalar *upper,
+                          const scalar *lower, scalar *dense);
+
+
+void non_symmetric_update_w_interface(
+    const label total_nnz,const label diag_nnz, const label upper_nnz, const label *permute,
+    const scalar scale, const scalar *diag, const scalar *upper,
+    const scalar *lower, const scalar *couple_coeffs, scalar *dense);
+
+
 /* The HostMatrixWrapper class handles the conversion from OpenFOAMs lduMatrix
  * format into Ginkgo array data structures
  *
@@ -117,6 +147,11 @@ private:
 
     const label verbose_;
 
+    // Whether the matrix coefficients should be reordered
+    // during copy or on device
+    // NOTE Not implemented
+    const bool reorder_on_copy_;
+
     // multiply the complete system by this factor, ie sAx=sb
     // NOTE this could be also achieved by just fliping the sign
     const scalar scaling_;
@@ -215,16 +250,41 @@ private:
     std::vector<std::tuple<label, label>> collect_cells_on_interface(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
-    /* Based on OpenFOAMs ldu matrix format this function computes two
-     *consecutive index arrays in row major ordering and scattering indices
-     **
-     */
+    /** Based on OpenFOAMs ldu matrix format this function computes two
+     ** consecutive index arrays (local_sparsisty_.row_idxs and col_idxs) in row
+     ** major ordering and the permutation index (local_sparsity_.ldu_mapping),
+     ** which are required to create to a ginkgo matrix
+     **/
     void init_local_sparsity_pattern(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
+    /** Based on OpenFOAMs interfaces this function computes two
+     ** consecutive index arrays (non_local_sparsisty_.row_idxs and col_idxs) in
+     ** row *major ordering and the permutation index
+     ** (non_local_sparsity_.ldu_mapping), which are required to create a ginkgo
+     ** matrix
+     **/
     void init_non_local_sparsity_pattern(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
+    // updater
+
+    /** Copies the LDU matrix coefficients to local_coeffs without changing or
+     ** reinstantiating the sparsity pattern.
+     ** This uses the local_sparsity_.ldu_mapping to permute the data already
+     ** on the host to be in row major order
+     **/
+    void update_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
+
+    /** Copies the interface matrix coefficients to non_local_coeffs without
+    ** changing or reinstantiating the sparsity pattern.
+    **/
+    void update_non_local_matrix_data(
+        const lduInterfaceFieldPtrsList &interfaces,
+        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
+
 public:
     // segregated wrapper constructor
     // TODO - compute local_interface_nnzs these are matrix coefficient
@@ -245,19 +305,10 @@ public:
         const dictionary &solverControls, const word &fieldName);
 
     // TODO reimplement this at some point
-    // coupled  wrapper constructor
+    // coupled wrapper constructor
     HostMatrixWrapper(const objectRegistry &db, const MatrixType &matrix,
                       const dictionary &solverControls, const word &fieldName);
 
-    // updater
-
-    void update_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
-
-    void update_non_local_matrix_data(
-        const lduInterfaceFieldPtrsList &interfaces,
-        const FieldField<Field, scalar> &interfaceBouCoeffs) const;
 
     // getter
 
diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
new file mode 100644
index 000000000..0865947c2
--- /dev/null
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -0,0 +1,117 @@
+#include "HostMatrix.H"
+
+const lduInterfaceField *interface_getter(
+    const lduInterfaceFieldPtrsList &interfaces, const label i)
+{
+#ifdef WITH_ESI_VERSION
+    return interfaces.get(i);
+#else
+    return interfaces.operator()(i);
+#endif
+}
+
+
+// free functions - for unit testing
+void symmetric_update(const label total_nnz, const label upper_nnz,
+                      const label *permute, const scalar scale,
+                      const scalar *diag, const scalar *upper,
+                      const scalar *lower, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        dense[i] =
+            scale * (pos >= upper_nnz) ? diag[pos - upper_nnz] : upper[pos];
+    }
+}
+
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, const label upper_nnz,
+                                  const label *permute, const scalar scale,
+                                  const scalar *diag, const scalar *upper,
+                                  const scalar *lower,
+                                  const scalar *couple_coeffs, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        // where the element is stored in a combined array
+        const label pos{permute[i]};
+        scalar value;
+        // all values up to upper_nnz_ are upper_nnz_ values
+        if (pos < upper_nnz) {
+            value = upper[pos];
+        }
+        // all values up to upper_nnz_ are upper_nnz_ values
+        if (pos >= upper_nnz && pos < upper_nnz + diag_nnz) {
+            value = diag[pos - upper_nnz];
+        }
+        if (pos >= upper_nnz + diag_nnz) {
+            value = -couple_coeffs[pos - upper_nnz - diag_nnz];
+        }
+        dense[i] = scale * value;
+    }
+}
+
+void non_symmetric_update_w_interface(
+    const label total_nnz,const label diag_nnz, const label upper_nnz, const label *permute,
+    const scalar scale, const scalar *diag, const scalar *upper,
+    const scalar *lower, const scalar *couple_coeffs, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        scalar value;
+        if (pos < upper_nnz) {
+            value = upper[pos];
+        }
+        if (pos >= upper_nnz && pos < 2 * upper_nnz) {
+            value = lower[pos - upper_nnz];
+        }
+        if (pos >= 2 * upper_nnz && pos < 2 * upper_nnz + diag_nnz) {
+            value = diag[pos - 2 * upper_nnz];
+        }
+        if (pos >= 2 * upper_nnz + diag_nnz) {
+            value = -couple_coeffs[pos - 2 * upper_nnz - diag_nnz];
+        }
+        dense[i] = scale * value;
+    }
+}
+
+
+void non_symmetric_update(const label total_nnz, const label upper_nnz,
+                          const label *permute, const scalar scale,
+                          const scalar *diag, const scalar *upper,
+                          const scalar *lower, scalar *dense)
+{
+    for (label i = 0; i < total_nnz; ++i) {
+        const label pos{permute[i]};
+        if (pos < upper_nnz) {
+            dense[i] = scale * upper[pos];
+            continue;
+        }
+        if (pos >= upper_nnz && pos < 2 * upper_nnz) {
+            dense[i] = scale * lower[pos - upper_nnz];
+            continue;
+        }
+        dense[i] = scale * diag[pos - 2 * upper_nnz];
+    }
+}
+
+
+template <class MatrixType>
+label HostMatrixWrapper<MatrixType>::count_interface_nnz(
+    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
+{
+    label ctr{0};
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+
+        bool count = (proc_interfaces)
+                         ? !!isA<processorLduInterface>(iface->interface())
+                         : !isA<processorLduInterface>(iface->interface());
+        if (count) {
+            ctr += iface->interface().faceCells().size();
+        }
+    }
+
+    return ctr;
+}
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 9114a532f..91bb35d4a 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -217,17 +217,14 @@ public:
     {
         auto ref_exec = this->get_exec_handler().get_ref_exec();
 
-        const auto communication_pattern = this->get_communication_pattern();
-
         PersistentPartition partition{
             db_,
             this->get_exec_handler(),
             verbose_,
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
             psi.size(),  // number of elements
-            communication_pattern.target_ids,
-            communication_pattern.target_sizes,
-            communication_pattern.send_idxs};
+            this->get_communication_pattern()
+        };
 
         // TODO this is now the local matrix part for dist_A
         MatrixWrapper dist_A{db_,

From cd91cba96ae4a3646b12df5479597e7e9f703ef5 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 13:58:37 +0100
Subject: [PATCH 068/125] format

---
 HostMatrix/HostMatrixFreeFunctions.C | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index 0865947c2..58f32b259 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -24,10 +24,10 @@ void symmetric_update(const label total_nnz, const label upper_nnz,
     }
 }
 
-void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, const label upper_nnz,
-                                  const label *permute, const scalar scale,
-                                  const scalar *diag, const scalar *upper,
-                                  const scalar *lower,
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
+                                  const label upper_nnz, const label *permute,
+                                  const scalar scale, const scalar *diag,
+                                  const scalar *upper, const scalar *lower,
                                   const scalar *couple_coeffs, scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
@@ -50,9 +50,10 @@ void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, c
 }
 
 void non_symmetric_update_w_interface(
-    const label total_nnz,const label diag_nnz, const label upper_nnz, const label *permute,
-    const scalar scale, const scalar *diag, const scalar *upper,
-    const scalar *lower, const scalar *couple_coeffs, scalar *dense)
+    const label total_nnz, const label diag_nnz, const label upper_nnz,
+    const label *permute, const scalar scale, const scalar *diag,
+    const scalar *upper, const scalar *lower, const scalar *couple_coeffs,
+    scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
         const label pos{permute[i]};

From ec97363a32aac7a6150bbe777aba49e5052832b0 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 15:53:11 +0100
Subject: [PATCH 069/125] improve documentation, remove unneeded parameters

---
 HostMatrix/HostMatrix.C              |  46 +++++++----
 HostMatrix/HostMatrix.H              | 109 +++++++++++++++++++++++----
 HostMatrix/HostMatrixFreeFunctions.C |  35 ++-------
 3 files changed, 135 insertions(+), 55 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index d5c16ffd7..58345a68c 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -173,6 +173,27 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
                          << abort(FatalError);
 }
 
+template <class MatrixType>
+label HostMatrixWrapper<MatrixType>::count_interface_nnz(
+    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
+{
+    label ctr{0};
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+        const auto iface{interface_getter(interfaces, i)};
+
+        bool count = (proc_interfaces)
+                         ? !!isA<processorLduInterface>(iface->interface())
+                         : !isA<processorLduInterface>(iface->interface());
+        if (count) {
+            ctr += iface->interface().faceCells().size();
+        }
+    }
+
+    return ctr;
+}
 
 template <class MatrixType>
 std::vector<scalar> HostMatrixWrapper<MatrixType>::collect_interface_coeffs(
@@ -595,25 +616,24 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
         auto couple_coeffs =
             collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
         if (is_symmetric) {
-            symmetric_update_w_interface(
-                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute, scaling_,
-                diag.data(), upper.data(), lower.data(), couple_coeffs.data(),
-                dense);
+            symmetric_update_w_interface(local_matrix_w_interfaces_nnz_,
+                                         diag_nnz, upper_nnz_, permute,
+                                         scaling_, diag.data(), upper.data(),
+                                         couple_coeffs.data(), dense);
         } else {
             non_symmetric_update_w_interface(
-                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute, scaling_,
-                diag.data(), upper.data(), lower.data(), couple_coeffs.data(),
-                dense);
+                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute,
+                scaling_, diag.data(), upper.data(), lower.data(),
+                couple_coeffs.data(), dense);
         }
     } else {
         if (is_symmetric) {
-            symmetric_update(local_matrix_nnz_, upper_nnz_,
-                                          permute, scaling_, diag.data(),
-                                          upper.data(), dense);
+            symmetric_update(local_matrix_nnz_, upper_nnz_, permute, scaling_,
+                             diag.data(), upper.data(), dense);
         } else {
-            non_symmetric_update(local_matrix_nnz_, upper_nnz_,
-                                          permute, scaling_, diag.data(),
-                                          upper.data(), lower.data(), dense);
+            non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                 scaling_, diag.data(), upper.data(),
+                                 lower.data(), dense);
         }
     }
 }
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 424975e65..393cc665c 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -103,29 +103,110 @@ const lduInterfaceField *interface_getter(
     const lduInterfaceFieldPtrsList &interfaces, const label i);
 
 
+/** Perform update of the coefficients array in row major order from a symmetric
+ *ldu matrix
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ *upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param out pointer to data of output matrix
+ **/
 void symmetric_update(const label total_nnz, const label upper_nnz,
                       const label *permute, const scalar scale,
-                      const scalar *diag, const scalar *upper,
-                      const scalar *lower, scalar *dense);
-
-
-void symmetric_update_w_interface(const label total_nnz, const label diag_nnz, const label upper_nnz,
-                                  const label *permute, const scalar scale,
-                                  const scalar *diag, const scalar *upper,
-                                  const scalar *lower,
-                                  const scalar *couple_coeffs, scalar *dense);
-
-
+                      const scalar *diag, const scalar *upper, scalar *out);
+
+
+/** Perform update of the coefficients array in row major order from a
+ *non-symmetric ldu matrix
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ *upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for lower entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param lower pointer to lower triangular entries of the original ldu matrix
+ ** @param out pointer to data of output matrix
+ **/
 void non_symmetric_update(const label total_nnz, const label upper_nnz,
                           const label *permute, const scalar scale,
                           const scalar *diag, const scalar *upper,
                           const scalar *lower, scalar *dense);
 
 
+/** Perform update of the coefficients array in row major order from a
+ ** symmetric ldu matrix including local interface data.
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ ** upper
+ **
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for interface entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param interface pointer to interface entries
+ ** @param out pointer to data of output matrix
+ **/
+void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
+                                  const label upper_nnz, const label *permute,
+                                  const scalar scale, const scalar *diag,
+                                  const scalar *upper, const scalar *interface,
+                                  scalar *dense);
+
+
+/** Perform update of the coefficients array in row major order from a
+ ** non-symmetric ldu matrix including local interface data.
+ **
+ ** @note this assumes out to be on the host
+ ** @note performs contiguous writes into out and random access of diag and
+ ** upper
+ ** @param total_nnz number of non zeros entries in the matrix
+ ** @param upper_nnz number of non zeros entries in the upper triangular matrix
+ ** @param permute mapping between the original position in the ldu input data
+ **        and the position in the row major ordered output data
+ **        ie. permute[i] maps out[i] to original position in diag or upper
+ **        with
+ **        upper_nnz <= permute < upper_nnz + diag_nnz for diag entries
+ **        upper_nnz + diag_nnz <= permute for lower entries
+ **        2*upper_nnz + diag_nnz <= permute for interface entries
+ ** @param scale scaling factor to scale the matrix entries
+ ** @param diag pointer to diagonal entries of the original ldu matrix
+ ** @param upper pointer to upper triangular entries of the original ldu matrix
+ ** @param lower pointer to lower triangular entries of the original ldu matrix
+ ** @param interface pointer to interface entries
+ ** @param out pointer to data of output matrix
+ **/
 void non_symmetric_update_w_interface(
-    const label total_nnz,const label diag_nnz, const label upper_nnz, const label *permute,
-    const scalar scale, const scalar *diag, const scalar *upper,
-    const scalar *lower, const scalar *couple_coeffs, scalar *dense);
+    const label total_nnz, const label diag_nnz, const label upper_nnz,
+    const label *permute, const scalar scale, const scalar *diag,
+    const scalar *upper, const scalar *lower, const scalar *couple_coeffs,
+    scalar *dense);
 
 
 /* The HostMatrixWrapper class handles the conversion from OpenFOAMs lduMatrix
diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index 58f32b259..6a7cd8338 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -15,11 +15,11 @@ const lduInterfaceField *interface_getter(
 void symmetric_update(const label total_nnz, const label upper_nnz,
                       const label *permute, const scalar scale,
                       const scalar *diag, const scalar *upper,
-                      const scalar *lower, scalar *dense)
+                      scalar *out)
 {
     for (label i = 0; i < total_nnz; ++i) {
         const label pos{permute[i]};
-        dense[i] =
+        out[i] =
             scale * (pos >= upper_nnz) ? diag[pos - upper_nnz] : upper[pos];
     }
 }
@@ -27,8 +27,8 @@ void symmetric_update(const label total_nnz, const label upper_nnz,
 void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
                                   const label upper_nnz, const label *permute,
                                   const scalar scale, const scalar *diag,
-                                  const scalar *upper, const scalar *lower,
-                                  const scalar *couple_coeffs, scalar *dense)
+                                  const scalar *upper, 
+                                  const scalar *interface, scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
         // where the element is stored in a combined array
@@ -43,7 +43,7 @@ void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
             value = diag[pos - upper_nnz];
         }
         if (pos >= upper_nnz + diag_nnz) {
-            value = -couple_coeffs[pos - upper_nnz - diag_nnz];
+            value = -interface[pos - upper_nnz - diag_nnz];
         }
         dense[i] = scale * value;
     }
@@ -52,7 +52,7 @@ void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
 void non_symmetric_update_w_interface(
     const label total_nnz, const label diag_nnz, const label upper_nnz,
     const label *permute, const scalar scale, const scalar *diag,
-    const scalar *upper, const scalar *lower, const scalar *couple_coeffs,
+    const scalar *upper, const scalar *lower, const scalar *interface,
     scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
@@ -68,7 +68,7 @@ void non_symmetric_update_w_interface(
             value = diag[pos - 2 * upper_nnz];
         }
         if (pos >= 2 * upper_nnz + diag_nnz) {
-            value = -couple_coeffs[pos - 2 * upper_nnz - diag_nnz];
+            value = -interface[pos - 2 * upper_nnz - diag_nnz];
         }
         dense[i] = scale * value;
     }
@@ -95,24 +95,3 @@ void non_symmetric_update(const label total_nnz, const label upper_nnz,
 }
 
 
-template <class MatrixType>
-label HostMatrixWrapper<MatrixType>::count_interface_nnz(
-    const lduInterfaceFieldPtrsList &interfaces, bool proc_interfaces) const
-{
-    label ctr{0};
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-        const auto iface{interface_getter(interfaces, i)};
-
-        bool count = (proc_interfaces)
-                         ? !!isA<processorLduInterface>(iface->interface())
-                         : !isA<processorLduInterface>(iface->interface());
-        if (count) {
-            ctr += iface->interface().faceCells().size();
-        }
-    }
-
-    return ctr;
-}

From 1698f3b7b977a89668471f45279b3bd712101e27 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 15:56:21 +0100
Subject: [PATCH 070/125] format

---
 HostMatrix/HostMatrixFreeFunctions.C | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index 6a7cd8338..460587279 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -14,8 +14,7 @@ const lduInterfaceField *interface_getter(
 // free functions - for unit testing
 void symmetric_update(const label total_nnz, const label upper_nnz,
                       const label *permute, const scalar scale,
-                      const scalar *diag, const scalar *upper,
-                      scalar *out)
+                      const scalar *diag, const scalar *upper, scalar *out)
 {
     for (label i = 0; i < total_nnz; ++i) {
         const label pos{permute[i]};
@@ -27,8 +26,8 @@ void symmetric_update(const label total_nnz, const label upper_nnz,
 void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
                                   const label upper_nnz, const label *permute,
                                   const scalar scale, const scalar *diag,
-                                  const scalar *upper, 
-                                  const scalar *interface, scalar *dense)
+                                  const scalar *upper, const scalar *interface,
+                                  scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
         // where the element is stored in a combined array
@@ -49,11 +48,13 @@ void symmetric_update_w_interface(const label total_nnz, const label diag_nnz,
     }
 }
 
-void non_symmetric_update_w_interface(
-    const label total_nnz, const label diag_nnz, const label upper_nnz,
-    const label *permute, const scalar scale, const scalar *diag,
-    const scalar *upper, const scalar *lower, const scalar *interface,
-    scalar *dense)
+void non_symmetric_update_w_interface(const label total_nnz,
+                                      const label diag_nnz,
+                                      const label upper_nnz,
+                                      const label *permute, const scalar scale,
+                                      const scalar *diag, const scalar *upper,
+                                      const scalar *lower,
+                                      const scalar *interface, scalar *dense)
 {
     for (label i = 0; i < total_nnz; ++i) {
         const label pos{permute[i]};
@@ -93,5 +94,3 @@ void non_symmetric_update(const label total_nnz, const label upper_nnz,
         dense[i] = scale * diag[pos - 2 * upper_nnz];
     }
 }
-
-

From ff0d5319742148c4a9a085756e52e13f2b10a980 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 10 Nov 2023 17:36:19 +0100
Subject: [PATCH 071/125] start unit tests

---
 CMakeLists.txt                 |  8 +++++++-
 unitTests/CMakeLists.txt       | 32 ++++++++++++++++++++++++++++++++
 unitTests/test_hello_world.cpp | 17 +++++++++++++++++
 3 files changed, 56 insertions(+), 1 deletion(-)
 create mode 100644 unitTests/CMakeLists.txt
 create mode 100644 unitTests/test_hello_world.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4c4564bc1..ca457a586 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,6 +20,7 @@ option(OGL_USE_EXTERNAL_GINKGO "Use external ginkgo" FALSE)
 option(OGL_DATA_VALIDATION "Check if resulting matrix data is valid" FALSE)
 option(OGL_USE_FOAM_FOUNDATION_VERSION
        "Build OGL for the OpenFOAM Foundation version" FALSE)
+option(OGL_BUILD_UNITTEST "Build OGL unittests" FALSE)
 option(GINKGO_BUILD_CUDA "Build Ginkgo with cuda backend" FALSE)
 option(GINKGO_BUILD_HIP "Build Ginkgo with hip backend" FALSE)
 option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE)
@@ -182,7 +183,7 @@ target_sources(
 
 target_include_directories(
   OGL SYSTEM
-  PRIVATE $ENV{FOAM_SRC}/finiteVolume/lnInclude
+  PUBLIC $ENV{FOAM_SRC}/finiteVolume/lnInclude
           $ENV{FOAM_SRC}/meshTools/lnInclude
           $ENV{FOAM_SRC}/OpenFOAM/lnInclude
           $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude
@@ -207,4 +208,9 @@ if(EXISTS $ENV{WM_PROJECT_DIR}/CONTRIBUTORS.md)
   target_compile_definitions(OGL PRIVATE WITH_ESI_VERSION=1)
 endif()
 
+if(${OGL_BUILD_UNITTEST})
+    add_subdirectory(unitTests)
+    enable_testing()
+endif()
+
 install(TARGETS OGL DESTINATION $ENV{FOAM_USER_LIBBIN})
diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
new file mode 100644
index 000000000..04e7a1a6d
--- /dev/null
+++ b/unitTests/CMakeLists.txt
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2023 NeoFOAM authors
+include(FetchContent)
+FetchContent_Declare(
+  googletest
+  URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip
+)
+FetchContent_MakeAvailable(googletest)
+
+include_directories(
+    ${googletest_SOURCE_DIR}/googletest/include
+)
+
+add_executable(
+  core_test
+  "test_hello_world.cpp"
+)
+
+# Your-external "mylib", add GLOBAL if the imported library is located in directories above the current.
+add_library( OpenFOAM SHARED IMPORTED )
+# You can define two import-locations: one for debug and one for release.
+set_target_properties( OpenFOAM PROPERTIES IMPORTED_LOCATION $ENV{FOAM_LIBBIN}/libOpenFOAM.so)
+
+
+target_link_libraries(
+  core_test
+  GTest::gtest_main OGL OpenFOAM
+)
+
+include(GoogleTest)
+gtest_discover_tests(core_test)
+
diff --git a/unitTests/test_hello_world.cpp b/unitTests/test_hello_world.cpp
new file mode 100644
index 000000000..fd70fe2f8
--- /dev/null
+++ b/unitTests/test_hello_world.cpp
@@ -0,0 +1,17 @@
+#include "gtest/gtest.h"
+#include "HostMatrix/HostMatrix.H"
+
+TEST(HelloWord, check_return_value){
+    int my_argc {1};
+    string str = "icoFoam";
+    std::vector<char> cstr(str.c_str(), str.c_str() + str.size() + 1);
+    char* argv = cstr.data();
+    char** argvp = &argv;
+
+    Foam::argList args(my_argc, argvp);
+    Foam::Time runTime(Foam::Time::controlDictName, argv);
+
+    Foam::objectRegistry objectReg{runTime};
+
+    EXPECT_EQ(1, 1);
+}

From 596f6461ef102abc5a1e9ba77797f24bec7f1512 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 13 Nov 2023 14:56:35 +0100
Subject: [PATCH 072/125] implement basic unit test

---
 unitTests/test_HostMatrix.C | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 unitTests/test_HostMatrix.C

diff --git a/unitTests/test_HostMatrix.C b/unitTests/test_HostMatrix.C
new file mode 100644
index 000000000..71d5a116b
--- /dev/null
+++ b/unitTests/test_HostMatrix.C
@@ -0,0 +1,27 @@
+#include "gtest/gtest.h"
+#include "HostMatrix/HostMatrix.H"
+
+TEST(HelloWord, symmetric_update){
+    /* test a 5x5 symmetric matrix
+     * A =
+     * | 1  1  .  2  .  |
+     * | 1  1  1  .  2  |
+     * | .  1  1  1  .  |
+     * | 2  .  1  1  1  |
+     * | .  2  .  1  1  |
+     */
+
+    std::vector<scalar> d { 1., 1., 1., 1., 1. }; 
+    std::vector<scalar> u { 1., 2., 1., 2., 1. }; 
+    std::vector<label>  p { 6, 0, 1, 7, 2, 3, 8, 4, 9};
+
+    label total_nnz {15};
+    label upper_nnz {5};
+
+    std::vector<scalar> res {15, 0.0};
+    std::vector<scalar> exp {1.,1.,2.,1.,1.,1.,2.,1.,1.,1.,2.,1.,1.,1.,2.,1.,1.};
+
+    Foam::symmetric_update(total_nnz, upper_nnz, p.data(), 1.0, d.data(), u.data(), res.data());
+
+    EXPECT_EQ(res, exp);
+}

From d20a94bc480113972a9c61d5a4adffa2417b08ac Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 13 Nov 2023 14:57:32 +0100
Subject: [PATCH 073/125] added missing namespace, fix double definition

---
 HostMatrix/HostMatrix.C              | 12 ------------
 HostMatrix/HostMatrixFreeFunctions.C |  4 ++++
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 58345a68c..f2e2ff5e2 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -577,18 +577,6 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
 }
 
 
-void symmetric_update(const label total_nnz, const label upper_nnz,
-                      const label *permute, const scalar scale,
-                      const scalar *diag, const scalar *upper, scalar *dense)
-{
-    for (label i = 0; i < total_nnz; ++i) {
-        const label pos{permute[i]};
-        dense[i] =
-            scale * (pos >= upper_nnz) ? diag[pos - upper_nnz] : upper[pos];
-    }
-}
-
-
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index 460587279..975d252e3 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -1,5 +1,7 @@
 #include "HostMatrix.H"
 
+namespace Foam {
+
 const lduInterfaceField *interface_getter(
     const lduInterfaceFieldPtrsList &interfaces, const label i)
 {
@@ -94,3 +96,5 @@ void non_symmetric_update(const label total_nnz, const label upper_nnz,
         dense[i] = scale * diag[pos - 2 * upper_nnz];
     }
 }
+
+}  // namespace Foam

From 0ae6b3586685aa4bb5879e22f7e0f09cc65041dc Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 13 Nov 2023 14:58:11 +0100
Subject: [PATCH 074/125] delete unneeded file

---
 unitTests/test_hello_world.cpp | 17 -----------------
 1 file changed, 17 deletions(-)
 delete mode 100644 unitTests/test_hello_world.cpp

diff --git a/unitTests/test_hello_world.cpp b/unitTests/test_hello_world.cpp
deleted file mode 100644
index fd70fe2f8..000000000
--- a/unitTests/test_hello_world.cpp
+++ /dev/null
@@ -1,17 +0,0 @@
-#include "gtest/gtest.h"
-#include "HostMatrix/HostMatrix.H"
-
-TEST(HelloWord, check_return_value){
-    int my_argc {1};
-    string str = "icoFoam";
-    std::vector<char> cstr(str.c_str(), str.c_str() + str.size() + 1);
-    char* argv = cstr.data();
-    char** argvp = &argv;
-
-    Foam::argList args(my_argc, argvp);
-    Foam::Time runTime(Foam::Time::controlDictName, argv);
-
-    Foam::objectRegistry objectReg{runTime};
-
-    EXPECT_EQ(1, 1);
-}

From b60d98902eaae99903845f36af212a8782d08e1d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 13 Nov 2023 15:01:30 +0100
Subject: [PATCH 075/125] link to stdfs

---
 CMakeLists.txt              |  3 ++-
 unitTests/test_HostMatrix.C | 23 +++++++++++++----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca457a586..0a9cbdf15 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -93,6 +93,7 @@ if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
 #- Rpass-analysis=loop-vectorize  -Rpass-missed=loop-vectorize
 )
 
+
 elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
   add_definitions(-DWITH_GNU)
 
@@ -190,7 +191,7 @@ target_include_directories(
           ${CMAKE_CURRENT_SOURCE_DIR}
           )
 
-target_link_libraries(OGL PUBLIC Ginkgo::ginkgo)
+target_link_libraries(OGL PUBLIC Ginkgo::ginkgo stdc++fs)
 
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)
diff --git a/unitTests/test_HostMatrix.C b/unitTests/test_HostMatrix.C
index 71d5a116b..3d731fcfd 100644
--- a/unitTests/test_HostMatrix.C
+++ b/unitTests/test_HostMatrix.C
@@ -1,7 +1,8 @@
-#include "gtest/gtest.h"
 #include "HostMatrix/HostMatrix.H"
+#include "gtest/gtest.h"
 
-TEST(HelloWord, symmetric_update){
+TEST(HelloWord, symmetric_update)
+{
     /* test a 5x5 symmetric matrix
      * A =
      * | 1  1  .  2  .  |
@@ -11,17 +12,19 @@ TEST(HelloWord, symmetric_update){
      * | .  2  .  1  1  |
      */
 
-    std::vector<scalar> d { 1., 1., 1., 1., 1. }; 
-    std::vector<scalar> u { 1., 2., 1., 2., 1. }; 
-    std::vector<label>  p { 6, 0, 1, 7, 2, 3, 8, 4, 9};
+    std::vector<scalar> d{1., 1., 1., 1., 1.};
+    std::vector<scalar> u{1., 2., 1., 2., 1.};
+    std::vector<label> p{6, 0, 1, 7, 2, 3, 8, 4, 9};
 
-    label total_nnz {15};
-    label upper_nnz {5};
+    label total_nnz{15};
+    label upper_nnz{5};
 
-    std::vector<scalar> res {15, 0.0};
-    std::vector<scalar> exp {1.,1.,2.,1.,1.,1.,2.,1.,1.,1.,2.,1.,1.,1.,2.,1.,1.};
+    std::vector<scalar> res{15, 0.0};
+    std::vector<scalar> exp{1., 1., 2., 1., 1., 1., 2., 1., 1.,
+                            1., 2., 1., 1., 1., 2., 1., 1.};
 
-    Foam::symmetric_update(total_nnz, upper_nnz, p.data(), 1.0, d.data(), u.data(), res.data());
+    Foam::symmetric_update(total_nnz, upper_nnz, p.data(), 1.0, d.data(),
+                           u.data(), res.data());
 
     EXPECT_EQ(res, exp);
 }

From e1fbac32110667adb52a1ba8a5e50ec008ccf84e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 13 Nov 2023 21:18:05 +0100
Subject: [PATCH 076/125] add basic unit test

---
 unitTests/CMakeLists.txt    | 28 ++++++++++++++++++++--------
 unitTests/test_HostMatrix.C | 13 +++++++------
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 04e7a1a6d..9d611e5c3 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 # SPDX-FileCopyrightText: 2023 NeoFOAM authors
+option(INSTALL_GTEST "Enable installation of googletest. (Projects embedding googletest may want to turn this OFF.)" OFF)
 include(FetchContent)
 FetchContent_Declare(
   googletest
@@ -12,21 +13,32 @@ include_directories(
 )
 
 add_executable(
-  core_test
-  "test_hello_world.cpp"
+   matrixConversion
+   "test_HostMatrix.C"
 )
 
 # Your-external "mylib", add GLOBAL if the imported library is located in directories above the current.
-add_library( OpenFOAM SHARED IMPORTED )
+add_library( libOpenFOAM SHARED IMPORTED )
+
 # You can define two import-locations: one for debug and one for release.
-set_target_properties( OpenFOAM PROPERTIES IMPORTED_LOCATION $ENV{FOAM_LIBBIN}/libOpenFOAM.so)
+set_target_properties(libOpenFOAM
+    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libOpenFOAM.so
+    )
 
+add_library(libfiniteVolume SHARED IMPORTED )
+set_target_properties(libfiniteVolume
+    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libfiniteVolume.so
+    )
 
 target_link_libraries(
-  core_test
-  GTest::gtest_main OGL OpenFOAM
+    matrixConversion
+  PUBLIC
+      GTest::gtest_main 
+      OGL
+      libOpenFOAM
+      libfiniteVolume
 )
 
-include(GoogleTest)
-gtest_discover_tests(core_test)
 
+include(GoogleTest)
+gtest_discover_tests(matrixConversion)
diff --git a/unitTests/test_HostMatrix.C b/unitTests/test_HostMatrix.C
index 3d731fcfd..a64e60cef 100644
--- a/unitTests/test_HostMatrix.C
+++ b/unitTests/test_HostMatrix.C
@@ -1,7 +1,7 @@
 #include "HostMatrix/HostMatrix.H"
 #include "gtest/gtest.h"
 
-TEST(HelloWord, symmetric_update)
+TEST(HostMatrixConversion, symmetric_update)
 {
     /* test a 5x5 symmetric matrix
      * A =
@@ -13,13 +13,14 @@ TEST(HelloWord, symmetric_update)
      */
 
     std::vector<scalar> d{1., 1., 1., 1., 1.};
-    std::vector<scalar> u{1., 2., 1., 2., 1.};
-    std::vector<label> p{6, 0, 1, 7, 2, 3, 8, 4, 9};
+    std::vector<scalar> u{1., 2., 1., 2., 1., 1.};
+    std::vector<label> p{6, 0, 1, 0, 7, 2, 3, 2, 8, 4, 1, 4, 9, 5, 3, 5, 10};
 
-    label total_nnz{15};
-    label upper_nnz{5};
+    label total_nnz{17};
+    label upper_nnz{6};
 
-    std::vector<scalar> res{15, 0.0};
+    std::vector<scalar> res{0., 0., 0., 0., 0., 0., 0., 0., 0.,
+                            0., 0., 0., 0., 0., 0., 0., 0.};
     std::vector<scalar> exp{1., 1., 2., 1., 1., 1., 2., 1., 1.,
                             1., 2., 1., 1., 1., 2., 1., 1.};
 

From c88cacdea1a9db046a58a509c29547f0a78ef99e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 14 Nov 2023 08:09:28 +0100
Subject: [PATCH 077/125] add non-symmetric test

---
 unitTests/test_HostMatrix.C | 38 +++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/unitTests/test_HostMatrix.C b/unitTests/test_HostMatrix.C
index a64e60cef..975d11ab2 100644
--- a/unitTests/test_HostMatrix.C
+++ b/unitTests/test_HostMatrix.C
@@ -29,3 +29,41 @@ TEST(HostMatrixConversion, symmetric_update)
 
     EXPECT_EQ(res, exp);
 }
+
+TEST(HostMatrixConversion, non_symmetric_update)
+{
+    /* test a 5x5 symmetric matrix
+     * A =
+     * | 1  1  .  2  .  |
+     * | 2  1  1  .  2  |
+     * | .  2  1  1  .  |
+     * | 3  .  2  1  1  |
+     * | .  3  .  2  1  |
+     */
+
+    std::vector<scalar> d{1., 1., 1., 1., 1.};
+    std::vector<scalar> u{1., 2., 1., 2., 1., 1.};
+    std::vector<scalar> l{2., 2., 3., 2., 3., 2.};
+    std::vector<label> p{12, 0, 1, 
+        6, 13, 2, 3,
+        7, 14, 4, 
+        8, 9, 15, 5, 
+        10, 11, 16};
+
+    label total_nnz{17};
+    label upper_nnz{6};
+
+    std::vector<scalar> res{0., 0., 0., 0., 0., 0., 0., 0., 0.,
+                            0., 0., 0., 0., 0., 0., 0., 0.};
+    std::vector<scalar> exp{
+        1., 1., 2.,
+        2., 1., 1., 2.,
+        2., 1., 1.,
+        3., 2., 1., 1.,
+        3., 2., 1.};
+
+    Foam::non_symmetric_update(total_nnz, upper_nnz, p.data(), 1.0, d.data(),
+                           u.data(), l.data(), res.data());
+
+    EXPECT_EQ(res, exp);
+}

From af534a6ef788b9a36ff91cf5eda30e18bf044d8e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 15 Nov 2023 10:38:10 +0100
Subject: [PATCH 078/125] move init_sparsity to free function and add unit test

---
 HostMatrix/HostMatrix.C              | 45 +---------------------
 HostMatrix/HostMatrix.H              | 17 +++++++++
 HostMatrix/HostMatrixFreeFunctions.C | 53 ++++++++++++++++++++++++++
 unitTests/test_HostMatrix.C          | 56 ++++++++++++++++++++++------
 4 files changed, 116 insertions(+), 55 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index f2e2ff5e2..153aee5d9 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -431,18 +431,11 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
 
     // row of upper, col of lower
     const auto lower = lower_local.get_const_data();
-
     // col of upper, row of lower
     const auto upper = upper_local.get_const_data();
 
     auto rows = local_sparsity_.row_idxs_.get_data();
     auto cols = local_sparsity_.col_idxs_.get_data();
-
-    label element_ctr = 0;
-    label upper_ctr = 0;
-
-    std::vector<std::vector<std::pair<label, label>>> lower_stack(upper_nnz_);
-
     const auto permute = local_sparsity_.ldu_mapping_.get_data();
 
     // Scan through given rows and insert row and column indices into array
@@ -454,42 +447,8 @@ void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     //  local_sparsity to size of nrows_w_interfaces, if interfaces exist
     //  local_sparsity is only valid till nrows_
     label after_neighbours = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
-    for (label row = 0; row < nrows_; row++) {
-        // add lower elements
-        // for now just scan till current upper ctr
-        for (const auto &[stored_upper_ctr, col] : lower_stack[row]) {
-            rows[element_ctr] = row;
-            cols[element_ctr] = col;
-            permute[element_ctr] = stored_upper_ctr;
-            element_ctr++;
-        }
-
-        // add diagonal elements
-        rows[element_ctr] = row;
-        cols[element_ctr] = row;
-        permute[element_ctr] = after_neighbours + row;
-        element_ctr++;
-
-        // add upper elements
-        // these are the transpose of the lower elements which are stored in
-        // row major order.
-        label row_upper = lower[upper_ctr];
-        while (upper_ctr < upper_nnz_ && row_upper == row) {
-            const label col_upper = upper[upper_ctr];
-
-            rows[element_ctr] = row_upper;
-            cols[element_ctr] = col_upper;
-
-            // insert into lower_stack
-            lower_stack[col_upper].emplace_back(
-                (is_symmetric) ? upper_ctr : upper_ctr + upper_nnz_, row_upper);
-            permute[element_ctr] = upper_ctr;
-
-            element_ctr++;
-            upper_ctr++;
-            row_upper = lower[upper_ctr];
-        }
-    }
+    init_local_sparsity(nrows_, upper_nnz_, is_symmetric, upper, lower, rows,
+                        cols, permute);
 
     // if no local interfaces are present we are done here
     // otherwise we need to add local interfaces in order
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 393cc665c..6aa762589 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -102,6 +102,23 @@ struct CommunicationPattern {
 const lduInterfaceField *interface_getter(
     const lduInterfaceFieldPtrsList &interfaces, const label i);
 
+/** Write contiguous row and col indices from OpenFOAM lower and upper indices
+ **
+ **
+ ** @param nrows number of rows
+ ** @param upper_nnz number of non zeros in the upper triangular matrix
+ ** @param is_symmetric whether matrix is symmetric, in the symmetric case
+ ** the lower elemnts indices in the permute array are computed differently
+ ** @param upper pointer to array of
+ ** @param lower pointer to rows array
+ ** @param rows pointer to rows array
+ ** @param cols pointer to columns array
+ ** @param permute pointer to permuter array
+ */
+void init_local_sparsity(const label nrows, const label upper_nnz,
+                         const bool is_symmetric, const label *upper,
+                         const label *lower, label *rows, label *cols,
+                         label *permute);
 
 /** Perform update of the coefficients array in row major order from a symmetric
  *ldu matrix
diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index 975d252e3..a8d1c908f 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -97,4 +97,57 @@ void non_symmetric_update(const label total_nnz, const label upper_nnz,
     }
 }
 
+void init_local_sparsity(
+        const label nrows,
+        const label upper_nnz,
+        const bool is_symmetric,
+        const label* upper,
+        const label* lower,
+        label *rows,
+        label *cols,
+        label *permute
+        ){
+    label after_neighbours = (is_symmetric) ? upper_nnz : 2 * upper_nnz;
+    label element_ctr = 0;
+    label upper_ctr = 0;
+    std::vector<std::vector<std::pair<label, label>>> lower_stack(upper_nnz);
+    for (label row = 0; row < nrows; row++) {
+        // add lower elements
+        // for now just scan till current upper ctr
+        for (const auto &[stored_upper_ctr, col] : lower_stack[row]) {
+            rows[element_ctr] = row;
+            cols[element_ctr] = col;
+            permute[element_ctr] = stored_upper_ctr;
+            element_ctr++;
+        }
+
+        // add diagonal elements
+        rows[element_ctr] = row;
+        cols[element_ctr] = row;
+        permute[element_ctr] = after_neighbours + row;
+        element_ctr++;
+
+        // add upper elements
+        // these are the transpose of the lower elements which are stored in
+        // row major order.
+        label row_upper = lower[upper_ctr];
+        while (upper_ctr < upper_nnz && row_upper == row) {
+            const label col_upper = upper[upper_ctr];
+
+            rows[element_ctr] = row_upper;
+            cols[element_ctr] = col_upper;
+
+            // insert into lower_stack
+            lower_stack[col_upper].emplace_back(
+                (is_symmetric) ? upper_ctr : upper_ctr + upper_nnz, row_upper);
+            permute[element_ctr] = upper_ctr;
+
+            element_ctr++;
+            upper_ctr++;
+            row_upper = lower[upper_ctr];
+        }
+    }
+}
+
+
 }  // namespace Foam
diff --git a/unitTests/test_HostMatrix.C b/unitTests/test_HostMatrix.C
index 975d11ab2..8bcf52433 100644
--- a/unitTests/test_HostMatrix.C
+++ b/unitTests/test_HostMatrix.C
@@ -44,26 +44,58 @@ TEST(HostMatrixConversion, non_symmetric_update)
     std::vector<scalar> d{1., 1., 1., 1., 1.};
     std::vector<scalar> u{1., 2., 1., 2., 1., 1.};
     std::vector<scalar> l{2., 2., 3., 2., 3., 2.};
-    std::vector<label> p{12, 0, 1, 
-        6, 13, 2, 3,
-        7, 14, 4, 
-        8, 9, 15, 5, 
-        10, 11, 16};
+    std::vector<label> p{12, 0, 1, 6,  13, 2,  3,  7, 14,
+                         4,  8, 9, 15, 5,  10, 11, 16};
 
     label total_nnz{17};
     label upper_nnz{6};
 
     std::vector<scalar> res{0., 0., 0., 0., 0., 0., 0., 0., 0.,
                             0., 0., 0., 0., 0., 0., 0., 0.};
-    std::vector<scalar> exp{
-        1., 1., 2.,
-        2., 1., 1., 2.,
-        2., 1., 1.,
-        3., 2., 1., 1.,
-        3., 2., 1.};
+    std::vector<scalar> exp{1., 1., 2., 2., 1., 1., 2., 2., 1.,
+                            1., 3., 2., 1., 1., 3., 2., 1.};
 
     Foam::non_symmetric_update(total_nnz, upper_nnz, p.data(), 1.0, d.data(),
-                           u.data(), l.data(), res.data());
+                               u.data(), l.data(), res.data());
 
     EXPECT_EQ(res, exp);
 }
+
+TEST(HostMatrixConversion, init_local_sparsisty)
+{
+    /* test a 5x5 symmetric matrix
+     * A =
+     *     0  1  2  3  4
+     * 0 | x  x  .  x  .  |
+     * 1 | x  x  x  .  x  |
+     * 2 | .  x  x  x  .  |
+     * 3 | x  .  x  x  x  |
+     * 4 | .  x  .  x  x  |
+     */
+
+    const label nrows = 5;
+    const label upper_nnz = 6;
+    const bool is_symmetric = true;
+
+    std::vector<label> upper{1, 3, 2, 4, 3, 4};
+    std::vector<label> lower{0, 0, 1, 1, 2, 3};
+
+    std::vector<label> rows{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    std::vector<label> cols{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    std::vector<label> permute{0, 0, 0, 0, 0, 0, 0, 0, 0,
+                               0, 0, 0, 0, 0, 0, 0, 0};
+
+    std::vector<label> rows_exp{0, 0, 0, 1, 1, 1, 1, 2, 2,
+                                2, 3, 3, 3, 3, 4, 4, 4};
+    std::vector<label> cols_exp{0, 1, 3, 0, 1, 2, 4, 1, 2,
+                                3, 0, 2, 3, 4, 1, 3, 4};
+    std::vector<label> permute_exp{6, 0, 1, 0, 7, 2, 3, 2, 8,
+                                   4, 1, 4, 9, 5, 3, 5, 10};
+    Foam::init_local_sparsity(nrows, upper_nnz, is_symmetric, upper.data(),
+                              lower.data(), rows.data(), cols.data(),
+                              permute.data());
+
+    EXPECT_EQ(rows, rows_exp);
+    EXPECT_EQ(cols, cols_exp);
+    EXPECT_EQ(permute, permute_exp);
+}

From 2cd456e10d27563354f42d08aab2f4e2ea9c95bf Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 15 Nov 2023 10:38:40 +0100
Subject: [PATCH 079/125] update required  versions

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3946e3e4a..3bbc61cf6 100644
--- a/README.md
+++ b/README.md
@@ -15,10 +15,10 @@ OGL is a wrapper for [ginkgo](https://github.com/ginkgo-project/ginkgo) solvers
 
 OGL has the following requirements
 
-*   _cmake 3.9+_
+*   _cmake 3.13+_
 *   _OpenFOAM 6+_ or _v2106_
 *   _Ginkgo 1.5.0+_ (recommended to install via OGL)
-*   C++14 compliant compiler (gcc or clang)
+*   C++17 compliant compiler (gcc or clang)
 
 See also [ginkgo's](https://github.com/ginkgo-project/ginkgo) documentation for additional requirements.
 

From b39f462c2246c55c3fb4c5442a32fada30a2f5aa Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 15 Nov 2023 10:51:13 +0100
Subject: [PATCH 080/125] add unit tests to pipeline

---
 .github/workflows/build-foam.yml |  10 +++---
 .github/workflows/unit-test.yml  |  53 +++++++++++++++++++++++++++++++
 common/.common.H.swo             | Bin 24576 -> 0 bytes
 3 files changed, 57 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/unit-test.yml
 delete mode 100644 common/.common.H.swo

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index 09c56769e..5c298370e 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -95,11 +95,6 @@ jobs:
       run: |
         cmake --build . --config $BUILD_TYPE
 
-    - name: Test OGL
-      working-directory: ${{github.workspace}}/build
-      shell: bash
-      run: ctest -C $BUILD_TYPE
-
     - name: Install OGL
       working-directory: ${{github.workspace}}
       run: |
@@ -117,9 +112,12 @@ jobs:
       with:
         name: ogl_build_${{ matrix.OF.path }}
         path: ${{github.workspace}}/build
+  unit_tests:
+    needs: [build, setup_build_matrix]
+    uses: ./.github/workflows/unit-tests.yml
 
   integration_tests:
-    needs: [build, setup_build_matrix]
+    needs: [build, unit_tests, setup_build_matrix]
     uses: ./.github/workflows/integration-tests.yml
     with:
       path: ${{ matrix.OF.path }}
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
new file mode 100644
index 000000000..3b229f58a
--- /dev/null
+++ b/.github/workflows/unit-test.yml
@@ -0,0 +1,53 @@
+name: Integration Tests
+
+defaults:
+ run:
+  shell: bash -o pipefail -i {0}
+
+on: 
+  workflow_call:
+    inputs:
+      path:
+        required: true
+        type: string
+      version:
+        required: true
+        type: string
+      cyclic_case:
+        required: true
+        type: string
+      foam_user_libbin:
+        required: true
+        type: string
+run-name: Integration test on ${{github.ref_name}}
+
+env:
+  BUILD_TYPE: Release
+  OMPI_ALLOW_RUN_AS_ROOT: 1
+  OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+  OMPI_MCA_btl_vader_single_copy_mechanism: none
+  OBR_RUN_CMD: "mpirun --bind-to core --map-by core -np {np} {solver} -parallel -case {path}/case > {path}/case/{solver}_{timestamp}.log 2>&1"
+  MPI_BUFFER_SIZE: 20000000
+  GINKGO_EXECUTOR: reference
+
+jobs:
+    - uses: actions/checkout@v4
+
+    - name: Get Ginkgo checkout version
+      shell: bash
+      run: |
+        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
+        export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
+        echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
+
+    - name: Cache build folder
+      uses: actions/cache@v3
+      with:
+        key: build-${{ matrix.OF.path }}-${{env.GINKGO_CHECKOUT_VERSION}}
+        path: |
+          ${{github.workspace}}/build
+
+    - name: Test OGL
+      working-directory: ${{github.workspace}}/build
+      shell: bash
+      run: ./unitTests/matrixConversion
diff --git a/common/.common.H.swo b/common/.common.H.swo
deleted file mode 100644
index 2ecc4277bb64ad17652e80614be1f89770ee053b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24576
zcmeI3dyFK<S-_jvgludlcI<@MASmC#dUkJj_I&<>WcESU>wV$d+sp1=oa62odZuf3
z`mU#Y=<eCQ<#O1LP#gybJ3-izk%=Y#ASOzb7zIKi2NMz#L}E)2B$Nj)1)K=tU>k*?
zL^i*w>YnMDo!#5Bg9K5J_BW5}s;{cPdiD49x0Qo$TU-)vDju|WU1?cs&%fI)y*YgC
zL*E^?)<YS1QeD{n<$Lz%db4$?=-Pgxc*c#3(rNW6@@q!mm}AHGST%6u*z){gzBC$v
zpL`S-`jpF)at-7fc$pe##kJ8lU2*KDLpSQBzJK5K;_BBOdznJ>;<*NL4dfcgHIQo{
z*Fdg;Tm!iVat-`{(m>q2)ViGpzanXLA^H4~p3h5@-{Z;WaL@HilHZ4t&q~krA5DI*
zCZG59Tu*-HAGro{4dfcgHIQo{*Fdg;Tm!iVat-7f$Tg5_AlJbEf(C5IvR)jptWUif
z;XMCO?f;Ly*0Meb?}58v4lMZOYb@*i@IIJ@?_FV8AA{e42Vo6f16RQ1@QuqY>uLB3
zJPBWf^H7IXSb?91LHJpC_A=T6x5Fe{16RX$Xy4QD6sWfTAv_BAfCE(+hG*%^C*V=|
z7*yf)@E5PLtSvYIU;8o3x)-M5`Bz%jx8UF5-{2{D7#@HI+yXN&3`1}kyc(Xxq46K^
zIQ%KR7w&;ygyZm5xC*|A<K%8wf}e!H!%^}cu;FLmc^og#!J}{|yaj$7zKjFr5%^&0
zh!O3VKkLaxR4({4((|ISI3Ny*Vc{Q<?s~lvw8E%7bng7P)z05_snD?F&^=r9oYu7C
zgoUAEtJ#!c@;!2YRThSZMzV!XEonK&wbv=MEn`V6X~An-O<5{Mfwv*ULd}()bHr}Q
z5n-ww5$4KF;KyO$MO`)N23G?=iiP7=V>j^aa7zsKl+aCE2%^~3S1<)OgU}I!nfgp}
zr%hroX@QmP+y6yPYBlt}xqd9enq8HPemoP%S}ht8?KHmkD5*1mdmWdyu$zNcQb9&h
zx;c9rnoP@00cIMFHm#pi6XIwbMu%NSd9Yn+)M;>1-jvn(_1lDgr=wNdv%{%OS%mM}
z;JwJ`>GaE)Ud>j;bE_pY%^I!Pu6R;JvBRRM+o5#I%{ZKJowH>-4DGEc5eA!Oenw?H
ztE%sNC{hi)Jr&uoy|fm-wq(1GWMJ%y^hC0oB(-;4$?PcoE}M-ZxzdVUe_gcuB__rC
znV?jfR-K&iR83Q6OEMYTP3z8>jPAJA4BZVomL;p|*-<1q10qVoXt>XQBhFc|Y&1Oz
ziV0J&vjI$TekuK)*YE0E=QS}Q_MvzYo%HvOcarz;?WDD1reZZ{`Eg-r#|+yq>>^G2
zPI>?SBhyQ>VwCA*hjH?AXZii4(NjGatMxGO86T$=+G?jQSM4aC=(^N!t6>mHW`q+>
z6+}mx$rZF>q4aiw7(Q`$dHxhCH0|5R8138Mxvmr=&8>;T+h$j8US6FY66d-yT+|OE
zZBE#&I1m|4O?G!Silyyrm8)J*JyR<A!DeBo-)I|YT2@xk%+xOZ>{;4=N7h_liq*xX
zqlah9Yl};>X|G3w8qpCkxAQ1=E8=ADB;I*27@Vcmc{N5l4{GtolTmyLdOvCPbb}%9
z^n!|Z>H#fImb|Q{i`^W>rP3|ZkK{yq(VKjY_-t>!nJ7$i?RS0ts$R>H>8F@*IbQ5I
z2&b%0S}h1S?a(QEoHW_=hTY8mTzCC5>%o}$DAt=?><HOcET-Y9t8vh5vc=7Wsxnqg
zabaI~#>M3Y8y(AVY;k!t2xBrYI%juD+f~BRP_H~!Elw@6JE2e6`CFIGZr7=@H>ppl
z*|W0BstG%VdQw!GOh-exzPKDzPRnZCEv>dOGk1&0oXmGNhVsnv($eyg@&XcSKIx(>
z@#OI65AHQQX4Zy@Cnk`1m3!f;oO3-Haj)Ns>p@r&^K>8x#j+=T_l%fW58hI5MsB0%
z$n=1)y?~Q!(~aw5xheg*<>{rO5Q{b82Ql{_lCz~A$Ia5%*yiSDaoukfgK&L}Gha>S
zXl$xzsU>?n;5exVEzc2kdqWC_Q@R_{5w@rX%`FksL|m6*{>U*gFBxIm6G!!(gE(v?
zna(haMI#8MaAa(|USx#NEv_AzU0oG(%PV499GzZSTbwy|czQ(~J+^X`&6M`8(#{K1
zpqe>-<hZzPdSzw$$lCF7-D$P2%MI!4(k`c|OQTsyq3y@I*-Nu4GYjOOzIpNR;@WW$
zgqD(&>Pol~x9<@~Mx_-~)HRaoK~puzj)fajkLQVs6s<_sTHc5tr?_o#ZDIM?n(20d
zd|M2k2%2gE3<=xc66#zJ1$ITGmY=Rv^>8kvtE)jRrjh*O&?UZ{93w(9hMf}Prc~*P
zhDXw-Q-W+t78Nw2G~3Q&BkaiBg+z6FSlKcP;2fmyHPxIbX--M?LR4*^@$V=_ser9w
zOpmSO*0xA6{a&jp3Cvg}smUR)+QO14TIsA9`++W*la{jo3)sh(V>>JRKYf1xC2ajC
z;URFK3WwmovH8CXpN0G2UhrTM7T_RU2mg%i|0#GMY{9KC3NK*e{}ns{zYgnAfnS0f
z;YDoyZ-VjxJPhaI9Z-RJ7=fRF>);aj4DEgxZU^N9n1flk0aTy9M&Is&tN8w9KA(U;
zgU8_`@VoFJJOKB@eQ+=QCP*m5BwPho!sGZ4?t*tg1*YJQ@EpE`Pr^IlG_1iCTm#?7
zU+~B98?Xf~NO0g6;0AaDydGY_XYePW{0Zma46K6#r{E+MVH94(pYQ@a0iT0+!ktip
zH^X!I7rqV8z$35$Ef|9${5^h#55XTm1d~vJ8$f`1?dQJ1tuYQljExB&TRv)HhO*A=
zhMx8;{eCZFNSK3SMXsw20bAaw42@gb8W8I;E^Di`+zjPLS)Hm;)|ijBqO!S9D63jW
z%mV3Gm0)L3mW-BTL~3bNwGp@uTlU4(#0IT=5Zl{M8?-=qw#oTk-jLO7gKHgUhY`Sa
zF?@C;5vM&AvD=U|%j(mOOqoi*GTEBZ4TjFHm8-j`;ptRR(iDc#7-9O3aCdtY!^Je|
z9D7&X3faQima0IE?!zu}QX_S{ob*b0d`CHL^KeQS%XPby)S_+6Fq0tT2HRdHb1_qS
zQc(MX`jjy~qE1_r?ArS6lRdGD+s#5%1Lv&)&4Bi`m8)@h-~h=30}@Z3^y5s5>DGzN
zg;U%O>vAXBrB96uN^jQHK~eC6O&KbGK<2uboK(Xl1_w=1Re_l3as&!2E`+eCZO1XW
zgsRn#xm)Vfvi=QI?!Xh&bV_w6k)_h`-FaDYN&^F?r5E3OUG7A?bXV_<PE_wZ*E79u
zC+^yNol7^D!HBE@b=Ore<4oTs4~{EkP8ffyvKjex;DFYj$<(Q3QXE7RCbd0W3}7cU
zGqOp#wlv%QATj1HV06lCe)wAZx@xIKmort<|J};zOvc5VmeciZFQb#`VpjV0Tv}cy
zmm2WxnT)46m1L$5xsr7|I;9Q(^`C%BPQMMY&mNg38J`u7>W-H?5laPnPioj_WZ7(R
z6I@E%U<K`yT9GvwO1~;oOQ%$-hLW4h0`BN`OOj=t@qr;bvw^5)r|n7dnx(I@C7XlE
zaZcyc`**8Jk7c=z`#L?`j0#vCi+(Lo)_OV*&6-u7she)>O53M$DRcgUg)W$(v;9!b
z1ld`__$t(0dnCF0YzC3GK-KEyQj&_>HV|gw;UKA=QTxRJyL!~SZIPyTssR?vwu_`O
zIoK8S;Z0T)r>9?LAIoGQI~jDX*$llKVdh$98!)Tu<!^%hI+^rimoDzIFB&bP=Cp5R
zt<hqITP9zxFr{~2N+%L19h)hXSQyIgKzB;87X`wuDa)uCC<leMV`_N!V*M?3jikA~
zWUH_0hX^M1bgQ9DRv1fVcm%$;rR!AYsupbqUP(74aQ1Q!X!ej~LyMGY+(;T3aeqW%
zGV6_Bza`th=^t$^>qeSB_Bn6*&#26-D<<WO#z(2AO5cgruT0GiqgWT|Iz>(6Qo%Y?
zI*ZJ4sXS-bX9ePKj<?u}(!*389e5VS=@z$;c($;9m9blGg={%-s{A-i#)G3*OG_8{
zMIAnw17Noqq&LK@JIwfbo3<Mo&-{p9f>Tr3HH~h}!t5-aT~D#(z)%ZBO!96XP;~F|
z=EPD9MNIPEFH0iE%&eDi60(eTJUz9d(^Ig$cdI@8g2zJgl%(s#Y4?CSx1Bj0$myBW
zIZ|G7Rc(D<s47)M=D2ZvH#Ub2MjLLEc&psBWA+NR7ISy6?f<W0OMV^OQ`!HV#rpLk
zw*E8ld3YE;05uqe|HQt32EGL!fVaXW@Nw+>b+{Ikjjx{bkHHWW;0Cx1zKY%cH}DAD
z0qR+Q4J0TVe-1`L+4|qdZvP(qC43(4hu?<x!V(O?=djD~hPT5t@C5ewL-5Nm21R%V
zJNz+t2du&y;9uyo?swAX{3F*uu7O+wxdwj7G@y<}^U7nVGO-j{2Vx#OwLK&)kDbb6
zr-;x>TuX_U$~eUnkI$@*#O!w^<2UMeIq?!3+aQmfO1J0qGmo80HPPoYkDbyHV|nb<
z#l}vJTiwrZaTRF$|0?X@E$m@s|EK?-|4-QYe+wUmd*BZ6p$;k*paMg1ExZw43E#m7
z@Em*_z6sxer{UA^3Ai6lgA3QfKjJ6&2T*YW55TX0iW9gR)c*_oIeZvYtUv>f!IkiJ
z`~rUspMg)qWAJ;Rd<g;ga2vc1{uTegm*BJTI6MY_1RsaH;9c-`xD9?9ehR*V4?)EW
z{1&_)ZiNMyhgZN;_!FLlcY}%r_&K-~E`cwg$NS+<xC4}4&q4$0Fa{5zZ}sed8Lor>
z!vFFG_<hJ<xdw6#<QnMFz@EZz_LRxoQ0yV=9<pq|Iq6%dZ({XAF+>+t>Y_61X7?}A
zKef};3tiu?5tCBY1$j#a>`%YsfmKdrBoD0Off4_n(~cU-1FL$Y&yxRk?5HRYtm5gH
zdA@EQNaul7X)GO&NmamA9$3}=?0KidsgwT07g&`%<~E8T4n>8c=x~&FTtoj*OtK{b
zD}+{<EUB7$!aqF1q02f<CJDu)fQq~IMu-qlv292)0WjvhSM0!m5^MrU_&zw8sd#kC
o#Gp(}DuGIp&YT2lsGyJQlbEO^Ziuk3!jR6F#I>mJy9KTM51QLKTmS$7


From 6f63f46d72a9dd5a821e7dfa3a8d3e080376b29e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 15 Nov 2023 10:52:29 +0100
Subject: [PATCH 081/125] add unit tests to pipeline

---
 .github/workflows/unit-test.yml | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 3b229f58a..6ea8f917f 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -4,31 +4,10 @@ defaults:
  run:
   shell: bash -o pipefail -i {0}
 
-on: 
-  workflow_call:
-    inputs:
-      path:
-        required: true
-        type: string
-      version:
-        required: true
-        type: string
-      cyclic_case:
-        required: true
-        type: string
-      foam_user_libbin:
-        required: true
-        type: string
-run-name: Integration test on ${{github.ref_name}}
+run-name: Unit tests on ${{github.ref_name}}
 
 env:
   BUILD_TYPE: Release
-  OMPI_ALLOW_RUN_AS_ROOT: 1
-  OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
-  OMPI_MCA_btl_vader_single_copy_mechanism: none
-  OBR_RUN_CMD: "mpirun --bind-to core --map-by core -np {np} {solver} -parallel -case {path}/case > {path}/case/{solver}_{timestamp}.log 2>&1"
-  MPI_BUFFER_SIZE: 20000000
-  GINKGO_EXECUTOR: reference
 
 jobs:
     - uses: actions/checkout@v4

From b7aae494489e1811f4aeebd886a649a02e53b1a1 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 15 Nov 2023 10:58:18 +0100
Subject: [PATCH 082/125] format

---
 HostMatrix/HostMatrixFreeFunctions.C | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/HostMatrix/HostMatrixFreeFunctions.C b/HostMatrix/HostMatrixFreeFunctions.C
index a8d1c908f..10298ae5d 100644
--- a/HostMatrix/HostMatrixFreeFunctions.C
+++ b/HostMatrix/HostMatrixFreeFunctions.C
@@ -97,16 +97,11 @@ void non_symmetric_update(const label total_nnz, const label upper_nnz,
     }
 }
 
-void init_local_sparsity(
-        const label nrows,
-        const label upper_nnz,
-        const bool is_symmetric,
-        const label* upper,
-        const label* lower,
-        label *rows,
-        label *cols,
-        label *permute
-        ){
+void init_local_sparsity(const label nrows, const label upper_nnz,
+                         const bool is_symmetric, const label *upper,
+                         const label *lower, label *rows, label *cols,
+                         label *permute)
+{
     label after_neighbours = (is_symmetric) ? upper_nnz : 2 * upper_nnz;
     label element_ctr = 0;
     label upper_ctr = 0;

From 0be7b0cd3650d1234dad47c1f61d8a776ee867c1 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 15 Nov 2023 10:59:32 +0100
Subject: [PATCH 083/125] format

---
 DevicePersistent/Partition/Partition.H | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index d83de9894..205d362aa 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -31,9 +31,9 @@ SourceFiles
 
 #include <ginkgo/ginkgo.hpp>
 
-#include "HostMatrix/HostMatrix.H"
 #include "DevicePersistent/Base/Base.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
+#include "HostMatrix/HostMatrix.H"
 #include "common/common.H"
 
 namespace Foam {
@@ -58,11 +58,9 @@ struct PartitionInitFunctor {
 
     const std::vector<std::pair<gko::array<label>, label>> send_idxs_;
 
-    PartitionInitFunctor(
-        const ExecutorHandler &exec, const label local_size,
-        const label verbose, const int ranks_per_gpu,
-        const CommunicationPattern& communication_pattern
-        )
+    PartitionInitFunctor(const ExecutorHandler &exec, const label local_size,
+                         const label verbose, const int ranks_per_gpu,
+                         const CommunicationPattern &communication_pattern)
         : exec_(exec),
           local_size_(local_size),
           verbose_(verbose),
@@ -125,11 +123,10 @@ public:
      * @param offset
      * @param elements
      */
-    PersistentPartition(
-        const objectRegistry &db, const ExecutorHandler &exec,
-        const label verbose, const int ranks_per_gpu, const label elements,
-        const CommunicationPattern &communication_pattern
-        )
+    PersistentPartition(const objectRegistry &db, const ExecutorHandler &exec,
+                        const label verbose, const int ranks_per_gpu,
+                        const label elements,
+                        const CommunicationPattern &communication_pattern)
         : PersistentBase<part_type, PartitionInitFunctor>(
               "device_partition", db,
               PartitionInitFunctor(exec, elements, verbose, ranks_per_gpu,

From ff0b489883d9705d329abc448d40c1a490a8c842 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 08:41:43 +0100
Subject: [PATCH 084/125] format

---
 Solver/GMRES/GKOGMRES.H | 5 ++---
 lduLduBase/lduLduBase.H | 3 +--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 4d3b253af..46382d9f6 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -69,9 +69,8 @@ public:
     std::unique_ptr<gmres::Factory, std::default_delete<gmres::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        auto gmres = gmres::build()
-                         .with_criteria(stoppingCriterionVec_)
-                         .on(exec);
+        auto gmres =
+            gmres::build().with_criteria(stoppingCriterionVec_).on(exec);
         return gmres;
     };
 
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 91bb35d4a..7a66b94d8 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -223,8 +223,7 @@ public:
             verbose_,
             solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1),
             psi.size(),  // number of elements
-            this->get_communication_pattern()
-        };
+            this->get_communication_pattern()};
 
         // TODO this is now the local matrix part for dist_A
         MatrixWrapper dist_A{db_,

From b819278dc0173db5a4919613933ea6d3caffd07d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 08:51:02 +0100
Subject: [PATCH 085/125] fix typo

---
 HostMatrix/HostMatrix.H | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 6aa762589..4d31446de 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -108,7 +108,7 @@ const lduInterfaceField *interface_getter(
  ** @param nrows number of rows
  ** @param upper_nnz number of non zeros in the upper triangular matrix
  ** @param is_symmetric whether matrix is symmetric, in the symmetric case
- ** the lower elemnts indices in the permute array are computed differently
+ ** the lower elements indices in the permute array are computed differently
  ** @param upper pointer to array of
  ** @param lower pointer to rows array
  ** @param rows pointer to rows array

From 7384793991a7fbd98acf316b244f9a5e5e53e13b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 08:52:59 +0100
Subject: [PATCH 086/125] format

---
 DevicePersistent/Partition/Partition.H | 1 -
 Solver/GMRES/GKOGMRES.H                | 5 ++---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/DevicePersistent/Partition/Partition.H b/DevicePersistent/Partition/Partition.H
index 2b2c377a2..205d362aa 100644
--- a/DevicePersistent/Partition/Partition.H
+++ b/DevicePersistent/Partition/Partition.H
@@ -31,7 +31,6 @@ SourceFiles
 
 #include <ginkgo/ginkgo.hpp>
 
-#include "HostMatrix/HostMatrix.H"
 #include "DevicePersistent/Base/Base.H"
 #include "DevicePersistent/ExecutorHandler/ExecutorHandler.H"
 #include "HostMatrix/HostMatrix.H"
diff --git a/Solver/GMRES/GKOGMRES.H b/Solver/GMRES/GKOGMRES.H
index 4d3b253af..46382d9f6 100644
--- a/Solver/GMRES/GKOGMRES.H
+++ b/Solver/GMRES/GKOGMRES.H
@@ -69,9 +69,8 @@ public:
     std::unique_ptr<gmres::Factory, std::default_delete<gmres::Factory>>
     create_default(std::shared_ptr<gko::Executor> exec) const
     {
-        auto gmres = gmres::build()
-                         .with_criteria(stoppingCriterionVec_)
-                         .on(exec);
+        auto gmres =
+            gmres::build().with_criteria(stoppingCriterionVec_).on(exec);
         return gmres;
     };
 

From c3f26f3e299f7e17d2e0f7f838018f9d8ee47764 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 08:56:42 +0100
Subject: [PATCH 087/125] remove note

---
 DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H | 1 -
 1 file changed, 1 deletion(-)

diff --git a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
index e3fd5feb3..e978beede 100644
--- a/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
+++ b/DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H
@@ -219,7 +219,6 @@ struct MatrixInitFunctor {
             gko::experimental::distributed::sparse_communicator::create(
                 *comm.get(), partition_.get_localized_partition());
 
-        // TODO FIXME
         auto recv_sizes = sparse_comm->get_recv_sizes();
         auto recv_offs = sparse_comm->get_recv_offsets();
         auto send_sizes = sparse_comm->get_send_sizes();

From 53f5a1ea1f9ac4fdc6a7142246611959c945ebf5 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 09:00:54 +0100
Subject: [PATCH 088/125] fixup workflow file name

---
 .github/workflows/build-foam.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index 5c298370e..a98783a49 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -114,7 +114,7 @@ jobs:
         path: ${{github.workspace}}/build
   unit_tests:
     needs: [build, setup_build_matrix]
-    uses: ./.github/workflows/unit-tests.yml
+    uses: ./.github/workflows/unit-test.yml
 
   integration_tests:
     needs: [build, unit_tests, setup_build_matrix]

From 6360493b3b8c3c5c4e12ef543d629435938d5bab Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 09:07:13 +0100
Subject: [PATCH 089/125] pass arguments to workflow, add job

---
 .github/workflows/build-foam.yml | 2 ++
 .github/workflows/unit-test.yml  | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index a98783a49..91f857659 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -115,6 +115,8 @@ jobs:
   unit_tests:
     needs: [build, setup_build_matrix]
     uses: ./.github/workflows/unit-test.yml
+    with:
+      path: ${{ matrix.OF.path }}
 
   integration_tests:
     needs: [build, unit_tests, setup_build_matrix]
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 6ea8f917f..353c5b924 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -6,10 +6,18 @@ defaults:
 
 run-name: Unit tests on ${{github.ref_name}}
 
+on: 
+  workflow_call:
+    inputs:
+      path:
+        required: true
+        type: string
+
 env:
   BUILD_TYPE: Release
 
 jobs:
+  unit_test:
     - uses: actions/checkout@v4
 
     - name: Get Ginkgo checkout version

From 75dcc0c917bd2dfb40fe693b46bb9c77b07fad66 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 09:11:04 +0100
Subject: [PATCH 090/125] fixup workflow

---
 .github/workflows/unit-test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 353c5b924..67eaa6a45 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -18,6 +18,9 @@ env:
 
 jobs:
   unit_test:
+    runs-on: ubuntu-latest
+    container: greole/ofbase
+    steps:
     - uses: actions/checkout@v4
 
     - name: Get Ginkgo checkout version

From cb35594d49aab9004081f31e9311c88f1a23a4cd Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 09:20:47 +0100
Subject: [PATCH 091/125] fixup path

---
 .github/workflows/unit-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 67eaa6a45..d263c031b 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -33,7 +33,7 @@ jobs:
     - name: Cache build folder
       uses: actions/cache@v3
       with:
-        key: build-${{ matrix.OF.path }}-${{env.GINKGO_CHECKOUT_VERSION}}
+        key: build-${{ inputs.path }}-${{env.GINKGO_CHECKOUT_VERSION}}
         path: |
           ${{github.workspace}}/build
 

From dd87745a360fce471422a3f6c91c1950259545e9 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 09:34:07 +0100
Subject: [PATCH 092/125] fix matrix call, clean up

---
 .github/workflows/build-foam.yml        | 4 ++++
 .github/workflows/integration-tests.yml | 8 --------
 .github/workflows/unit-test.yml         | 7 ++-----
 3 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index 91f857659..bde860d07 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -112,11 +112,15 @@ jobs:
       with:
         name: ogl_build_${{ matrix.OF.path }}
         path: ${{github.workspace}}/build
+
   unit_tests:
     needs: [build, setup_build_matrix]
     uses: ./.github/workflows/unit-test.yml
     with:
       path: ${{ matrix.OF.path }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup_build_matrix.outputs.matrix) }}
 
   integration_tests:
     needs: [build, unit_tests, setup_build_matrix]
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index d5ad28029..5aa69c777 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -53,14 +53,6 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Get Ginkgo checkout version
-      shell: bash
-      run: |
-        grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o  "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION
-        export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION)
-        echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV
-        echo "Ginkgo_DIR=${{inputs.foam_user_libbin}}/cmake/Ginkgo" >> $GITHUB_ENV
-
     - name: Cache workspace
       uses: actions/cache@v3
       with:
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index d263c031b..6c8126c6a 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -1,11 +1,9 @@
-name: Integration Tests
+name: Unit Tests
 
 defaults:
  run:
   shell: bash -o pipefail -i {0}
 
-run-name: Unit tests on ${{github.ref_name}}
-
 on: 
   workflow_call:
     inputs:
@@ -13,8 +11,7 @@ on:
         required: true
         type: string
 
-env:
-  BUILD_TYPE: Release
+run-name: Unit tests on ${{github.ref_name}}
 
 jobs:
   unit_test:

From 4ec167f7b4815081c05b3117f91cbbe0d1d00751 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 14:40:04 +0100
Subject: [PATCH 093/125] add compression to non local column index

---
 HostMatrix/HostMatrix.C | 31 +++++++++++++++++++++++--------
 HostMatrix/HostMatrix.H |  8 +++++---
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 153aee5d9..a05fb2b05 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -359,12 +359,13 @@ HostMatrixWrapper<MatrixType>::collect_local_interface_indices(
 }
 
 template <class MatrixType>
-std::vector<std::tuple<label, label>>
+std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
-    // vector of local cell ids on other side
-    std::vector<std::tuple<label, label>> non_local_idxs{};
+    // vector of local cell idx connected to interface
+    std::vector<std::tuple<label, label, label>> non_local_idxs{};
+    std::map<label, label> unique_index {};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
     interface_iterator<processorFvPatch>(
@@ -372,8 +373,20 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
         [&](label &element_ctr, const label interface_size,
             const processorFvPatch &, const lduInterfaceField *iface) {
             const auto &face_cells{iface->interface().faceCells()};
+            label uniqueIdCtr {0};
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                non_local_idxs.push_back({element_ctr, face_cells[cellI]});
+                const label cellId = face_cells[cellI];
+                label uniqueId = uniqueIdCtr;
+                auto search = unique_index.find(cellId);
+                // cellId is new and unique
+                if (search == unique_index.end()) {
+                    uniqueId = uniqueIdCtr;
+                    unique_index[cellId] = uniqueIdCtr;
+                    uniqueIdCtr++;
+                } else {
+                    uniqueId = unique_index[cellId]; 
+                }
+                non_local_idxs.push_back({element_ctr, cellId, uniqueId});
                 element_ctr += 1;
             }
         });
@@ -393,10 +406,12 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     auto cols = non_local_sparsity_.col_idxs_.get_data();
     auto permute = non_local_sparsity_.ldu_mapping_.get_data();
 
+    // TODO check if sorting is actually needed since interfaces should
+    // be sorted already
     std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
               [&](const auto &a, const auto &b) {
-                  auto [interface_idx_a, row_a] = a;
-                  auto [interface_idx_b, row_b] = b;
+                  auto [interface_idx_a, row_a, unique_col_a] = a;
+                  auto [interface_idx_b, row_b, unique_col_b] = b;
                   return std::tie(row_a, interface_idx_a) <
                          std::tie(row_b, interface_idx_b);
               });
@@ -405,9 +420,9 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
         interfaces, [&](label &element_ctr, const label interface_size,
                         const processorFvPatch &, const lduInterfaceField *) {
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto [interface_idx, row] = non_local_row_indices[element_ctr];
+                auto [interface_idx, row, unique_col] = non_local_row_indices[element_ctr];
                 rows[element_ctr] = row;
-                cols[element_ctr] = interface_idx;
+                cols[element_ctr] = unique_col;
                 permute[element_ctr] = interface_idx;
                 element_ctr += 1;
             }
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 4d31446de..5140f299d 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -341,11 +341,13 @@ private:
     /** Iterates all interfaces and collect the corresponding cell id (row)
     ** and a unique counter
     **
-    ** @return vector off size nnz_non_local, with sections for each interface
-    ** ret = [(1,2),(2,20), ...]
+    ** @return vector of size nnz_non_local, with a running index, row 
+    ** index, and a unique compressed column index
+    ** sections for each interface
+    ** ret = [(1,2,1),(2,20, 2), (3, 20, 2) ...]
     **         i0   i1,   i...
     */
-    std::vector<std::tuple<label, label>> collect_cells_on_interface(
+    std::vector<std::tuple<label, label, label>> collect_cells_on_interface(
         const lduInterfaceFieldPtrsList &interfaces) const;
 
     /** Based on OpenFOAMs ldu matrix format this function computes two

From f186732ee02fb25fdd9b8cc7bc6de47485b3e614 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 20:10:55 +0100
Subject: [PATCH 094/125] communicate neighbour cell indices to calculate
 compressed col idxs

---
 HostMatrix/HostMatrix.C | 107 +++++++++++++++++++++++++++++++++-------
 HostMatrix/HostMatrix.H |   3 ++
 2 files changed, 93 insertions(+), 17 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index a05fb2b05..796cec317 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -57,6 +57,7 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
       non_diag_nnz_(2 * upper_nnz_),
       local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
       local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
+      global_row_index_{nrows_},
       local_sparsity_{
           fieldName + "_local",           db,       exec_,
           local_matrix_w_interfaces_nnz_, verbose_,
@@ -132,6 +133,7 @@ HostMatrixWrapper<MatrixType>::HostMatrixWrapper(
       non_diag_nnz_(2 * upper_nnz_),
       local_matrix_nnz_(nrows_ + 2 * upper_nnz_),
       local_matrix_w_interfaces_nnz_(local_matrix_nnz_ + local_interface_nnz_),
+      global_row_index_{nrows_},
       local_sparsity_{
           fieldName + "_cols", db, exec_, local_matrix_nnz_, verbose_,
       },
@@ -363,35 +365,106 @@ std::vector<std::tuple<label, label, label>>
 HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
-    // vector of local cell idx connected to interface
+    // vector of neighbour cell idx connected to interface
     std::vector<std::tuple<label, label, label>> non_local_idxs{};
+
     std::map<label, label> unique_index {};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
-    interface_iterator<processorFvPatch>(
-        interfaces,
-        [&](label &element_ctr, const label interface_size,
-            const processorFvPatch &, const lduInterfaceField *iface) {
-            const auto &face_cells{iface->interface().faceCells()};
-            label uniqueIdCtr {0};
+    label startOfRequests = Pstream::nRequests();
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+
+        const auto iface{interface_getter(interfaces, i)};
+        const auto &face_cells{iface->interface().faceCells()};
+
+        if (isA<processorLduInterface>(iface->interface())) {
+            const processorLduInterface &pldui =
+                refCast<const processorLduInterface>(iface->interface());
+            const label neighbProcNo = pldui.neighbProcNo();
+
+            word msg = "send face cells interface " + std::to_string(i) +
+                       " from proc " + std::to_string(Pstream::myProcNo()) +
+                       " to neighbour proc " + std::to_string(neighbProcNo);
+
+            LOG_2(verbose_, msg)
+            pldui.send(Pstream::commsTypes::nonBlocking, face_cells);
+        }
+    }
+    Pstream::waitRequests(startOfRequests);
+    LOG_2(verbose_, "send face cells done")
+
+
+    label interface_ctr = 0;
+    label uniqueIdCtr = 0;
+    for (int i = 0; i < interfaces.size(); i++) {
+        if (interface_getter(interfaces, i) == nullptr) {
+            continue;
+        }
+
+        const auto iface{interface_getter(interfaces, i)};
+        const auto &face_cells{iface->interface().faceCells()};
+        const label interface_size = face_cells.size();
+
+        if (isA<processorLduInterface>(iface->interface())) {
+            const processorLduInterface &pldui =
+                refCast<const processorLduInterface>(iface->interface());
+            const label neighbProcNo = pldui.neighbProcNo();
+
+            word msg_2 = "receive face cells interface " + std::to_string(i) +
+                         " from proc " + std::to_string(neighbProcNo);
+            LOG_2(verbose_, msg_2)
+
+            auto otherSide_tmp = pldui.receive<label>(
+                Pstream::commsTypes::nonBlocking, interface_size);
+            LOG_2(verbose_, "receive face cells done")
+
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                const label cellId = face_cells[cellI];
+                auto global_row = global_row_index_.toGlobal(
+                    neighbProcNo, otherSide_tmp()[cellI]);
+                auto search = unique_index.find(global_row);
+                // global_row is new and unique
                 label uniqueId = uniqueIdCtr;
-                auto search = unique_index.find(cellId);
-                // cellId is new and unique
                 if (search == unique_index.end()) {
                     uniqueId = uniqueIdCtr;
-                    unique_index[cellId] = uniqueIdCtr;
+                    unique_index[global_row] = uniqueIdCtr;
                     uniqueIdCtr++;
                 } else {
-                    uniqueId = unique_index[cellId]; 
+                    uniqueId = unique_index[global_row]; 
                 }
-                non_local_idxs.push_back({element_ctr, cellId, uniqueId});
-                element_ctr += 1;
+                non_local_idxs.push_back(
+                    {interface_ctr, face_cells[cellI], uniqueId});
+                interface_ctr += 1;
             }
-        });
-
-    word msg = "done collecting neighbouring processor cell ids";
+        }
+    }
+    word msg = "done collecting neighbouring processor cell id";
+
+    // interface_iterator<processorFvPatch>(
+    //     interfaces,
+    //     [&](label &element_ctr, const label interface_size,
+    //         const processorFvPatch &, const lduInterfaceField *iface) {
+    //         const auto &face_cells{iface->interface().faceCells()};
+    //         for (label cellI = 0; cellI < interface_size; cellI++) {
+    //             const label cellId = face_cells[cellI];
+    //             label uniqueId = uniqueIdCtr;
+    //             auto search = unique_index.find(cellId);
+    //             // cellId is new and unique
+    //             if (search == unique_index.end()) {
+    //                 uniqueId = uniqueIdCtr;
+    //                 unique_index[cellId] = uniqueIdCtr;
+    //                 uniqueIdCtr++;
+    //             } else {
+    //                 uniqueId = unique_index[cellId]; 
+    //             }
+    //             non_local_idxs.push_back({element_ctr, cellId, uniqueId});
+    //             element_ctr += 1;
+    //         }
+    //     });
+    //
+    // word msg = "done collecting neighbouring processor cell ids";
 
     LOG_2(verbose_, msg)
     return non_local_idxs;
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 5140f299d..69f519dcb 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -277,6 +277,9 @@ private:
     // nnzs of local matrix including local interfaces
     const label local_matrix_w_interfaces_nnz_;
 
+    // provides mapping between local and global indexing
+    const gkoGlobalIndex global_row_index_;
+
     mutable PersistentSparsityPattern local_sparsity_;
 
     mutable PersistentArray<scalar> local_coeffs_;

From 249a7d7ebd40b4190834d6d90c9b4c7e1ca156d7 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 20:15:16 +0100
Subject: [PATCH 095/125] add statistics message

---
 lduLduBase/lduLduBase.H | 42 ++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 7a66b94d8..3a1b18f14 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -320,27 +320,27 @@ public:
         solverPerf.initialResidual() = this->get_init_res_norm();
         solverPerf.finalResidual() = this->get_res_norm();
         solverPerf.nIterations() = this->get_number_of_iterations();
-        // this->store_number_of_iterations();
-        // auto time_for_res_norm_eval = this->get_res_norm_time();
-        // auto time_per_iter =
-        //     delta_t_solve / max(this->get_number_of_iterations(), 1);
-        // scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
-        // this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
-        //     this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1,
-        //     0);
-        // this->set_prev_rel_res_cost(prev_rel_res_cost);
-        // auto time_per_iter_and_dof =
-        //     time_per_iter * 1000.0 / partition.get_total_size();
-        // word msg =
-        //     "\nStatistics:\n\tTime per iteration: " +
-        //     std::to_string(time_per_iter) +
-        //     std::string(" [mu s]\n\tTime per residual norm calculation: ") +
-        //     std::to_string(time_for_res_norm_eval) +
-        //     std::string(" [mu s]\n\tTime per iteration and DOF: ") +
-        //     std::to_string(time_per_iter_and_dof) + std::string(" [ns]") +
-        //     std::string("\n\tRetrieve results bandwidth ") +
-        //     std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
-        // MLOG_0(verbose_, msg)
+        this->store_number_of_iterations();
+        auto time_for_res_norm_eval = this->get_res_norm_time();
+        auto time_per_iter =
+            delta_t_solve / max(this->get_number_of_iterations(), 1);
+        scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
+        this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
+            this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1,
+            0);
+        this->set_prev_rel_res_cost(prev_rel_res_cost);
+        auto time_per_iter_and_dof =
+            time_per_iter * 1000.0 / partition.get_total_size();
+        word msg =
+            "\nStatistics:\n\tTime per iteration: " +
+            std::to_string(time_per_iter) +
+            std::string(" [mu s]\n\tTime per residual norm calculation: ") +
+            std::to_string(time_for_res_norm_eval) +
+            std::string(" [mu s]\n\tTime per iteration and DOF: ") +
+            std::to_string(time_per_iter_and_dof) + std::string(" [ns]") +
+            std::string("\n\tRetrieve results bandwidth ") +
+            std::to_string(bandwidth_copy_back) + std::string(" [GByte/s]");
+        MLOG_0(verbose_, msg)
 
         return solverPerf;
     }

From da367d02946a148379307e70116185ac075d260d Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Fri, 17 Nov 2023 20:16:15 +0100
Subject: [PATCH 096/125] compile unittests

---
 .github/workflows/build-foam.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index bde860d07..aca29f26e 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -88,7 +88,7 @@ jobs:
         mkdir -p build
         cd build
         [ -d "/github/home/$GINKGO_CHECKOUT_VERSION" ] && cp -rp /github/home/$GINKGO_CHECKOUT_VERSION third_party
-        cmake -G Ninja -DOGL_ALLOW_REFERENCE_ONLY=On -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+        cmake -G Ninja -DOGL_ALLOW_REFERENCE_ONLY=On -DOGL_BUILD_UNITTEST=On -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
 
     - name: Build OGL
       working-directory: ${{github.workspace}}/build

From dfccc49ad2710502289bc0d37acaaad3cf0c0333 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 10:42:34 +0100
Subject: [PATCH 097/125] fix formating

---
 HostMatrix/HostMatrix.C | 9 +++++----
 HostMatrix/HostMatrix.H | 2 +-
 lduLduBase/lduLduBase.H | 3 +--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 796cec317..9e596a5ff 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -368,7 +368,7 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     // vector of neighbour cell idx connected to interface
     std::vector<std::tuple<label, label, label>> non_local_idxs{};
 
-    std::map<label, label> unique_index {};
+    std::map<label, label> unique_index{};
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
     label startOfRequests = Pstream::nRequests();
@@ -432,7 +432,7 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
                     unique_index[global_row] = uniqueIdCtr;
                     uniqueIdCtr++;
                 } else {
-                    uniqueId = unique_index[global_row]; 
+                    uniqueId = unique_index[global_row];
                 }
                 non_local_idxs.push_back(
                     {interface_ctr, face_cells[cellI], uniqueId});
@@ -457,7 +457,7 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     //                 unique_index[cellId] = uniqueIdCtr;
     //                 uniqueIdCtr++;
     //             } else {
-    //                 uniqueId = unique_index[cellId]; 
+    //                 uniqueId = unique_index[cellId];
     //             }
     //             non_local_idxs.push_back({element_ctr, cellId, uniqueId});
     //             element_ctr += 1;
@@ -493,7 +493,8 @@ void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
         interfaces, [&](label &element_ctr, const label interface_size,
                         const processorFvPatch &, const lduInterfaceField *) {
             for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto [interface_idx, row, unique_col] = non_local_row_indices[element_ctr];
+                auto [interface_idx, row, unique_col] =
+                    non_local_row_indices[element_ctr];
                 rows[element_ctr] = row;
                 cols[element_ctr] = unique_col;
                 permute[element_ctr] = interface_idx;
diff --git a/HostMatrix/HostMatrix.H b/HostMatrix/HostMatrix.H
index 69f519dcb..0eb9bf1a9 100644
--- a/HostMatrix/HostMatrix.H
+++ b/HostMatrix/HostMatrix.H
@@ -344,7 +344,7 @@ private:
     /** Iterates all interfaces and collect the corresponding cell id (row)
     ** and a unique counter
     **
-    ** @return vector of size nnz_non_local, with a running index, row 
+    ** @return vector of size nnz_non_local, with a running index, row
     ** index, and a unique compressed column index
     ** sections for each interface
     ** ret = [(1,2,1),(2,20, 2), (3, 20, 2) ...]
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 3a1b18f14..f23e3e5cf 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -326,8 +326,7 @@ public:
             delta_t_solve / max(this->get_number_of_iterations(), 1);
         scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
         this->get_exec_handler().get_gko_mpi_host_comm()->broadcast(
-            this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1,
-            0);
+            this->get_exec_handler().get_ref_exec(), &prev_rel_res_cost, 1, 0);
         this->set_prev_rel_res_cost(prev_rel_res_cost);
         auto time_per_iter_and_dof =
             time_per_iter * 1000.0 / partition.get_total_size();

From dfdbb9d9019ace3604f7d54fafb49855abc9e435 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 10:59:40 +0100
Subject: [PATCH 098/125] add libPstream

---
 unitTests/CMakeLists.txt | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 9d611e5c3..39ae64889 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -20,6 +20,12 @@ add_executable(
 # Your-external "mylib", add GLOBAL if the imported library is located in directories above the current.
 add_library( libOpenFOAM SHARED IMPORTED )
 
+# You can define two import-locations: one for debug and one for release.
+set_target_properties(libPstream
+    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/dummy/libPstream.so
+    )
+
+
 # You can define two import-locations: one for debug and one for release.
 set_target_properties(libOpenFOAM
     PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libOpenFOAM.so
@@ -37,6 +43,7 @@ target_link_libraries(
       OGL
       libOpenFOAM
       libfiniteVolume
+      libPstream
 )
 
 

From 575e94ecca47e72149a1f440d18ab075a0577091 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 11:02:23 +0100
Subject: [PATCH 099/125] add more link libs

---
 unitTests/CMakeLists.txt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 39ae64889..9175ff7d1 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -20,6 +20,10 @@ add_executable(
 # Your-external "mylib", add GLOBAL if the imported library is located in directories above the current.
 add_library( libOpenFOAM SHARED IMPORTED )
 
+set_target_properties(libOSspecific)
+    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libOSspecific.o
+    )
+./platforms/linux64GccDPInt32Opt/lib/libOSspecific.o
 # You can define two import-locations: one for debug and one for release.
 set_target_properties(libPstream
     PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/dummy/libPstream.so
@@ -44,6 +48,7 @@ target_link_libraries(
       libOpenFOAM
       libfiniteVolume
       libPstream
+      libOSspecific
 )
 
 

From 49ae888a3993acb5675d168af6eddf99bfa0d361 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 11:54:00 +0100
Subject: [PATCH 100/125] fix lib specs

---
 unitTests/CMakeLists.txt | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 9175ff7d1..f47fb57a9 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -17,20 +17,21 @@ add_executable(
    "test_HostMatrix.C"
 )
 
-# Your-external "mylib", add GLOBAL if the imported library is located in directories above the current.
-add_library( libOpenFOAM SHARED IMPORTED )
-
+add_library(libOSspecific SHARED IMPORTED )
 set_target_properties(libOSspecific)
     PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libOSspecific.o
     )
-./platforms/linux64GccDPInt32Opt/lib/libOSspecific.o
+
 # You can define two import-locations: one for debug and one for release.
+add_library(libPstream SHARED IMPORTED )
 set_target_properties(libPstream
     PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/dummy/libPstream.so
     )
 
 
-# You can define two import-locations: one for debug and one for release.
+# Your-external "mylib", add GLOBAL if the imported library is located in directories above the current.
+add_library( libOpenFOAM SHARED IMPORTED )
+
 set_target_properties(libOpenFOAM
     PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libOpenFOAM.so
     )

From bfc5e98f166a04309579ae79228a82528f99d63e Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 12:35:25 +0100
Subject: [PATCH 101/125] fix lib specs

---
 unitTests/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index f47fb57a9..29b9c35ed 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -18,7 +18,7 @@ add_executable(
 )
 
 add_library(libOSspecific SHARED IMPORTED )
-set_target_properties(libOSspecific)
+set_target_properties(libOSspecific
     PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libOSspecific.o
     )
 

From d69701be84b5d3b820f30edef5619545a1e8dd7f Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 13:26:51 +0100
Subject: [PATCH 102/125] source OF

---
 .github/workflows/unit-test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 6c8126c6a..0269d910a 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -36,5 +36,6 @@ jobs:
 
     - name: Test OGL
       working-directory: ${{github.workspace}}/build
-      shell: bash
-      run: ./unitTests/matrixConversion
+      run:
+        source /root/OpenFOAM/${{inputs.path}}/etc/bashrc
+        ./unitTests/matrixConversion

From 2f1d769860bd25740b04d8a8c2279b139de50e8c Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 13:27:28 +0100
Subject: [PATCH 103/125] linked sys-openmpi libPsream

---
 unitTests/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 29b9c35ed..1beb35e58 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -25,7 +25,7 @@ set_target_properties(libOSspecific
 # You can define two import-locations: one for debug and one for release.
 add_library(libPstream SHARED IMPORTED )
 set_target_properties(libPstream
-    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/dummy/libPstream.so
+    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/sys-openmpi/libPstream.so
     )
 
 

From a463daedb623c97450d5788fbc378a63ceb4affc Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 14:12:10 +0100
Subject: [PATCH 104/125] source OF for unitTestBuild

---
 .github/workflows/build-foam.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index aca29f26e..c0f9ac4b9 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -93,6 +93,7 @@ jobs:
     - name: Build OGL
       working-directory: ${{github.workspace}}/build
       run: |
+        source /root/OpenFOAM/${{matrix.OF.path}}/etc/bashrc
         cmake --build . --config $BUILD_TYPE
 
     - name: Install OGL

From 4a10253ecbe80acbbc53dffdae25f81ee242e3f7 Mon Sep 17 00:00:00 2001
From: HendriceH <110101174+HendriceH@users.noreply.github.com>
Date: Fri, 17 Nov 2023 16:54:37 +0100
Subject: [PATCH 105/125] Update CMakeLists.txt

---
 CMakeLists.txt | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7635b335..8d21a5161 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -47,7 +47,7 @@ include(cmake/package_helpers.cmake)
 include(cmake/build_helpers.cmake)
 include(cmake/install_helpers.cmake)
 
-ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.4.0)
+ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.7.0)
 
 add_subdirectory(third_party)
 
@@ -56,13 +56,15 @@ if(NOT ${OGL_USE_EXTERNAL_GINKGO})
   find_package(
     Ginkgo
     CONFIG
-    1.5.0
+    1.7.0
     REQUIRED
     PATHS
     $ENV{FOAM_USER_LIBBIN}/lib/cmake/Ginkgo
     ${CMAKE_CURRENT_BINARY_DIR}/third_party/ginkgo/build/install/lib/cmake)
 endif()
 
+find_package(CUDAToolkit REQUIRED)
+
 # C++ 14 standard or later is required to interface with ginkgo 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -195,7 +197,7 @@ if(${GINKGO_WITH_OGL_EXTENSIONS})
 endif()
 if(${GINKGO_BUILD_CUDA})
   target_compile_definitions(OGL PRIVATE GINKGO_BUILD_CUDA=1)
-  target_link_libraries(OGL PUBLIC nvToolsExt)
+  target_link_libraries(OGL PUBLIC CUDA::nvToolsExt)
 endif()
 
 if(OGL_DATA_VALIDATION)

From 872511307e92cda4d4e95f9b71d86c4ded996fef Mon Sep 17 00:00:00 2001
From: HendriceH <110101174+HendriceH@users.noreply.github.com>
Date: Wed, 15 Nov 2023 17:27:41 +0100
Subject: [PATCH 106/125] Update CMakeLists.txt

---
 third_party/ginkgo/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/third_party/ginkgo/CMakeLists.txt b/third_party/ginkgo/CMakeLists.txt
index f63d5dda6..3b0e2e9ac 100644
--- a/third_party/ginkgo/CMakeLists.txt
+++ b/third_party/ginkgo/CMakeLists.txt
@@ -12,6 +12,7 @@ ginkgo_load_git_package(
   "-DGINKGO_BUILD_BENCHMARKS=OFF"
   "-DGINKGO_BUILD_REFERENCE=${GINKGO_BUILD_REFERENCE}"
   "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+  "-DCMAKE_CUDA_ARCHITECTURES=75"
   "-DGINKGO_BUILD_HWLOC=OFF"
   "-DGINKGO_FORCE_GPU_AWARE_MPI=${GINKGO_FORCE_GPU_AWARE_MPI}"
   "-DCMAKE_INSTALL_PREFIX=$ENV{FOAM_USER_LIBBIN}/.."

From 8f1ab5db63718b38e181bda712813f0cf54958e7 Mon Sep 17 00:00:00 2001
From: HendriceH <110101174+HendriceH@users.noreply.github.com>
Date: Thu, 16 Nov 2023 11:47:52 +0100
Subject: [PATCH 107/125] Update common.C

---
 common/common.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/common.C b/common/common.C
index 644485c28..96e82ab58 100644
--- a/common/common.C
+++ b/common/common.C
@@ -62,7 +62,7 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
                 const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
-    std::filesystem::create_directories(folder);
+    //std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
     std::ofstream stream{fn};
     stream << std::setprecision(15);

From d28aa403ee2d9b9ac3963c111e8200fe6ac27844 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 20:51:46 +0100
Subject: [PATCH 108/125] build tests for different mpi versions

---
 unitTests/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 1beb35e58..5c515630c 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -25,7 +25,7 @@ set_target_properties(libOSspecific
 # You can define two import-locations: one for debug and one for release.
 add_library(libPstream SHARED IMPORTED )
 set_target_properties(libPstream
-    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/sys-openmpi/libPstream.so
+    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so
     )
 
 

From dd4c73dc340fda6d6c618cec541dbddf58566476 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 21:01:57 +0100
Subject: [PATCH 109/125] add DL libs

---
 unitTests/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 5c515630c..bee01093a 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -50,6 +50,7 @@ target_link_libraries(
       libfiniteVolume
       libPstream
       libOSspecific
+      ${CMAKE_DL_LIBS}
 )
 
 

From 3bdae8f8be8ec73244243d641230dcd36a85e3cb Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 21:36:33 +0100
Subject: [PATCH 110/125] add cmake format target and format cmake files

---
 CMakeLists.txt                    | 62 +++++++++++++++++--------------
 third_party/CMakeLists.txt        |  4 +-
 third_party/ginkgo/CMakeLists.txt |  3 +-
 unitTests/CMakeLists.txt          | 57 +++++++---------------------
 4 files changed, 53 insertions(+), 73 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0a9cbdf15..c7ad98c38 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -26,15 +26,21 @@ option(GINKGO_BUILD_HIP "Build Ginkgo with hip backend" FALSE)
 option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE)
 option(GINKGO_BUILD_OMP "Build Ginkgo with omp backend" FALSE)
 option(GINKGO_BUILD_REFERENCE "Build Ginkgo with reference backend" TRUE)
-option(OGL_ALLOW_REFERENCE_ONLY "Enable builds of Ginkgo with reference backend only" FALSE)
+option(OGL_ALLOW_REFERENCE_ONLY
+       "Enable builds of Ginkgo with reference backend only" FALSE)
 option(GINKGO_FORCE_GPU_AWARE_MPI "Build Ginkgo using device aware MPI" TRUE)
 option(GINKGO_WITH_OGL_EXTENSIONS "Whether ginkgo was build with OGL extension"
        FALSE)
 
 if(NOT OGL_ALLOW_REFERENCE_ONLY)
-    if ((NOT GINKGO_BUILD_CUDA) AND (NOT GINKGO_BUILD_HIP) AND (NOT GINKGO_BUILD_SYCL))
-        message(FATAL_ERROR "No GPU Backend was selected, set either:\n -DGINKGO_BUILD_CUDA=True\n -DGINKGO_BUILD_HIP=True\n -DGINKGO_BUILD_SYCL=True\n or set:\n -DOGL_ALLOW_REFERENCE_ONLY=True\n to turn off this check")
-    endif()
+  if((NOT GINKGO_BUILD_CUDA)
+     AND (NOT GINKGO_BUILD_HIP)
+     AND (NOT GINKGO_BUILD_SYCL))
+    message(
+      FATAL_ERROR
+        "No GPU Backend was selected, set either:\n -DGINKGO_BUILD_CUDA=True\n -DGINKGO_BUILD_HIP=True\n -DGINKGO_BUILD_SYCL=True\n or set:\n -DOGL_ALLOW_REFERENCE_ONLY=True\n to turn off this check"
+    )
+  endif()
 endif()
 
 set(GINKGO_CHECKOUT_VERSION
@@ -64,7 +70,7 @@ if(NOT ${OGL_USE_EXTERNAL_GINKGO})
     ${CMAKE_CURRENT_BINARY_DIR}/third_party/ginkgo/build/install/lib/cmake)
 endif()
 
-# C++ 14 standard or later is required to interface with ginkgo 
+# C++ 14 standard or later is required to interface with ginkgo
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -90,9 +96,8 @@ if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
     -Rpass=loop-vectorize \
     -ffast-math \
     "
-#- Rpass-analysis=loop-vectorize  -Rpass-missed=loop-vectorize
-)
-
+      # * Rpass-analysis=loop-vectorize  -Rpass-missed=loop-vectorize
+  )
 
 elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
   add_definitions(-DWITH_GNU)
@@ -102,7 +107,7 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
       --coverage\
       -fprofile-arcs \
       -ggdb3")
-#- fsanitize = address \ - ftest - coverage \
+  # * fsanitize = address \ - ftest - coverage \
 
   set(CMAKE_CXX_FLAGS_RELEASE
       "${CMAKE_CXX_FLAGS_RELEASE} \
@@ -116,19 +121,20 @@ endif()
 option(BUILD_SHARED_LIBS "Build shared library" ON)
 add_library(OGL "")
 
-add_compile_definitions(WM_LABEL_SIZE=32 WM_ARCH_OPTION=64 NoRepository WM_DP)
+target_compile_definitions(OGL PUBLIC WM_LABEL_SIZE=32 WM_ARCH_OPTION=64
+                                      NoRepository WM_DP)
 
 if(APPLE)
   message("Compiling on MacOS")
   target_link_options(OGL PUBLIC "LINKER:-undefined,dynamic_lookup")
-  add_compile_definitions(APPLE arm64)
+  target_compile_definitions(OGL PRIVATE APPLE arm64)
 else()
-  add_compile_definitions(linux64)
+  target_compile_definitions(OGL PRIVATE linux64)
 endif()
 
-#Add a custom command that produces version.cpp, plus a dummy output that's not
-# actually produced, in order to always force re-run version.cmake before
-#the build
+# Add a custom command that produces version.cpp, plus a dummy output that's not
+# actually produced, in order to always force re-run version.cmake before the
+# build
 add_custom_command(
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/version.C
          ${CMAKE_CURRENT_BINARY_DIR}/_version.C
@@ -156,9 +162,9 @@ target_sources(
           HostMatrix/HostMatrixFreeFunctions.C
           Solver/CG/GKOCG.C
           Solver/BiCGStab/GKOBiCGStab.C
-#Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C
+          # Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C
           Solver/GMRES/GKOGMRES.C
-#LduMatrix / GKOACG / GKOACG.C
+  # LduMatrix / GKOACG / GKOACG.C
   PUBLIC common/common.H
          StoppingCriterion/StoppingCriterion.H
          lduLduBase/lduLduBase.H
@@ -176,20 +182,17 @@ target_sources(
          BaseWrapper/CoupledLduBase/GKOCoupledLduBase.H
          Solver/CG/GKOCG.H
          Solver/IR/GKOIR.H
-#Solver / Multigrid / GKOMultigrid.H
+         # Solver / Multigrid / GKOMultigrid.H
          Solver/BiCGStab/GKOBiCGStab.H
          Solver/GMRES/GKOGMRES.H
-#LduMatrix / GKOACG / GKOACG.H
-         )
+  # LduMatrix / GKOACG / GKOACG.H
+)
 
 target_include_directories(
   OGL SYSTEM
   PUBLIC $ENV{FOAM_SRC}/finiteVolume/lnInclude
-          $ENV{FOAM_SRC}/meshTools/lnInclude
-          $ENV{FOAM_SRC}/OpenFOAM/lnInclude
-          $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude
-          ${CMAKE_CURRENT_SOURCE_DIR}
-          )
+         $ENV{FOAM_SRC}/meshTools/lnInclude $ENV{FOAM_SRC}/OpenFOAM/lnInclude
+         $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude ${CMAKE_CURRENT_SOURCE_DIR})
 
 target_link_libraries(OGL PUBLIC Ginkgo::ginkgo stdc++fs)
 
@@ -210,8 +213,13 @@ if(EXISTS $ENV{WM_PROJECT_DIR}/CONTRIBUTORS.md)
 endif()
 
 if(${OGL_BUILD_UNITTEST})
-    add_subdirectory(unitTests)
-    enable_testing()
+  add_subdirectory(unitTests)
+  enable_testing()
 endif()
 
+add_custom_target(
+  cmake-format
+  COMMAND find . -name CMakeLists.txt -exec cmake-format -i {} +
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
+
 install(TARGETS OGL DESTINATION $ENV{FOAM_USER_LIBBIN})
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index cec94bc37..eda57cf67 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -1,2 +1,4 @@
 ginkgo_add_subdirectory(Ginkgo ginkgo)
-set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
+set(CMAKE_MODULE_PATH
+    "${CMAKE_MODULE_PATH}"
+    PARENT_SCOPE)
diff --git a/third_party/ginkgo/CMakeLists.txt b/third_party/ginkgo/CMakeLists.txt
index 56195154f..f25f35df3 100644
--- a/third_party/ginkgo/CMakeLists.txt
+++ b/third_party/ginkgo/CMakeLists.txt
@@ -16,5 +16,4 @@ ginkgo_load_git_package(
   "-DGINKGO_FORCE_GPU_AWARE_MPI=${GINKGO_FORCE_GPU_AWARE_MPI}"
   "-DCMAKE_INSTALL_PREFIX=$ENV{FOAM_USER_LIBBIN}/.."
   "-DBUILD_SHARED_LIBS=ON"
-  "-DCMAKE_CXX_STANDARD=17"
-  )
+  "-DCMAKE_CXX_STANDARD=17")
diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index bee01093a..0ea1d329e 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -1,6 +1,9 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 # SPDX-FileCopyrightText: 2023 NeoFOAM authors
-option(INSTALL_GTEST "Enable installation of googletest. (Projects embedding googletest may want to turn this OFF.)" OFF)
+option(
+  INSTALL_GTEST
+  "Enable installation of googletest. (Projects embedding googletest may want to turn this OFF.)"
+  OFF)
 include(FetchContent)
 FetchContent_Declare(
   googletest
@@ -8,51 +11,19 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(googletest)
 
-include_directories(
-    ${googletest_SOURCE_DIR}/googletest/include
-)
-
-add_executable(
-   matrixConversion
-   "test_HostMatrix.C"
-)
-
-add_library(libOSspecific SHARED IMPORTED )
-set_target_properties(libOSspecific
-    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libOSspecific.o
-    )
-
-# You can define two import-locations: one for debug and one for release.
-add_library(libPstream SHARED IMPORTED )
-set_target_properties(libPstream
-    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so
-    )
+include_directories(${googletest_SOURCE_DIR}/googletest/include)
 
-
-# Your-external "mylib", add GLOBAL if the imported library is located in directories above the current.
-add_library( libOpenFOAM SHARED IMPORTED )
-
-set_target_properties(libOpenFOAM
-    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libOpenFOAM.so
-    )
-
-add_library(libfiniteVolume SHARED IMPORTED )
-set_target_properties(libfiniteVolume
-    PROPERTIES IMPORTED_LOCATION_RELEASE $ENV{FOAM_LIBBIN}/libfiniteVolume.so
-    )
+add_executable(matrixConversion "test_HostMatrix.C")
 
 target_link_libraries(
-    matrixConversion
-  PUBLIC
-      GTest::gtest_main 
-      OGL
-      libOpenFOAM
-      libfiniteVolume
-      libPstream
-      libOSspecific
-      ${CMAKE_DL_LIBS}
-)
-
+  matrixConversion
+  PUBLIC GTest::gtest_main
+         OGL
+         $ENV{FOAM_LIBBIN}/libOpenFOAM.so
+         $ENV{FOAM_LIBBIN}/libfiniteVolume.so
+         $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so
+         $ENV{FOAM_LIBBIN}/libOSspecific.o
+         ${CMAKE_DL_LIBS})
 
 include(GoogleTest)
 gtest_discover_tests(matrixConversion)

From 8165ebb36675d65bf364ceae252804dacef8a56a Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 21:51:40 +0100
Subject: [PATCH 111/125] clean up

---
 CMakeLists.txt           | 2 +-
 unitTests/CMakeLists.txt | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c7ad98c38..6817ceb0b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -213,8 +213,8 @@ if(EXISTS $ENV{WM_PROJECT_DIR}/CONTRIBUTORS.md)
 endif()
 
 if(${OGL_BUILD_UNITTEST})
-  add_subdirectory(unitTests)
   enable_testing()
+  add_subdirectory(unitTests)
 endif()
 
 add_custom_target(
diff --git a/unitTests/CMakeLists.txt b/unitTests/CMakeLists.txt
index 0ea1d329e..b3bd4ffa9 100644
--- a/unitTests/CMakeLists.txt
+++ b/unitTests/CMakeLists.txt
@@ -11,8 +11,6 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(googletest)
 
-include_directories(${googletest_SOURCE_DIR}/googletest/include)
-
 add_executable(matrixConversion "test_HostMatrix.C")
 
 target_link_libraries(

From b7f23fb7e881c0f4cee0bb42fa138d7a50b374b2 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Mon, 20 Nov 2023 21:52:15 +0100
Subject: [PATCH 112/125] add cmake-format file

---
 .cmake-format.py | 243 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 243 insertions(+)
 create mode 100644 .cmake-format.py

diff --git a/.cmake-format.py b/.cmake-format.py
new file mode 100644
index 000000000..2e3e7141f
--- /dev/null
+++ b/.cmake-format.py
@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: Unlicense
+# SPDX-FileCopyrightText: 2023 NeoFOAM authors
+# ----------------------------------
+# Options affecting listfile parsing
+# ----------------------------------
+with section("parse"):
+
+  # Specify structure for custom cmake functions
+  additional_commands = { 'foo': { 'flags': ['BAR', 'BAZ'],
+             'kwargs': {'DEPENDS': '*', 'HEADERS': '*', 'SOURCES': '*'}}}
+
+  # Override configurations per-command where available
+  override_spec = {}
+
+  # Specify variable tags.
+  vartags = []
+
+  # Specify property tags.
+  proptags = []
+
+# -----------------------------
+# Options affecting formatting.
+# -----------------------------
+with section("format"):
+
+  # Disable formatting entirely, making cmake-format a no-op
+  disable = False
+
+  # How wide to allow formatted cmake files
+  line_width = 80
+
+  # How many spaces to tab for indent
+  tab_size = 2
+
+  # If true, lines are indented using tab characters (utf-8 0x09) instead of
+  # <tab_size> space characters (utf-8 0x20). In cases where the layout would
+  # require a fractional tab character, the behavior of the  fractional
+  # indentation is governed by <fractional_tab_policy>
+  use_tabchars = False
+
+  # If <use_tabchars> is True, then the value of this variable indicates how
+  # fractional indentions are handled during whitespace replacement. If set to
+  # 'use-space', fractional indentation is left as spaces (utf-8 0x20). If set
+  # to `round-up` fractional indentation is replaced with a single tab character
+  # (utf-8 0x09) effectively shifting the column to the next tabstop
+  fractional_tab_policy = 'use-space'
+
+  # If an argument group contains more than this many sub-groups (parg or kwarg
+  # groups) then force it to a vertical layout.
+  max_subgroups_hwrap = 2
+
+  # If a positional argument group contains more than this many arguments, then
+  # force it to a vertical layout.
+  max_pargs_hwrap = 6
+
+  # If a cmdline positional group consumes more than this many lines without
+  # nesting, then invalidate the layout (and nest)
+  max_rows_cmdline = 2
+
+  # If true, separate flow control names from their parentheses with a space
+  separate_ctrl_name_with_space = False
+
+  # If true, separate function names from parentheses with a space
+  separate_fn_name_with_space = False
+
+  # If a statement is wrapped to more than one line, than dangle the closing
+  # parenthesis on its own line.
+  dangle_parens = False
+
+  # If the trailing parenthesis must be 'dangled' on its on line, then align it
+  # to this reference: `prefix`: the start of the statement,  `prefix-indent`:
+  # the start of the statement, plus one indentation  level, `child`: align to
+  # the column of the arguments
+  dangle_align = 'prefix'
+
+  # If the statement spelling length (including space and parenthesis) is
+  # smaller than this amount, then force reject nested layouts.
+  min_prefix_chars = 4
+
+  # If the statement spelling length (including space and parenthesis) is larger
+  # than the tab width by more than this amount, then force reject un-nested
+  # layouts.
+  max_prefix_chars = 10
+
+  # If a candidate layout is wrapped horizontally but it exceeds this many
+  # lines, then reject the layout.
+  max_lines_hwrap = 2
+
+  # What style line endings to use in the output.
+  line_ending = 'unix'
+
+  # Format command names consistently as 'lower' or 'upper' case
+  command_case = 'canonical'
+
+  # Format keywords consistently as 'lower' or 'upper' case
+  keyword_case = 'unchanged'
+
+  # A list of command names which should always be wrapped
+  always_wrap = []
+
+  # If true, the argument lists which are known to be sortable will be sorted
+  # lexicographicall
+  enable_sort = True
+
+  # If true, the parsers may infer whether or not an argument list is sortable
+  # (without annotation).
+  autosort = False
+
+  # By default, if cmake-format cannot successfully fit everything into the
+  # desired linewidth it will apply the last, most agressive attempt that it
+  # made. If this flag is True, however, cmake-format will print error, exit
+  # with non-zero status code, and write-out nothing
+  require_valid_layout = False
+
+  # A dictionary mapping layout nodes to a list of wrap decisions. See the
+  # documentation for more information.
+  layout_passes = {}
+
+# ------------------------------------------------
+# Options affecting comment reflow and formatting.
+# ------------------------------------------------
+with section("markup"):
+
+  # What character to use for bulleted lists
+  bullet_char = '*'
+
+  # What character to use as punctuation after numerals in an enumerated list
+  enum_char = '.'
+
+  # If comment markup is enabled, don't reflow the first comment block in each
+  # listfile. Use this to preserve formatting of your copyright/license
+  # statements.
+  first_comment_is_literal = True
+
+  # If comment markup is enabled, don't reflow any comment block which matches
+  # this (regex) pattern. Default is `None` (disabled).
+  literal_comment_pattern = None
+
+  # Regular expression to match preformat fences in comments default=
+  # ``r'^\s*([`~]{3}[`~]*)(.*)$'``
+  fence_pattern = '^\\s*([`~]{3}[`~]*)(.*)$'
+
+  # Regular expression to match rulers in comments default=
+  # ``r'^\s*[^\w\s]{3}.*[^\w\s]{3}$'``
+  ruler_pattern = '^\\s*[^\\w\\s]{3}.*[^\\w\\s]{3}$'
+
+  # If a comment line matches starts with this pattern then it is explicitly a
+  # trailing comment for the preceeding argument. Default is '#<'
+  explicit_trailing_pattern = '#<'
+
+  # If a comment line starts with at least this many consecutive hash
+  # characters, then don't lstrip() them off. This allows for lazy hash rulers
+  # where the first hash char is not separated by space
+  hashruler_min_length = 10
+
+  # If true, then insert a space between the first hash char and remaining hash
+  # chars in a hash ruler, and normalize its length to fill the column
+  canonicalize_hashrulers = True
+
+  # enable comment markup parsing and reflow
+  enable_markup = True
+
+# ----------------------------
+# Options affecting the linter
+# ----------------------------
+with section("lint"):
+
+  # a list of lint codes to disable
+  disabled_codes = []
+
+  # regular expression pattern describing valid function names
+  function_pattern = '[0-9a-z_]+'
+
+  # regular expression pattern describing valid macro names
+  macro_pattern = '[0-9A-Z_]+'
+
+  # regular expression pattern describing valid names for variables with global
+  # (cache) scope
+  global_var_pattern = '[A-Z][0-9A-Z_]+'
+
+  # regular expression pattern describing valid names for variables with global
+  # scope (but internal semantic)
+  internal_var_pattern = '_[A-Z][0-9A-Z_]+'
+
+  # regular expression pattern describing valid names for variables with local
+  # scope
+  local_var_pattern = '[a-z][a-z0-9_]+'
+
+  # regular expression pattern describing valid names for privatedirectory
+  # variables
+  private_var_pattern = '_[0-9a-z_]+'
+
+  # regular expression pattern describing valid names for public directory
+  # variables
+  public_var_pattern = '[A-Z][0-9A-Z_]+'
+
+  # regular expression pattern describing valid names for function/macro
+  # arguments and loop variables.
+  argument_var_pattern = '[a-z][a-z0-9_]+'
+
+  # regular expression pattern describing valid names for keywords used in
+  # functions or macros
+  keyword_pattern = '[A-Z][0-9A-Z_]+'
+
+  # In the heuristic for C0201, how many conditionals to match within a loop in
+  # before considering the loop a parser.
+  max_conditionals_custom_parser = 2
+
+  # Require at least this many newlines between statements
+  min_statement_spacing = 1
+
+  # Require no more than this many newlines between statements
+  max_statement_spacing = 2
+  max_returns = 6
+  max_branches = 12
+  max_arguments = 5
+  max_localvars = 15
+  max_statements = 50
+
+# -------------------------------
+# Options affecting file encoding
+# -------------------------------
+with section("encode"):
+
+  # If true, emit the unicode byte-order mark (BOM) at the start of the file
+  emit_byteorder_mark = False
+
+  # Specify the encoding of the input file. Defaults to utf-8
+  input_encoding = 'utf-8'
+
+  # Specify the encoding of the output file. Defaults to utf-8. Note that cmake
+  # only claims to support utf-8 so be careful when using anything else
+  output_encoding = 'utf-8'
+
+# -------------------------------------
+# Miscellaneous configurations options.
+# -------------------------------------
+with section("misc"):
+
+  # A dictionary containing any per-command configuration overrides. Currently
+  # only `command_case` is supported.
+  per_command = {}
+

From e5f09ebae6366d48acd79d2cc8e0ad2541ffe0a4 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 21 Nov 2023 08:47:15 +0100
Subject: [PATCH 113/125] deactivate unit test for now

---
 .github/workflows/build-foam.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index c0f9ac4b9..d2454b9b6 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -88,7 +88,7 @@ jobs:
         mkdir -p build
         cd build
         [ -d "/github/home/$GINKGO_CHECKOUT_VERSION" ] && cp -rp /github/home/$GINKGO_CHECKOUT_VERSION third_party
-        cmake -G Ninja -DOGL_ALLOW_REFERENCE_ONLY=On -DOGL_BUILD_UNITTEST=On -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+        cmake -G Ninja -DOGL_ALLOW_REFERENCE_ONLY=On -DOGL_BUILD_UNITTEST=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
 
     - name: Build OGL
       working-directory: ${{github.workspace}}/build

From a0f467040d8824e6d67c2fdea759f4569a7b8c49 Mon Sep 17 00:00:00 2001
From: HendriceH <110101174+HendriceH@users.noreply.github.com>
Date: Tue, 21 Nov 2023 08:50:22 +0100
Subject: [PATCH 114/125] Update common/common.C

Co-authored-by: Gregor Olenik <gregor.olenik@web.de>
---
 common/common.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/common.C b/common/common.C
index 96e82ab58..644485c28 100644
--- a/common/common.C
+++ b/common/common.C
@@ -62,7 +62,7 @@ void export_mtx(const word fieldName, std::shared_ptr<const gko::LinOp> A,
                 const word matrixFormat)
 {
     std::string folder{db.time().timePath()};
-    //std::filesystem::create_directories(folder);
+    std::filesystem::create_directories(folder);
     std::string fn{folder + "/" + fieldName + "_A_" + local + ".mtx"};
     std::ofstream stream{fn};
     stream << std::setprecision(15);

From 1d402e3cba10a43981c3cb8c0459de75004d2779 Mon Sep 17 00:00:00 2001
From: HendriceH <110101174+HendriceH@users.noreply.github.com>
Date: Tue, 21 Nov 2023 08:50:39 +0100
Subject: [PATCH 115/125] Update CMakeLists.txt

Co-authored-by: Gregor Olenik <gregor.olenik@web.de>
---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8d21a5161..8694d9e19 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -196,6 +196,7 @@ if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)
 endif()
 if(${GINKGO_BUILD_CUDA})
+  find_package(CUDAToolkit REQUIRED)
   target_compile_definitions(OGL PRIVATE GINKGO_BUILD_CUDA=1)
   target_link_libraries(OGL PUBLIC CUDA::nvToolsExt)
 endif()

From 0db5b8d2b38afe44375361a005c40e8b53f5c42f Mon Sep 17 00:00:00 2001
From: HendriceH <110101174+HendriceH@users.noreply.github.com>
Date: Tue, 21 Nov 2023 08:50:47 +0100
Subject: [PATCH 116/125] Update CMakeLists.txt

Co-authored-by: Gregor Olenik <gregor.olenik@web.de>
---
 CMakeLists.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8694d9e19..8d82183e8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -63,7 +63,6 @@ if(NOT ${OGL_USE_EXTERNAL_GINKGO})
     ${CMAKE_CURRENT_BINARY_DIR}/third_party/ginkgo/build/install/lib/cmake)
 endif()
 
-find_package(CUDAToolkit REQUIRED)
 
 # C++ 14 standard or later is required to interface with ginkgo 
 set(CMAKE_CXX_STANDARD 17)

From de03f24598aa6392f2e5c4f73d2d17f8524dcfc0 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 21 Nov 2023 08:53:53 +0100
Subject: [PATCH 117/125] deactivate unit test for now

---
 .github/workflows/build-foam.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index d2454b9b6..9f990a044 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -114,14 +114,14 @@ jobs:
         name: ogl_build_${{ matrix.OF.path }}
         path: ${{github.workspace}}/build
 
-  unit_tests:
-    needs: [build, setup_build_matrix]
-    uses: ./.github/workflows/unit-test.yml
-    with:
-      path: ${{ matrix.OF.path }}
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.setup_build_matrix.outputs.matrix) }}
+  # unit_tests:
+  #   needs: [build, setup_build_matrix]
+  #   uses: ./.github/workflows/unit-test.yml
+  #   with:
+  #     path: ${{ matrix.OF.path }}
+  #   strategy:
+  #     fail-fast: false
+  #     matrix: ${{ fromJson(needs.setup_build_matrix.outputs.matrix) }}
 
   integration_tests:
     needs: [build, unit_tests, setup_build_matrix]

From 033778ca7914838d4a89ea160b0ee784f750939b Mon Sep 17 00:00:00 2001
From: HendriceH <110101174+HendriceH@users.noreply.github.com>
Date: Tue, 21 Nov 2023 08:54:22 +0100
Subject: [PATCH 118/125] Link stdc++fs into OGL

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8d82183e8..c7b9f05a1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -189,7 +189,7 @@ target_include_directories(
           ${CMAKE_CURRENT_SOURCE_DIR}
           )
 
-target_link_libraries(OGL PUBLIC Ginkgo::ginkgo)
+target_link_libraries(OGL PUBLIC Ginkgo::ginkgo stdc++fs)
 
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)

From 9a0ac1c8df0a4df7711f8413a5fc228658d79423 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 21 Nov 2023 11:20:07 +0100
Subject: [PATCH 119/125] add common notation and scotch decomposition

---
 test/cavity.yaml | 56 +++++++++++++++++++++---------------------------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/test/cavity.yaml b/test/cavity.yaml
index 093468be4..4feb84910 100644
--- a/test/cavity.yaml
+++ b/test/cavity.yaml
@@ -15,64 +15,58 @@ case:
 variation:
   - operation: fvSolution
     schema: "linearSolver/{solver}{preconditioner}{executor}"
-    values:
-      - set: solvers/p
-        preconditioner: none
-        solver: GKOCG
+    common:
         forceHostBuffer: 1
         verbose: 1
         debug: 1
         executor: reference
         matrixFormat: Coo
+    values:
+      - set: solvers/p
+        preconditioner: none
+        solver: GKOCG
       - set: solvers/U
         preconditioner: BJ
         solver: GKOBiCGStab
-        forceHostBuffer: 1
-        verbose: 1
-        debug: 1
-        executor: reference
-        matrixFormat: Coo
+    variation:
+      - operation: decomposePar
+        schema: "decomposition/{method}-{numberOfSubdomains}"
+        values:
+          - method: simple
+            numberOfSubdomains: 2
+            coeffs: [2,1,1]
+          - method: simple
+            numberOfSubdomains: 8
+            coeffs: [4,2,1]
+          - method: scotch
+            numberOfSubdomains: 8
   - operation: fvSolution
     schema: "matrixFormat/{matrixFormat}"
-    values:
-      - set: solvers/p
+    common:
         preconditioner: none
         solver: GKOCG
         forceHostBuffer: 1
         verbose: 1
         debug: 1
-        matrixFormat: Csr
         executor: reference
+    values:
+      - set: solvers/p
+        matrixFormat: Csr
       - set: solvers/p
-        preconditioner: none
-        solver: GKOCG
-        forceHostBuffer: 1
-        verbose: 1
-        debug: 1
         regenerate: 1
         matrixFormat: Ell
-        executor: reference
   - operation: fvSolution
     schema: "preconditioner/{preconditioner}"
-    values:
-      - set: solvers/p
-        preconditioner: BJ
+    common:
         solver: GKOCG
         forceHostBuffer: 1
         verbose: 1
         debug: 1
         executor: reference
+    values:
+      - set: solvers/p
+        preconditioner: BJ
       - set: solvers/p
         preconditioner: GISAI
-        solver: GKOCG
-        forceHostBuffer: 1
-        verbose: 1
-        debug: 1
-        executor: reference
       - set: solvers/p
         preconditioner: ILU
-        solver: GKOCG
-        forceHostBuffer: 1
-        verbose: 1
-        debug: 1
-        executor: reference

From 34d8333a1a0a931be3b2805bb34bd1f7c8730c2c Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Tue, 21 Nov 2023 11:24:27 +0100
Subject: [PATCH 120/125] remove unit test dependency

---
 .github/workflows/build-foam.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml
index 9f990a044..c75cc9243 100644
--- a/.github/workflows/build-foam.yml
+++ b/.github/workflows/build-foam.yml
@@ -124,7 +124,7 @@ jobs:
   #     matrix: ${{ fromJson(needs.setup_build_matrix.outputs.matrix) }}
 
   integration_tests:
-    needs: [build, unit_tests, setup_build_matrix]
+    needs: [build, setup_build_matrix]
     uses: ./.github/workflows/integration-tests.yml
     with:
       path: ${{ matrix.OF.path }}

From a92ab41fbcfa2ef050bb16d44872d65e40136852 Mon Sep 17 00:00:00 2001
From: HendriceH <110101174+HendriceH@users.noreply.github.com>
Date: Tue, 21 Nov 2023 12:56:23 +0100
Subject: [PATCH 121/125] Use CUDA_ARCHITECTURES=native for GPU detection

---
 third_party/ginkgo/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/ginkgo/CMakeLists.txt b/third_party/ginkgo/CMakeLists.txt
index 3b0e2e9ac..1b46b390f 100644
--- a/third_party/ginkgo/CMakeLists.txt
+++ b/third_party/ginkgo/CMakeLists.txt
@@ -12,7 +12,7 @@ ginkgo_load_git_package(
   "-DGINKGO_BUILD_BENCHMARKS=OFF"
   "-DGINKGO_BUILD_REFERENCE=${GINKGO_BUILD_REFERENCE}"
   "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
-  "-DCMAKE_CUDA_ARCHITECTURES=75"
+  "-DCMAKE_CUDA_ARCHITECTURES=native"
   "-DGINKGO_BUILD_HWLOC=OFF"
   "-DGINKGO_FORCE_GPU_AWARE_MPI=${GINKGO_FORCE_GPU_AWARE_MPI}"
   "-DCMAKE_INSTALL_PREFIX=$ENV{FOAM_USER_LIBBIN}/.."

From 95c5db3a05527e7001a4accf5c6b583f37f3782b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 22 Nov 2023 10:58:45 +0100
Subject: [PATCH 122/125] use interface_iterator

---
 HostMatrix/HostMatrix.C | 512 +++++++++++++++++++---------------------
 1 file changed, 241 insertions(+), 271 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index 9e596a5ff..e2c4d8d0b 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -29,6 +29,7 @@ SourceFiles
 #include "lduMatrix.H"
 #include "processorFvPatch.H"
 
+#include <map>
 
 namespace Foam {
 
@@ -372,256 +373,224 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     non_local_idxs.reserve(non_local_matrix_nnz_);
 
     label startOfRequests = Pstream::nRequests();
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
-        const auto &face_cells{iface->interface().faceCells()};
-
-        if (isA<processorLduInterface>(iface->interface())) {
+    interface_iterator<processorLduInterface>(
+        interfaces, [&](label, const label interface_size,
+                        const processorLduInterface &patch,
+                        const lduInterfaceField *iface) {
             const processorLduInterface &pldui =
                 refCast<const processorLduInterface>(iface->interface());
+            const auto &face_cells{iface->interface().faceCells()};
             const label neighbProcNo = pldui.neighbProcNo();
 
-            word msg = "send face cells interface " + std::to_string(i) +
-                       " from proc " + std::to_string(Pstream::myProcNo()) +
+            word msg = "send face cells interface from proc " +
+                       std::to_string(Pstream::myProcNo()) +
                        " to neighbour proc " + std::to_string(neighbProcNo);
 
             LOG_2(verbose_, msg)
             pldui.send(Pstream::commsTypes::nonBlocking, face_cells);
-        }
-    }
+        });
+
     Pstream::waitRequests(startOfRequests);
     LOG_2(verbose_, "send face cells done")
 
 
     label interface_ctr = 0;
     label uniqueIdCtr = 0;
-    for (int i = 0; i < interfaces.size(); i++) {
-        if (interface_getter(interfaces, i) == nullptr) {
-            continue;
-        }
-
-        const auto iface{interface_getter(interfaces, i)};
+    interface_iterator<processorLduInterface>(
+        interfaces,
+        [&](label, const label interface_size, const processorLduInterface &patch,
+            const lduInterfaceField *iface) {
+        const processorLduInterface &pldui =
+            refCast<const processorLduInterface>(iface->interface());
         const auto &face_cells{iface->interface().faceCells()};
-        const label interface_size = face_cells.size();
-
-        if (isA<processorLduInterface>(iface->interface())) {
-            const processorLduInterface &pldui =
-                refCast<const processorLduInterface>(iface->interface());
-            const label neighbProcNo = pldui.neighbProcNo();
-
-            word msg_2 = "receive face cells interface " + std::to_string(i) +
-                         " from proc " + std::to_string(neighbProcNo);
-            LOG_2(verbose_, msg_2)
-
-            auto otherSide_tmp = pldui.receive<label>(
-                Pstream::commsTypes::nonBlocking, interface_size);
-            LOG_2(verbose_, "receive face cells done")
-
-            for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto global_row = global_row_index_.toGlobal(
-                    neighbProcNo, otherSide_tmp()[cellI]);
-                auto search = unique_index.find(global_row);
-                // global_row is new and unique
-                label uniqueId = uniqueIdCtr;
-                if (search == unique_index.end()) {
-                    uniqueId = uniqueIdCtr;
-                    unique_index[global_row] = uniqueIdCtr;
-                    uniqueIdCtr++;
-                } else {
-                    uniqueId = unique_index[global_row];
-                }
-                non_local_idxs.push_back(
-                    {interface_ctr, face_cells[cellI], uniqueId});
-                interface_ctr += 1;
+        const label neighbProcNo = pldui.neighbProcNo();
+
+        word msg_2 = "receive face cells interface from proc " +
+                     std::to_string(neighbProcNo);
+        LOG_2(verbose_, msg_2)
+
+        auto otherSide_tmp = pldui.receive<label>(
+            Pstream::commsTypes::nonBlocking, interface_size);
+        LOG_2(verbose_, "receive face cells done")
+
+        for (label cellI = 0; cellI < interface_size; cellI++) {
+            auto global_row = global_row_index_.toGlobal(
+                neighbProcNo, otherSide_tmp()[cellI]);
+            auto search = unique_index.find(global_row);
+            // global_row is new and unique
+            label uniqueId = uniqueIdCtr;
+            if (search == unique_index.end()) {
+                uniqueId = uniqueIdCtr;
+                unique_index[global_row] = uniqueIdCtr;
+                uniqueIdCtr++;
+            } else {
+                uniqueId = unique_index[global_row];
             }
-        }
-    }
-    word msg = "done collecting neighbouring processor cell id";
-
-    // interface_iterator<processorFvPatch>(
-    //     interfaces,
-    //     [&](label &element_ctr, const label interface_size,
-    //         const processorFvPatch &, const lduInterfaceField *iface) {
-    //         const auto &face_cells{iface->interface().faceCells()};
-    //         for (label cellI = 0; cellI < interface_size; cellI++) {
-    //             const label cellId = face_cells[cellI];
-    //             label uniqueId = uniqueIdCtr;
-    //             auto search = unique_index.find(cellId);
-    //             // cellId is new and unique
-    //             if (search == unique_index.end()) {
-    //                 uniqueId = uniqueIdCtr;
-    //                 unique_index[cellId] = uniqueIdCtr;
-    //                 uniqueIdCtr++;
-    //             } else {
-    //                 uniqueId = unique_index[cellId];
-    //             }
-    //             non_local_idxs.push_back({element_ctr, cellId, uniqueId});
-    //             element_ctr += 1;
-    //         }
-    //     });
-    //
-    // word msg = "done collecting neighbouring processor cell ids";
+            non_local_idxs.push_back(
+                {interface_ctr, face_cells[cellI], uniqueId});
+            interface_ctr += 1;
+        }});
+
+        word msg = "done collecting neighbouring processor cell id";
 
-    LOG_2(verbose_, msg)
-    return non_local_idxs;
+        LOG_2(verbose_, msg)
+        return non_local_idxs;
 }
 
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
-    auto non_local_row_indices = collect_cells_on_interface(interfaces);
-    auto rows = non_local_sparsity_.row_idxs_.get_data();
-    auto cols = non_local_sparsity_.col_idxs_.get_data();
-    auto permute = non_local_sparsity_.ldu_mapping_.get_data();
-
-    // TODO check if sorting is actually needed since interfaces should
-    // be sorted already
-    std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
-              [&](const auto &a, const auto &b) {
-                  auto [interface_idx_a, row_a, unique_col_a] = a;
-                  auto [interface_idx_b, row_b, unique_col_b] = b;
-                  return std::tie(row_a, interface_idx_a) <
-                         std::tie(row_b, interface_idx_b);
-              });
+        auto non_local_row_indices = collect_cells_on_interface(interfaces);
+        auto rows = non_local_sparsity_.row_idxs_.get_data();
+        auto cols = non_local_sparsity_.col_idxs_.get_data();
+        auto permute = non_local_sparsity_.ldu_mapping_.get_data();
+
+        // TODO check if sorting is actually needed since interfaces should
+        // be sorted already
+        std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
+                  [&](const auto &a, const auto &b) {
+                      auto [interface_idx_a, row_a, unique_col_a] = a;
+                      auto [interface_idx_b, row_b, unique_col_b] = b;
+                      return std::tie(row_a, interface_idx_a) <
+                             std::tie(row_b, interface_idx_b);
+                  });
 
-    interface_iterator<processorFvPatch>(
-        interfaces, [&](label &element_ctr, const label interface_size,
-                        const processorFvPatch &, const lduInterfaceField *) {
-            for (label cellI = 0; cellI < interface_size; cellI++) {
-                auto [interface_idx, row, unique_col] =
-                    non_local_row_indices[element_ctr];
-                rows[element_ctr] = row;
-                cols[element_ctr] = unique_col;
-                permute[element_ctr] = interface_idx;
-                element_ctr += 1;
-            }
-        });
+        interface_iterator<processorFvPatch>(
+            interfaces,
+            [&](label &element_ctr, const label interface_size,
+                const processorFvPatch &, const lduInterfaceField *) {
+                for (label cellI = 0; cellI < interface_size; cellI++) {
+                    auto [interface_idx, row, unique_col] =
+                        non_local_row_indices[element_ctr];
+                    rows[element_ctr] = row;
+                    cols[element_ctr] = unique_col;
+                    permute[element_ctr] = interface_idx;
+                    element_ctr += 1;
+                }
+            });
 }
 
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
-    LOG_1(verbose_, "start init host sparsity pattern")
-    bool is_symmetric{this->matrix().symmetric()};
-
-    auto lower_local = idx_array::view(
-        exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
-        const_cast<label *>(&this->matrix().lduAddr().lowerAddr()[0]));
+        LOG_1(verbose_, "start init host sparsity pattern")
+        bool is_symmetric{this->matrix().symmetric()};
 
-    auto upper_local = idx_array::view(
-        exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
-        const_cast<label *>(&this->matrix().lduAddr().upperAddr()[0]));
+        auto lower_local = idx_array::view(
+            exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
+            const_cast<label *>(&this->matrix().lduAddr().lowerAddr()[0]));
 
-    // row of upper, col of lower
-    const auto lower = lower_local.get_const_data();
-    // col of upper, row of lower
-    const auto upper = upper_local.get_const_data();
+        auto upper_local = idx_array::view(
+            exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
+            const_cast<label *>(&this->matrix().lduAddr().upperAddr()[0]));
 
-    auto rows = local_sparsity_.row_idxs_.get_data();
-    auto cols = local_sparsity_.col_idxs_.get_data();
-    const auto permute = local_sparsity_.ldu_mapping_.get_data();
-
-    // Scan through given rows and insert row and column indices into array
-    //
-    //  position after all local offdiagonal elements, needed for
-    //  permutation matrix
-    //
-    //  TODO in order to simplify when local interfaces exists set
-    //  local_sparsity to size of nrows_w_interfaces, if interfaces exist
-    //  local_sparsity is only valid till nrows_
-    label after_neighbours = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
-    init_local_sparsity(nrows_, upper_nnz_, is_symmetric, upper, lower, rows,
-                        cols, permute);
-
-    // if no local interfaces are present we are done here
-    // otherwise we need to add local interfaces in order
-    if (local_interface_nnz_) {
-        auto local_interfaces = collect_local_interface_indices(interfaces);
-        // // sort local interfaces to be in row major order
-        // // and keep original indices
-        std::sort(local_interfaces.begin(), local_interfaces.end(),
-                  [&](const auto &a, const auto &b) {
-                      auto [interface_idx_a, row_a, col_a] = a;
-                      auto [interface_idx_b, row_b, col_b] = b;
-                      return std::tie(row_a, col_a) < std::tie(row_b, col_b);
-                  });
-
-        // copy current rows and columns to tmp array
-        // TODO this has full length but is only valid till nrows_
-        auto rows_copy_a = gko::array<label>(
-            *local_sparsity_.row_idxs_.get_persistent_object().get());
-        auto cols_copy_a = gko::array<label>(
-            *local_sparsity_.col_idxs_.get_persistent_object().get());
-        auto permute_copy_a = gko::array<label>(
-            *local_sparsity_.ldu_mapping_.get_persistent_object().get());
-
-        auto rows_copy = rows_copy_a.get_data();
-        auto cols_copy = cols_copy_a.get_data();
-        auto permute_copy = permute_copy_a.get_data();
+        // row of upper, col of lower
+        const auto lower = lower_local.get_const_data();
+        // col of upper, row of lower
+        const auto upper = upper_local.get_const_data();
 
         auto rows = local_sparsity_.row_idxs_.get_data();
         auto cols = local_sparsity_.col_idxs_.get_data();
-        auto permute = local_sparsity_.ldu_mapping_.get_data();
-
-        label current_idx_ctr = 0;
-        label total_ctr = 0;
-        // iterate local interfaces i
-        // insert all coeffs for row < interface[i].row
-        // find local_interface with lowes row, column
-        for (auto const &interface : local_interfaces) {
-            auto [interface_idx, interface_row, interface_col] = interface;
-
-            // copy from existing matrix coefficients
-            while ([&]() {
-                // check for length
-                if (current_idx_ctr == local_matrix_nnz_) {
-                    return false;
+        const auto permute = local_sparsity_.ldu_mapping_.get_data();
+
+        // Scan through given rows and insert row and column indices into array
+        //
+        //  position after all local offdiagonal elements, needed for
+        //  permutation matrix
+        //
+        //  TODO in order to simplify when local interfaces exists set
+        //  local_sparsity to size of nrows_w_interfaces, if interfaces exist
+        //  local_sparsity is only valid till nrows_
+        label after_neighbours = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
+        init_local_sparsity(nrows_, upper_nnz_, is_symmetric, upper, lower,
+                            rows, cols, permute);
+
+        // if no local interfaces are present we are done here
+        // otherwise we need to add local interfaces in order
+        if (local_interface_nnz_) {
+            auto local_interfaces = collect_local_interface_indices(interfaces);
+            // // sort local interfaces to be in row major order
+            // // and keep original indices
+            std::sort(local_interfaces.begin(), local_interfaces.end(),
+                      [&](const auto &a, const auto &b) {
+                          auto [interface_idx_a, row_a, col_a] = a;
+                          auto [interface_idx_b, row_b, col_b] = b;
+                          return std::tie(row_a, col_a) <
+                                 std::tie(row_b, col_b);
+                      });
+
+            // copy current rows and columns to tmp array
+            // TODO this has full length but is only valid till nrows_
+            auto rows_copy_a = gko::array<label>(
+                *local_sparsity_.row_idxs_.get_persistent_object().get());
+            auto cols_copy_a = gko::array<label>(
+                *local_sparsity_.col_idxs_.get_persistent_object().get());
+            auto permute_copy_a = gko::array<label>(
+                *local_sparsity_.ldu_mapping_.get_persistent_object().get());
+
+            auto rows_copy = rows_copy_a.get_data();
+            auto cols_copy = cols_copy_a.get_data();
+            auto permute_copy = permute_copy_a.get_data();
+
+            auto rows = local_sparsity_.row_idxs_.get_data();
+            auto cols = local_sparsity_.col_idxs_.get_data();
+            auto permute = local_sparsity_.ldu_mapping_.get_data();
+
+            label current_idx_ctr = 0;
+            label total_ctr = 0;
+            // iterate local interfaces i
+            // insert all coeffs for row < interface[i].row
+            // find local_interface with lowes row, column
+            for (auto const &interface : local_interfaces) {
+                auto [interface_idx, interface_row, interface_col] = interface;
+
+                // copy from existing matrix coefficients
+                while ([&]() {
+                    // check for length
+                    if (current_idx_ctr == local_matrix_nnz_) {
+                        return false;
+                    }
+
+                    if (rows_copy[current_idx_ctr] > interface_row) {
+                        return false;
+                    }
+
+                    // the copy rows are or equal
+                    // in that case we need to check if the
+                    // copy columns are lower
+                    if (rows_copy[current_idx_ctr] == interface_row &&
+                        cols_copy[current_idx_ctr] > interface_col) {
+                        return false;
+                    }
+                    return true;
+                }()) {
+                    // insert coeffs
+                    rows[total_ctr] = rows_copy[current_idx_ctr];
+                    cols[total_ctr] = cols_copy[current_idx_ctr];
+                    permute[total_ctr] = permute_copy[current_idx_ctr];
+                    current_idx_ctr++;
+                    total_ctr++;
                 }
-
-                if (rows_copy[current_idx_ctr] > interface_row) {
-                    return false;
-                }
-
-                // the copy rows are or equal
-                // in that case we need to check if the
-                // copy columns are lower
-                if (rows_copy[current_idx_ctr] == interface_row &&
-                    cols_copy[current_idx_ctr] > interface_col) {
-                    return false;
-                }
-                return true;
-            }()) {
-                // insert coeffs
-                rows[total_ctr] = rows_copy[current_idx_ctr];
-                cols[total_ctr] = cols_copy[current_idx_ctr];
-                permute[total_ctr] = permute_copy[current_idx_ctr];
-                current_idx_ctr++;
+                rows[total_ctr] = interface_row;
+                cols[total_ctr] = interface_col;
+                // store the original position of the contiguous interfaces
+                permute[total_ctr] = after_neighbours + nrows_ + interface_idx;
                 total_ctr++;
             }
-            rows[total_ctr] = interface_row;
-            cols[total_ctr] = interface_col;
-            // store the original position of the contiguous interfaces
-            permute[total_ctr] = after_neighbours + nrows_ + interface_idx;
-            total_ctr++;
-        }
 
-        // post insert if local interfaces were consumed but stuff remains in
-        // rows_copy and cols_copy
-        for (int i = total_ctr; i < local_matrix_w_interfaces_nnz_; i++) {
-            rows[i] = rows_copy[current_idx_ctr];
-            cols[i] = cols_copy[current_idx_ctr];
-            permute[i] = permute_copy[current_idx_ctr];
-            current_idx_ctr++;
+            // post insert if local interfaces were consumed but stuff remains
+            // in rows_copy and cols_copy
+            for (int i = total_ctr; i < local_matrix_w_interfaces_nnz_; i++) {
+                rows[i] = rows_copy[current_idx_ctr];
+                cols[i] = cols_copy[current_idx_ctr];
+                permute[i] = permute_copy[current_idx_ctr];
+                current_idx_ctr++;
+            }
         }
-    }
 
-    LOG_1(verbose_, "done init host matrix")
+        LOG_1(verbose_, "done init host matrix")
 }
 
 
@@ -630,48 +599,48 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs) const
 {
-    auto ref_exec = exec_.get_ref_exec();
-    auto upper = this->matrix().upper();
-    auto lower = this->matrix().lower();
-    auto diag = this->matrix().diag();
-    label diag_nnz = diag.size();
-    bool is_symmetric{this->matrix().symmetric()};
-
-    auto dense_vec = vec::create(
-        ref_exec,
-        gko::dim<2>{(gko::dim<2>::dimension_type)local_matrix_nnz_, 1},
-        gko::array<scalar>::view(ref_exec, local_matrix_nnz_,
-                                 local_coeffs_.get_data()),
-        1);
-
-    // TODO this does not work for Ell
-    const auto permute = local_sparsity_.ldu_mapping_.get_data();
-    auto dense = dense_vec->get_values();
-
-    if (local_interface_nnz_) {
-        auto couple_coeffs =
-            collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
-        if (is_symmetric) {
-            symmetric_update_w_interface(local_matrix_w_interfaces_nnz_,
-                                         diag_nnz, upper_nnz_, permute,
-                                         scaling_, diag.data(), upper.data(),
-                                         couple_coeffs.data(), dense);
-        } else {
-            non_symmetric_update_w_interface(
-                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute,
-                scaling_, diag.data(), upper.data(), lower.data(),
-                couple_coeffs.data(), dense);
-        }
-    } else {
-        if (is_symmetric) {
-            symmetric_update(local_matrix_nnz_, upper_nnz_, permute, scaling_,
-                             diag.data(), upper.data(), dense);
+        auto ref_exec = exec_.get_ref_exec();
+        auto upper = this->matrix().upper();
+        auto lower = this->matrix().lower();
+        auto diag = this->matrix().diag();
+        label diag_nnz = diag.size();
+        bool is_symmetric{this->matrix().symmetric()};
+
+        auto dense_vec = vec::create(
+            ref_exec,
+            gko::dim<2>{(gko::dim<2>::dimension_type)local_matrix_nnz_, 1},
+            gko::array<scalar>::view(ref_exec, local_matrix_nnz_,
+                                     local_coeffs_.get_data()),
+            1);
+
+        // TODO this does not work for Ell
+        const auto permute = local_sparsity_.ldu_mapping_.get_data();
+        auto dense = dense_vec->get_values();
+
+        if (local_interface_nnz_) {
+            auto couple_coeffs =
+                collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
+            if (is_symmetric) {
+                symmetric_update_w_interface(
+                    local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
+                    permute, scaling_, diag.data(), upper.data(),
+                    couple_coeffs.data(), dense);
+            } else {
+                non_symmetric_update_w_interface(
+                    local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
+                    permute, scaling_, diag.data(), upper.data(), lower.data(),
+                    couple_coeffs.data(), dense);
+            }
         } else {
-            non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                 scaling_, diag.data(), upper.data(),
-                                 lower.data(), dense);
+            if (is_symmetric) {
+                symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                 scaling_, diag.data(), upper.data(), dense);
+            } else {
+                non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                     scaling_, diag.data(), upper.data(),
+                                     lower.data(), dense);
+            }
         }
-    }
 }
 
 
@@ -680,30 +649,31 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs) const
 {
-    auto ref_exec = exec_.get_ref_exec();
-    auto interface_coeffs =
-        collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
-    auto permute = non_local_sparsity_.ldu_mapping_.get_data();
-
-    // copy interfaces
-    auto tmp_contiguous_iface =
-        gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
-    auto contiguous_iface = tmp_contiguous_iface.get_data();
-
-    interface_iterator<processorFvPatch>(
-        interfaces, [&](label &element_ctr, const label interface_size,
-                        const processorFvPatch &, const lduInterfaceField *) {
-            for (label cellI = 0; cellI < interface_size; cellI++) {
-                contiguous_iface[element_ctr] =
-                    -interface_coeffs[permute[element_ctr]];
-                element_ctr++;
-            }
-        });
+        auto ref_exec = exec_.get_ref_exec();
+        auto interface_coeffs =
+            collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
+        auto permute = non_local_sparsity_.ldu_mapping_.get_data();
+
+        // copy interfaces
+        auto tmp_contiguous_iface =
+            gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
+        auto contiguous_iface = tmp_contiguous_iface.get_data();
+
+        interface_iterator<processorFvPatch>(
+            interfaces,
+            [&](label &element_ctr, const label interface_size,
+                const processorFvPatch &, const lduInterfaceField *) {
+                for (label cellI = 0; cellI < interface_size; cellI++) {
+                    contiguous_iface[element_ctr] =
+                        -interface_coeffs[permute[element_ctr]];
+                    element_ctr++;
+                }
+            });
 
-    // copy to persistent
-    auto i_device_view = gko::array<scalar>::view(
-        ref_exec, non_local_matrix_nnz_, non_local_coeffs_.get_data());
-    i_device_view = tmp_contiguous_iface;
+        // copy to persistent
+        auto i_device_view = gko::array<scalar>::view(
+            ref_exec, non_local_matrix_nnz_, non_local_coeffs_.get_data());
+        i_device_view = tmp_contiguous_iface;
 }
 
 template HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(

From 2376f66bb6354da59b9a172bae2d9632126c8083 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 22 Nov 2023 11:15:03 +0100
Subject: [PATCH 123/125] format

---
 HostMatrix/HostMatrix.C | 458 ++++++++++++++++++++--------------------
 1 file changed, 228 insertions(+), 230 deletions(-)

diff --git a/HostMatrix/HostMatrix.C b/HostMatrix/HostMatrix.C
index e2c4d8d0b..3ae49a1ac 100644
--- a/HostMatrix/HostMatrix.C
+++ b/HostMatrix/HostMatrix.C
@@ -397,200 +397,199 @@ HostMatrixWrapper<MatrixType>::collect_cells_on_interface(
     label interface_ctr = 0;
     label uniqueIdCtr = 0;
     interface_iterator<processorLduInterface>(
-        interfaces,
-        [&](label, const label interface_size, const processorLduInterface &patch,
-            const lduInterfaceField *iface) {
-        const processorLduInterface &pldui =
-            refCast<const processorLduInterface>(iface->interface());
-        const auto &face_cells{iface->interface().faceCells()};
-        const label neighbProcNo = pldui.neighbProcNo();
-
-        word msg_2 = "receive face cells interface from proc " +
-                     std::to_string(neighbProcNo);
-        LOG_2(verbose_, msg_2)
-
-        auto otherSide_tmp = pldui.receive<label>(
-            Pstream::commsTypes::nonBlocking, interface_size);
-        LOG_2(verbose_, "receive face cells done")
-
-        for (label cellI = 0; cellI < interface_size; cellI++) {
-            auto global_row = global_row_index_.toGlobal(
-                neighbProcNo, otherSide_tmp()[cellI]);
-            auto search = unique_index.find(global_row);
-            // global_row is new and unique
-            label uniqueId = uniqueIdCtr;
-            if (search == unique_index.end()) {
-                uniqueId = uniqueIdCtr;
-                unique_index[global_row] = uniqueIdCtr;
-                uniqueIdCtr++;
-            } else {
-                uniqueId = unique_index[global_row];
+        interfaces, [&](label, const label interface_size,
+                        const processorLduInterface &patch,
+                        const lduInterfaceField *iface) {
+            const processorLduInterface &pldui =
+                refCast<const processorLduInterface>(iface->interface());
+            const auto &face_cells{iface->interface().faceCells()};
+            const label neighbProcNo = pldui.neighbProcNo();
+
+            word msg_2 = "receive face cells interface from proc " +
+                         std::to_string(neighbProcNo);
+            LOG_2(verbose_, msg_2)
+
+            auto otherSide_tmp = pldui.receive<label>(
+                Pstream::commsTypes::nonBlocking, interface_size);
+            LOG_2(verbose_, "receive face cells done")
+
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                auto global_row = global_row_index_.toGlobal(
+                    neighbProcNo, otherSide_tmp()[cellI]);
+                auto search = unique_index.find(global_row);
+                // global_row is new and unique
+                label uniqueId = uniqueIdCtr;
+                if (search == unique_index.end()) {
+                    uniqueId = uniqueIdCtr;
+                    unique_index[global_row] = uniqueIdCtr;
+                    uniqueIdCtr++;
+                } else {
+                    uniqueId = unique_index[global_row];
+                }
+                non_local_idxs.push_back(
+                    {interface_ctr, face_cells[cellI], uniqueId});
+                interface_ctr += 1;
             }
-            non_local_idxs.push_back(
-                {interface_ctr, face_cells[cellI], uniqueId});
-            interface_ctr += 1;
-        }});
+        });
 
-        word msg = "done collecting neighbouring processor cell id";
+    word msg = "done collecting neighbouring processor cell id";
 
-        LOG_2(verbose_, msg)
-        return non_local_idxs;
+    LOG_2(verbose_, msg)
+    return non_local_idxs;
 }
 
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_non_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
-        auto non_local_row_indices = collect_cells_on_interface(interfaces);
-        auto rows = non_local_sparsity_.row_idxs_.get_data();
-        auto cols = non_local_sparsity_.col_idxs_.get_data();
-        auto permute = non_local_sparsity_.ldu_mapping_.get_data();
-
-        // TODO check if sorting is actually needed since interfaces should
-        // be sorted already
-        std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
-                  [&](const auto &a, const auto &b) {
-                      auto [interface_idx_a, row_a, unique_col_a] = a;
-                      auto [interface_idx_b, row_b, unique_col_b] = b;
-                      return std::tie(row_a, interface_idx_a) <
-                             std::tie(row_b, interface_idx_b);
-                  });
+    auto non_local_row_indices = collect_cells_on_interface(interfaces);
+    auto rows = non_local_sparsity_.row_idxs_.get_data();
+    auto cols = non_local_sparsity_.col_idxs_.get_data();
+    auto permute = non_local_sparsity_.ldu_mapping_.get_data();
+
+    // TODO check if sorting is actually needed since interfaces should
+    // be sorted already
+    std::sort(non_local_row_indices.begin(), non_local_row_indices.end(),
+              [&](const auto &a, const auto &b) {
+                  auto [interface_idx_a, row_a, unique_col_a] = a;
+                  auto [interface_idx_b, row_b, unique_col_b] = b;
+                  return std::tie(row_a, interface_idx_a) <
+                         std::tie(row_b, interface_idx_b);
+              });
 
-        interface_iterator<processorFvPatch>(
-            interfaces,
-            [&](label &element_ctr, const label interface_size,
-                const processorFvPatch &, const lduInterfaceField *) {
-                for (label cellI = 0; cellI < interface_size; cellI++) {
-                    auto [interface_idx, row, unique_col] =
-                        non_local_row_indices[element_ctr];
-                    rows[element_ctr] = row;
-                    cols[element_ctr] = unique_col;
-                    permute[element_ctr] = interface_idx;
-                    element_ctr += 1;
-                }
-            });
+    interface_iterator<processorFvPatch>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const processorFvPatch &, const lduInterfaceField *) {
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                auto [interface_idx, row, unique_col] =
+                    non_local_row_indices[element_ctr];
+                rows[element_ctr] = row;
+                cols[element_ctr] = unique_col;
+                permute[element_ctr] = interface_idx;
+                element_ctr += 1;
+            }
+        });
 }
 
 template <class MatrixType>
 void HostMatrixWrapper<MatrixType>::init_local_sparsity_pattern(
     const lduInterfaceFieldPtrsList &interfaces) const
 {
-        LOG_1(verbose_, "start init host sparsity pattern")
-        bool is_symmetric{this->matrix().symmetric()};
+    LOG_1(verbose_, "start init host sparsity pattern")
+    bool is_symmetric{this->matrix().symmetric()};
+
+    auto lower_local = idx_array::view(
+        exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
+        const_cast<label *>(&this->matrix().lduAddr().lowerAddr()[0]));
+
+    auto upper_local = idx_array::view(
+        exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
+        const_cast<label *>(&this->matrix().lduAddr().upperAddr()[0]));
+
+    // row of upper, col of lower
+    const auto lower = lower_local.get_const_data();
+    // col of upper, row of lower
+    const auto upper = upper_local.get_const_data();
 
-        auto lower_local = idx_array::view(
-            exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
-            const_cast<label *>(&this->matrix().lduAddr().lowerAddr()[0]));
+    auto rows = local_sparsity_.row_idxs_.get_data();
+    auto cols = local_sparsity_.col_idxs_.get_data();
+    const auto permute = local_sparsity_.ldu_mapping_.get_data();
 
-        auto upper_local = idx_array::view(
-            exec_.get_ref_exec(), upper_nnz_ - local_interface_nnz_,
-            const_cast<label *>(&this->matrix().lduAddr().upperAddr()[0]));
+    // Scan through given rows and insert row and column indices into array
+    //
+    //  position after all local offdiagonal elements, needed for
+    //  permutation matrix
+    //
+    //  TODO in order to simplify when local interfaces exists set
+    //  local_sparsity to size of nrows_w_interfaces, if interfaces exist
+    //  local_sparsity is only valid till nrows_
+    label after_neighbours = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
+    init_local_sparsity(nrows_, upper_nnz_, is_symmetric, upper, lower, rows,
+                        cols, permute);
+
+    // if no local interfaces are present we are done here
+    // otherwise we need to add local interfaces in order
+    if (local_interface_nnz_) {
+        auto local_interfaces = collect_local_interface_indices(interfaces);
+        // // sort local interfaces to be in row major order
+        // // and keep original indices
+        std::sort(local_interfaces.begin(), local_interfaces.end(),
+                  [&](const auto &a, const auto &b) {
+                      auto [interface_idx_a, row_a, col_a] = a;
+                      auto [interface_idx_b, row_b, col_b] = b;
+                      return std::tie(row_a, col_a) < std::tie(row_b, col_b);
+                  });
 
-        // row of upper, col of lower
-        const auto lower = lower_local.get_const_data();
-        // col of upper, row of lower
-        const auto upper = upper_local.get_const_data();
+        // copy current rows and columns to tmp array
+        // TODO this has full length but is only valid till nrows_
+        auto rows_copy_a = gko::array<label>(
+            *local_sparsity_.row_idxs_.get_persistent_object().get());
+        auto cols_copy_a = gko::array<label>(
+            *local_sparsity_.col_idxs_.get_persistent_object().get());
+        auto permute_copy_a = gko::array<label>(
+            *local_sparsity_.ldu_mapping_.get_persistent_object().get());
+
+        auto rows_copy = rows_copy_a.get_data();
+        auto cols_copy = cols_copy_a.get_data();
+        auto permute_copy = permute_copy_a.get_data();
 
         auto rows = local_sparsity_.row_idxs_.get_data();
         auto cols = local_sparsity_.col_idxs_.get_data();
-        const auto permute = local_sparsity_.ldu_mapping_.get_data();
-
-        // Scan through given rows and insert row and column indices into array
-        //
-        //  position after all local offdiagonal elements, needed for
-        //  permutation matrix
-        //
-        //  TODO in order to simplify when local interfaces exists set
-        //  local_sparsity to size of nrows_w_interfaces, if interfaces exist
-        //  local_sparsity is only valid till nrows_
-        label after_neighbours = (is_symmetric) ? upper_nnz_ : 2 * upper_nnz_;
-        init_local_sparsity(nrows_, upper_nnz_, is_symmetric, upper, lower,
-                            rows, cols, permute);
-
-        // if no local interfaces are present we are done here
-        // otherwise we need to add local interfaces in order
-        if (local_interface_nnz_) {
-            auto local_interfaces = collect_local_interface_indices(interfaces);
-            // // sort local interfaces to be in row major order
-            // // and keep original indices
-            std::sort(local_interfaces.begin(), local_interfaces.end(),
-                      [&](const auto &a, const auto &b) {
-                          auto [interface_idx_a, row_a, col_a] = a;
-                          auto [interface_idx_b, row_b, col_b] = b;
-                          return std::tie(row_a, col_a) <
-                                 std::tie(row_b, col_b);
-                      });
-
-            // copy current rows and columns to tmp array
-            // TODO this has full length but is only valid till nrows_
-            auto rows_copy_a = gko::array<label>(
-                *local_sparsity_.row_idxs_.get_persistent_object().get());
-            auto cols_copy_a = gko::array<label>(
-                *local_sparsity_.col_idxs_.get_persistent_object().get());
-            auto permute_copy_a = gko::array<label>(
-                *local_sparsity_.ldu_mapping_.get_persistent_object().get());
-
-            auto rows_copy = rows_copy_a.get_data();
-            auto cols_copy = cols_copy_a.get_data();
-            auto permute_copy = permute_copy_a.get_data();
-
-            auto rows = local_sparsity_.row_idxs_.get_data();
-            auto cols = local_sparsity_.col_idxs_.get_data();
-            auto permute = local_sparsity_.ldu_mapping_.get_data();
-
-            label current_idx_ctr = 0;
-            label total_ctr = 0;
-            // iterate local interfaces i
-            // insert all coeffs for row < interface[i].row
-            // find local_interface with lowes row, column
-            for (auto const &interface : local_interfaces) {
-                auto [interface_idx, interface_row, interface_col] = interface;
-
-                // copy from existing matrix coefficients
-                while ([&]() {
-                    // check for length
-                    if (current_idx_ctr == local_matrix_nnz_) {
-                        return false;
-                    }
-
-                    if (rows_copy[current_idx_ctr] > interface_row) {
-                        return false;
-                    }
-
-                    // the copy rows are or equal
-                    // in that case we need to check if the
-                    // copy columns are lower
-                    if (rows_copy[current_idx_ctr] == interface_row &&
-                        cols_copy[current_idx_ctr] > interface_col) {
-                        return false;
-                    }
-                    return true;
-                }()) {
-                    // insert coeffs
-                    rows[total_ctr] = rows_copy[current_idx_ctr];
-                    cols[total_ctr] = cols_copy[current_idx_ctr];
-                    permute[total_ctr] = permute_copy[current_idx_ctr];
-                    current_idx_ctr++;
-                    total_ctr++;
+        auto permute = local_sparsity_.ldu_mapping_.get_data();
+
+        label current_idx_ctr = 0;
+        label total_ctr = 0;
+        // iterate local interfaces i
+        // insert all coeffs for row < interface[i].row
+        // find local_interface with lowes row, column
+        for (auto const &interface : local_interfaces) {
+            auto [interface_idx, interface_row, interface_col] = interface;
+
+            // copy from existing matrix coefficients
+            while ([&]() {
+                // check for length
+                if (current_idx_ctr == local_matrix_nnz_) {
+                    return false;
                 }
-                rows[total_ctr] = interface_row;
-                cols[total_ctr] = interface_col;
-                // store the original position of the contiguous interfaces
-                permute[total_ctr] = after_neighbours + nrows_ + interface_idx;
-                total_ctr++;
-            }
 
-            // post insert if local interfaces were consumed but stuff remains
-            // in rows_copy and cols_copy
-            for (int i = total_ctr; i < local_matrix_w_interfaces_nnz_; i++) {
-                rows[i] = rows_copy[current_idx_ctr];
-                cols[i] = cols_copy[current_idx_ctr];
-                permute[i] = permute_copy[current_idx_ctr];
+                if (rows_copy[current_idx_ctr] > interface_row) {
+                    return false;
+                }
+
+                // the copy rows are or equal
+                // in that case we need to check if the
+                // copy columns are lower
+                if (rows_copy[current_idx_ctr] == interface_row &&
+                    cols_copy[current_idx_ctr] > interface_col) {
+                    return false;
+                }
+                return true;
+            }()) {
+                // insert coeffs
+                rows[total_ctr] = rows_copy[current_idx_ctr];
+                cols[total_ctr] = cols_copy[current_idx_ctr];
+                permute[total_ctr] = permute_copy[current_idx_ctr];
                 current_idx_ctr++;
+                total_ctr++;
             }
+            rows[total_ctr] = interface_row;
+            cols[total_ctr] = interface_col;
+            // store the original position of the contiguous interfaces
+            permute[total_ctr] = after_neighbours + nrows_ + interface_idx;
+            total_ctr++;
         }
 
-        LOG_1(verbose_, "done init host matrix")
+        // post insert if local interfaces were consumed but stuff remains
+        // in rows_copy and cols_copy
+        for (int i = total_ctr; i < local_matrix_w_interfaces_nnz_; i++) {
+            rows[i] = rows_copy[current_idx_ctr];
+            cols[i] = cols_copy[current_idx_ctr];
+            permute[i] = permute_copy[current_idx_ctr];
+            current_idx_ctr++;
+        }
+    }
+
+    LOG_1(verbose_, "done init host matrix")
 }
 
 
@@ -599,48 +598,48 @@ void HostMatrixWrapper<MatrixType>::update_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs) const
 {
-        auto ref_exec = exec_.get_ref_exec();
-        auto upper = this->matrix().upper();
-        auto lower = this->matrix().lower();
-        auto diag = this->matrix().diag();
-        label diag_nnz = diag.size();
-        bool is_symmetric{this->matrix().symmetric()};
-
-        auto dense_vec = vec::create(
-            ref_exec,
-            gko::dim<2>{(gko::dim<2>::dimension_type)local_matrix_nnz_, 1},
-            gko::array<scalar>::view(ref_exec, local_matrix_nnz_,
-                                     local_coeffs_.get_data()),
-            1);
-
-        // TODO this does not work for Ell
-        const auto permute = local_sparsity_.ldu_mapping_.get_data();
-        auto dense = dense_vec->get_values();
-
-        if (local_interface_nnz_) {
-            auto couple_coeffs =
-                collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
-            if (is_symmetric) {
-                symmetric_update_w_interface(
-                    local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
-                    permute, scaling_, diag.data(), upper.data(),
-                    couple_coeffs.data(), dense);
-            } else {
-                non_symmetric_update_w_interface(
-                    local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_,
-                    permute, scaling_, diag.data(), upper.data(), lower.data(),
-                    couple_coeffs.data(), dense);
-            }
+    auto ref_exec = exec_.get_ref_exec();
+    auto upper = this->matrix().upper();
+    auto lower = this->matrix().lower();
+    auto diag = this->matrix().diag();
+    label diag_nnz = diag.size();
+    bool is_symmetric{this->matrix().symmetric()};
+
+    auto dense_vec = vec::create(
+        ref_exec,
+        gko::dim<2>{(gko::dim<2>::dimension_type)local_matrix_nnz_, 1},
+        gko::array<scalar>::view(ref_exec, local_matrix_nnz_,
+                                 local_coeffs_.get_data()),
+        1);
+
+    // TODO this does not work for Ell
+    const auto permute = local_sparsity_.ldu_mapping_.get_data();
+    auto dense = dense_vec->get_values();
+
+    if (local_interface_nnz_) {
+        auto couple_coeffs =
+            collect_interface_coeffs(interfaces, interfaceBouCoeffs, true);
+        if (is_symmetric) {
+            symmetric_update_w_interface(local_matrix_w_interfaces_nnz_,
+                                         diag_nnz, upper_nnz_, permute,
+                                         scaling_, diag.data(), upper.data(),
+                                         couple_coeffs.data(), dense);
         } else {
-            if (is_symmetric) {
-                symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                 scaling_, diag.data(), upper.data(), dense);
-            } else {
-                non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
-                                     scaling_, diag.data(), upper.data(),
-                                     lower.data(), dense);
-            }
+            non_symmetric_update_w_interface(
+                local_matrix_w_interfaces_nnz_, diag_nnz, upper_nnz_, permute,
+                scaling_, diag.data(), upper.data(), lower.data(),
+                couple_coeffs.data(), dense);
         }
+    } else {
+        if (is_symmetric) {
+            symmetric_update(local_matrix_nnz_, upper_nnz_, permute, scaling_,
+                             diag.data(), upper.data(), dense);
+        } else {
+            non_symmetric_update(local_matrix_nnz_, upper_nnz_, permute,
+                                 scaling_, diag.data(), upper.data(),
+                                 lower.data(), dense);
+        }
+    }
 }
 
 
@@ -649,31 +648,30 @@ void HostMatrixWrapper<MatrixType>::update_non_local_matrix_data(
     const lduInterfaceFieldPtrsList &interfaces,
     const FieldField<Field, scalar> &interfaceBouCoeffs) const
 {
-        auto ref_exec = exec_.get_ref_exec();
-        auto interface_coeffs =
-            collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
-        auto permute = non_local_sparsity_.ldu_mapping_.get_data();
-
-        // copy interfaces
-        auto tmp_contiguous_iface =
-            gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
-        auto contiguous_iface = tmp_contiguous_iface.get_data();
-
-        interface_iterator<processorFvPatch>(
-            interfaces,
-            [&](label &element_ctr, const label interface_size,
-                const processorFvPatch &, const lduInterfaceField *) {
-                for (label cellI = 0; cellI < interface_size; cellI++) {
-                    contiguous_iface[element_ctr] =
-                        -interface_coeffs[permute[element_ctr]];
-                    element_ctr++;
-                }
-            });
+    auto ref_exec = exec_.get_ref_exec();
+    auto interface_coeffs =
+        collect_interface_coeffs(interfaces, interfaceBouCoeffs, false);
+    auto permute = non_local_sparsity_.ldu_mapping_.get_data();
+
+    // copy interfaces
+    auto tmp_contiguous_iface =
+        gko::array<scalar>(ref_exec, non_local_matrix_nnz_);
+    auto contiguous_iface = tmp_contiguous_iface.get_data();
+
+    interface_iterator<processorFvPatch>(
+        interfaces, [&](label &element_ctr, const label interface_size,
+                        const processorFvPatch &, const lduInterfaceField *) {
+            for (label cellI = 0; cellI < interface_size; cellI++) {
+                contiguous_iface[element_ctr] =
+                    -interface_coeffs[permute[element_ctr]];
+                element_ctr++;
+            }
+        });
 
-        // copy to persistent
-        auto i_device_view = gko::array<scalar>::view(
-            ref_exec, non_local_matrix_nnz_, non_local_coeffs_.get_data());
-        i_device_view = tmp_contiguous_iface;
+    // copy to persistent
+    auto i_device_view = gko::array<scalar>::view(
+        ref_exec, non_local_matrix_nnz_, non_local_coeffs_.get_data());
+    i_device_view = tmp_contiguous_iface;
 }
 
 template HostMatrixWrapper<lduMatrix>::HostMatrixWrapper(

From c64200063a7423690fe6844b897d0b515b07ec3f Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 22 Nov 2023 11:21:05 +0100
Subject: [PATCH 124/125] check for TODO DONT MERGE instead of TODO FIXME

---
 .github/workflows/format-check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml
index 3d2519756..6e520486c 100644
--- a/.github/workflows/format-check.yml
+++ b/.github/workflows/format-check.yml
@@ -16,6 +16,6 @@ jobs:
         fallback-style: 'Mozilla' # optional
     - name: check for todo fixme note
       run: |
-        NTODOS="$(grep -r 'TODO FIXME' | wc -l)"
+        NTODOS="$(grep -r 'TODO DONT MERGE' | wc -l)"
         echo "Found $NTODOS TODO FIXME notes" 
         ! grep -q -r "TODO FIXME"  

From f4d9c412f565c51dd8723cf37cd7e6a1ec180f95 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Wed, 22 Nov 2023 11:27:19 +0100
Subject: [PATCH 125/125] exclude .github folder

---
 .github/workflows/format-check.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml
index 6e520486c..80419bf13 100644
--- a/.github/workflows/format-check.yml
+++ b/.github/workflows/format-check.yml
@@ -17,5 +17,5 @@ jobs:
     - name: check for todo fixme note
       run: |
         NTODOS="$(grep -r 'TODO DONT MERGE' | wc -l)"
-        echo "Found $NTODOS TODO FIXME notes" 
-        ! grep -q -r "TODO FIXME"  
+        echo "Found $NTODOS TODO DONT MERGE notes" 
+        ! grep -q -r "TODO DONT MERGE" --exclude-dir=.github