diff --git a/README.md b/README.md
index 50fa430a..78b5f30d 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,7 @@ adaptMinIter | true | based on the previous solution set minIter to be relaxatio
 relaxationFactor | 0.8 | use relaxationFactor*previousIters as new minIters
 scaling | 1.0 | Scale the complete system by the scaling factor
 forceHostBuffer  | false | whether to copy to host before MPI calls
+splitComm  | true | whether to split communicator for the host and device side
 export | false | write the complete system (matrix and rhs) to disk as .mtx file using controlDict/writeControl
 writeGlobal | false | convert all indices to global indices when exporting .mtx files
 
diff --git a/include/OGL/CommunicationPattern.hpp b/include/OGL/CommunicationPattern.hpp
index 5d622669..cdc37af6 100644
--- a/include/OGL/CommunicationPattern.hpp
+++ b/include/OGL/CommunicationPattern.hpp
@@ -117,6 +117,9 @@ std::vector<label> gather_labels_to_owner(const ExecutorHandler &exec_handler,
                                           const label *send_buffer,
                                           label send_size, label offset = 0);
 
+/* @class Struct to store communication related data
+**
+*/
 struct CommunicationPattern {
     using comm_size_type = label;
 
@@ -149,12 +152,6 @@ struct CommunicationPattern {
         }
     }
 
-    const gko::experimental::mpi::communicator &get_comm() const
-    {
-        return *exec_handler.get_communicator().get();
-    }
-
-
     /* @brief concatenate all separate send idxs arrays into one contiguous
      * array
      */
diff --git a/include/OGL/DevicePersistent/ExecutorHandler.hpp b/include/OGL/DevicePersistent/ExecutorHandler.hpp
index 432740df..376755e2 100644
--- a/include/OGL/DevicePersistent/ExecutorHandler.hpp
+++ b/include/OGL/DevicePersistent/ExecutorHandler.hpp
@@ -13,10 +13,70 @@
 
 namespace Foam {
 
-struct ExecutorInitFunctor {
-    mutable std::shared_ptr<gko::experimental::mpi::communicator> comm_;
 
-    const label device_id_;
+struct DeviceIdHandler {
+    // ratio of inactive to active ranks on the GPU
+    label ranks_per_gpu;
+
+    /*
+     * @param gpus_per_rank ratio between active host ranks and active ranks
+     * gpus on the node i.e. if gpus_per_rank == 1 all host ranks are active on
+     * the gpu, which might lead to oversubscription gpus_per_rank == 2 only
+     * every second rank will be active ...
+     */
+    DeviceIdHandler(label ranks_per_gpu_in) : ranks_per_gpu(ranks_per_gpu_in)
+    {
+        bool par_run = Pstream::parRun();
+        if (!par_run) {
+            FatalErrorInFunction << "Only parallel runs are supported for OGL"
+                                 << exit(FatalError);
+        }
+    }
+
+    /* @brief compute the local device id
+     *
+     * @param num_devices_per_node number of devices per node
+     * @returns
+     */
+    label compute_device_id(label num_devices_per_node) const
+    {
+        // if zero devices present device id is always zero
+        if (num_devices_per_node == 0) {
+            return 0;
+        }
+        label global_rank = Pstream::myProcNo();
+        // clang-format off
+        /* example: machine with 4 cpu cores per node 2 accelerators per node, on node 2
+        * global ranks [0, 1, 2, 3 | 4, 5, 6, 7]
+        * global device id w/o repart  [0, 0, 1, 1 | 2, 2, 3, 3]
+        * global device id w repart  [0, x, 1, x | 2, x, 3, x] x-inactive
+        * local device_id w/o repart  [0, 1, 0, 1 | 0, 1, 0, 1]
+        * local device_id w repart  [0, x, 1, x | 0, x, 1, x] x-inactive
+        */
+        // clang-format on
+
+        // global_id rpg = 1: [0, 1, 2, 3 | 4, 5, 6, 7]
+        // global_id rpg = 2: [0, 0, 1, 1 | 2, 2, 3, 3]
+        label device_global_id = global_rank / ranks_per_gpu;
+
+        // compute local round robin id
+        // mod of global_id / num_devices_per_node
+        return device_global_id % num_devices_per_node;
+    }
+
+    /* @brief compute the group id for the split communicator
+     */
+    label compute_group() const
+    {
+        label rank = Pstream::myProcNo();
+        label owner_rank = rank - (rank % ranks_per_gpu);
+        bool is_owner = owner_rank == rank;
+        return (is_owner) ? 0 : 1;
+    }
+};
+
+struct ExecutorInitFunctor {
+    const DeviceIdHandler device_id_handler_;
 
     const word executor_name_;
 
@@ -24,15 +84,10 @@ struct ExecutorInitFunctor {
 
     const label verbose_;
 
-    ExecutorInitFunctor(bool par_run, const word executor_name,
-                        const word field_name, const label verbose,
-                        const label gpus_per_rank,
-                        const bool force_host_buffer = false)
-        : comm_((par_run)  // TODO make this DRY
-                    ? std::make_shared<gko::experimental::mpi::communicator>(
-                          MPI_COMM_WORLD, force_host_buffer)
-                    : NULL),
-          device_id_((par_run) ? comm_->rank() / gpus_per_rank : 0),
+    ExecutorInitFunctor(const word executor_name, const word field_name,
+                        const label verbose,
+                        const DeviceIdHandler device_id_handler)
+        : device_id_handler_(device_id_handler),
           executor_name_(executor_name),
           field_name_(field_name),
           verbose_(verbose)
@@ -47,6 +102,14 @@ struct ExecutorInitFunctor {
     {
         auto host_exec = gko::share(gko::ReferenceExecutor::create());
 
+        auto msg = [](auto exec, auto id) {
+            std::string s;
+            s += std::string("Create ") + std::string(exec) +
+                 std::string(" executor on device ") + std::to_string(id) +
+                 std::string(" on rank ") + std::to_string(Pstream::myProcNo());
+            return s;
+        };
+
         if (executor_name_ == "cuda") {
             if (version.cuda_version.tag == not_compiled_tag) {
                 FatalErrorInFunction
@@ -54,10 +117,9 @@ struct ExecutorInitFunctor {
                        "with CUDA backend enabled."
                     << abort(FatalError);
             }
-            label id = device_id_ % gko::CudaExecutor::get_num_devices();
-            word msg = "Create CUDA executor on device " + std::to_string(id) +
-                       " on rank " + std::to_string(comm_->rank());
-            LOG_0(verbose_, msg)
+            label id = device_id_handler_.compute_device_id(
+                gko::CudaExecutor::get_num_devices());
+            LOG_0(verbose_, msg(executor_name_, id))
             return gko::share(gko::CudaExecutor::create(id, host_exec));
         }
         if (executor_name_ == "sycl" || executor_name_ == "dpcpp") {
@@ -67,21 +129,10 @@ struct ExecutorInitFunctor {
                        "with SYCL backend enabled."
                     << abort(FatalError);
             }
-
-            if (executor_name_ == "dpcpp") {
-                Info << "Warning: the executor name dpcpp is deprecated.\n"
-                     << "Use sycl as the executor name for the same executor "
-                        "instead."
-                     << endl;
-            }
-
-            auto devices = gko::DpcppExecutor::get_num_devices("gpu");
-            if (devices == 0) {
-                return gko::share(gko::DpcppExecutor::create(0, host_exec));
-            } else {
-                return gko::share(gko::DpcppExecutor::create(
-                    device_id_ % devices, host_exec));
-            }
+            label id = device_id_handler_.compute_device_id(
+                gko::DpcppExecutor::get_num_devices("gpu"));
+            LOG_0(verbose_, msg(executor_name_, id))
+            return gko::share(gko::DpcppExecutor::create(id, host_exec));
         }
         if (executor_name_ == "hip") {
             if (version.hip_version.tag == not_compiled_tag) {
@@ -90,10 +141,9 @@ struct ExecutorInitFunctor {
                        "with HIP backend enabled."
                     << abort(FatalError);
             }
-            label id = device_id_ % gko::HipExecutor::get_num_devices();
-            word msg = "Create HIP executor on device " + std::to_string(id) +
-                       " on rank " + std::to_string(comm_->rank());
-            LOG_0(verbose_, msg)
+            label id = device_id_handler_.compute_device_id(
+                gko::HipExecutor::get_num_devices());
+            LOG_0(verbose_, msg(executor_name_, id))
             auto ret = gko::share(gko::HipExecutor::create(id, host_exec));
             return ret;
         }
@@ -125,7 +175,11 @@ class ExecutorHandler
 
     const bool non_orig_device_comm_;
 
-    const bool par_run_;
+    const bool split_comm_;
+
+    mutable std::shared_ptr<gko::experimental::mpi::communicator> host_comm_;
+
+    const bool host_rank_;
 
     mutable std::shared_ptr<gko::experimental::mpi::communicator> device_comm_;
 
@@ -133,29 +187,40 @@ class ExecutorHandler
 
 public:
     ExecutorHandler(const objectRegistry &db, const dictionary &solverControls,
-                    const word field_name, bool par_run = Pstream::parRun())
+                    const word field_name, DeviceIdHandler device_id_handler
+		    )
         : PersistentBase<gko::Executor, ExecutorInitFunctor>(
               solverControls.lookupOrDefault("executor", word("reference")) +
                   +"_" + field_name,
               db,
               ExecutorInitFunctor(
-                  par_run,
                   solverControls.lookupOrDefault("executor", word("reference")),
                   field_name,
                   solverControls.lookupOrDefault("verbose", label(0)),
-                  solverControls.lookupOrDefault("ranksPerGPU", label(1)),
-                  solverControls.lookupOrDefault("forceHostBuffer", false)),
+                  device_id_handler
+                  ),
               true, 0),
           gko_force_host_buffer_(
               solverControls.lookupOrDefault("forceHostBuffer", false)),
           non_orig_device_comm_(
               solverControls.lookupOrDefault("MPIxRankOffload", false)),
-          par_run_(par_run),
+          split_comm_(
+              solverControls.lookupOrDefault("splitMPIComm", true)),
+          host_comm_(std::make_shared<gko::experimental::mpi::communicator>(
+                        MPI_COMM_WORLD, gko_force_host_buffer_)),
+	  host_rank_(host_comm_->rank()),
           device_comm_(
-              (par_run_)
-                  ? std::make_shared<gko::experimental::mpi::communicator>(
-                        MPI_COMM_WORLD, gko_force_host_buffer_)
-                  : NULL),
+              (split_comm_)
+                  ? [this, device_id_handler](){
+		    label group = device_id_handler.compute_group();
+		    MPI_Comm gko_comm;
+            label host_rank =0;
+		    MPI_Comm_split(MPI_COMM_WORLD, group, host_rank, &gko_comm);
+
+		    return std::make_shared<gko::experimental::mpi::communicator>(
+			gko_comm, gko_force_host_buffer_);
+		  }()
+                  : host_comm_),
           device_executor_name_(
               solverControls.lookupOrDefault("executor", word("reference")))
     {}
@@ -168,10 +233,7 @@ class ExecutorHandler
     /* whether the mpi allows to send data directly to a remote rank
      * via pair-wise communication
      * */
-    bool get_non_orig_device_comm() const
-    {
-        return non_orig_device_comm_;
-    }
+    bool get_non_orig_device_comm() const { return non_orig_device_comm_; }
 
     const std::shared_ptr<gko::Executor> get_device_exec() const
     {
@@ -185,26 +247,23 @@ class ExecutorHandler
 
     word get_exec_name() const { return device_executor_name_; }
 
-    std::shared_ptr<gko::experimental::mpi::communicator>
-    get_gko_mpi_host_comm() const
-    {
-        return std::make_shared<gko::experimental::mpi::communicator>(
-            MPI_COMM_WORLD, gko_force_host_buffer_);
-    }
-
     std::shared_ptr<const gko::experimental::mpi::communicator>
-    get_gko_mpi_device_comm() const
+    get_device_comm() const
     {
         return this->device_comm_;
     }
 
-    std::shared_ptr<const gko::experimental::mpi::communicator>
-    get_communicator() const
+    std::shared_ptr<const gko::experimental::mpi::communicator> get_host_comm()
+        const
     {
-        return this->device_comm_;
+        return this->host_comm_;
     }
 
-    label get_rank() const { return get_communicator()->rank(); };
+    bool get_split_comm() const { return split_comm_; };
+
+    label get_host_rank() const { return get_host_comm()->rank(); };
+
+    label get_device_rank() const { return get_device_comm()->rank(); };
 };
 
 using PersistentExecutor = ExecutorHandler;
diff --git a/include/OGL/DevicePersistent/Vector.hpp b/include/OGL/DevicePersistent/Vector.hpp
index c66d4a65..4d75f21b 100644
--- a/include/OGL/DevicePersistent/Vector.hpp
+++ b/include/OGL/DevicePersistent/Vector.hpp
@@ -53,7 +53,7 @@ struct VectorInitFunctor {
     {
         auto ref_exec = exec_.get_ref_exec();
         auto exec = exec_.get_device_exec();
-        auto comm = exec_.get_communicator();
+        auto comm = exec_.get_host_comm();
         auto repartitioner = dist_matrix_->get_repartitioner();
         auto host_size = repartitioner->get_orig_size();
         auto repart_size = repartitioner->get_repart_size();
@@ -78,7 +78,6 @@ struct VectorInitFunctor {
     {
         auto exec = exec_.get_device_exec();
         auto ref_exec = exec_.get_ref_exec();
-        auto comm = exec_.get_communicator();
         auto repartitioner = dist_matrix_->get_repartitioner();
         auto host_size = repartitioner->get_orig_size();
         auto repart_size = repartitioner->get_repart_size();
@@ -100,8 +99,9 @@ struct VectorInitFunctor {
         communicate_values(exec_, comm_pattern, host_view.get_const_data(),
                            values.get_data());
 
+        auto device_comm = exec_.get_device_comm();
         auto ret = gko::share(dist_vec::create(
-            exec, *comm.get(),
+            exec, *device_comm.get(),
             vec::create(
                 exec, gko::dim<2>{static_cast<gko::size_type>(repart_size), 1},
                 values, 1)));
@@ -174,9 +174,9 @@ class PersistentVector
     void copy_back()
     {
         auto exec = exec_.get_device_exec();
-        auto rank = exec_.get_rank();
+        auto rank = exec_.get_host_rank();
         auto ref_exec = exec_.get_ref_exec();
-        auto comm = exec_.get_communicator();
+        auto comm = exec_.get_host_comm();
         bool host_buffer = !exec_.get_non_orig_device_comm();
 
         auto repartitioner = dist_matrix_->get_repartitioner();
diff --git a/include/OGL/Repartitioner.hpp b/include/OGL/Repartitioner.hpp
index d97b6a82..047ec386 100644
--- a/include/OGL/Repartitioner.hpp
+++ b/include/OGL/Repartitioner.hpp
@@ -63,11 +63,12 @@ class Repartitioner {
                                                           exec_handler)),
           ranks_per_gpu_(ranks_per_gpu),
           verbose_(verbose),
+          // TODO we build the partition from the local size before
+          // repartitioning thus we need the host comm to compute correct sizes
           orig_partition_(gko::share(
               gko::experimental::distributed::build_partition_from_local_size<
                   label, label>(exec_handler.get_ref_exec(),
-                                *exec_handler.get_communicator().get(),
-                                size))){};
+                                *exec_handler.get_host_comm().get(), size))){};
 
     /* returns the owner rank for a given rank */
     label get_owner_rank(label rank) const
@@ -78,14 +79,14 @@ class Repartitioner {
     /* returns the owner rank for a given rank */
     label get_owner_rank(const ExecutorHandler &exec_handler) const
     {
-        return get_owner_rank(exec_handler.get_rank());
+        return get_owner_rank(exec_handler.get_host_rank());
     };
 
     /* returns if current rank is an owner  */
     bool is_owner(const ExecutorHandler &exec_handler) const
     {
-        return exec_handler.get_rank() ==
-               get_owner_rank(exec_handler.get_rank());
+        return exec_handler.get_host_rank() ==
+               get_owner_rank(exec_handler.get_host_rank());
     };
 
     /* @brief check if the given rank gets local after repartitioning
diff --git a/include/OGL/lduLduBase.hpp b/include/OGL/lduLduBase.hpp
index 1cf423e0..aaf6f2b4 100644
--- a/include/OGL/lduLduBase.hpp
+++ b/include/OGL/lduLduBase.hpp
@@ -38,6 +38,8 @@ class OGL_Info {
 
     const bool force_host_buffer_;
 
+    const bool split_mpi_comm_;
+
     const bool fused_;
 
     const label ranks_per_gpu_;
@@ -53,6 +55,8 @@ class OGL_Info {
           force_host_buffer_(
               solverControls.lookupOrDefault("forceHostBuffer", false)),
           fused_(solverControls.lookupOrDefault<Switch>("fuse", true)),
+          split_mpi_comm_(
+              solverControls.lookupOrDefault<Switch>("splitMPIComm", true)),
           ranks_per_gpu_(
               solverControls.lookupOrDefault<label>("ranksPerGPU", 1)),
           matrix_format_(
@@ -93,6 +97,8 @@ class OGL_Info {
             std::to_string(fused_) +
             std::string("\n\tForces host buffer based communication: ") +
             std::to_string(force_host_buffer_) +
+            std::string("\n\tSplits MTI communicator: ") +
+            std::to_string(split_mpi_comm_) +
             std::string("\n\tCPU ranks per GPU: ") +
             std::to_string(ranks_per_gpu_) +
             std::string("\n\tMatrix format: ") + matrix_format_ +
@@ -143,6 +149,10 @@ class lduLduBase : public OGL_Info,
 
     const word field_name_;
 
+    const label ranks_per_gpu_;
+
+    const DeviceIdHandler device_id_handler_;
+
     const ExecutorHandler exec_handler_;
 
     const std::shared_ptr<HostMatrixWrapper> host_matrix_wrapper_;
@@ -165,7 +175,10 @@ class lduLduBase : public OGL_Info,
           solver_controls_(solverControls),
           verbose_(solverControls.lookupOrDefault<label>("verbose", 0)),
           field_name_(fieldName),
-          exec_handler_{db_, solver_controls_, fieldName},
+          ranks_per_gpu_(
+              solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1)),
+          device_id_handler_(ranks_per_gpu_),
+          exec_handler_{db_, solver_controls_, fieldName, device_id_handler_},
           host_matrix_wrapper_{std::make_shared<HostMatrixWrapper>(
               exec_handler_, db_, matrix.diag().size(), matrix.upper().size(),
               matrix.symmetric(), matrix.diag().begin(), matrix.upper().begin(),
@@ -202,12 +215,10 @@ class lduLduBase : public OGL_Info,
                                            const scalarField &source,
                                            solverPerformance &solverPerf) const
     {
-        label ranks_per_gpu =
-            solver_controls_.lookupOrDefault<label>("ranksPerGPU", 1);
         bool fused = solver_controls_.lookupOrDefault<Switch>("fuse", true);
 
         auto repartitioner = std::make_shared<Repartitioner>(
-            host_matrix_wrapper_->get_local_nrows(), ranks_per_gpu, verbose_,
+            host_matrix_wrapper_->get_local_nrows(), ranks_per_gpu_, verbose_,
             exec_handler_);
 
         PersistentDistributedMatrix dist_A{db_,
@@ -282,8 +293,18 @@ class lduLduBase : public OGL_Info,
                             auto solver = solver_gen->generate(dist_A_v);)
         LOG_1(verbose_, "done create solver")
 
-        TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
-                            solver->apply(dist_b_v, dist_x_v);)
+        // solve only on active rank
+        bool active = repartitioner->get_repart_size() != 0;
+        label delta_t_solve_ = 0;
+        bool split_mpi_comm =
+            solver_controls_.lookupOrDefault<Switch>("splitMPIComm", true);
+
+        if (!active && split_mpi_comm) {
+        } else {
+            TIME_WITH_FIELDNAME(verbose_, solve, this->fieldName(),
+                                solver->apply(dist_b_v, dist_x_v);)
+            delta_t_solve_ = delta_t_solve;
+        }
 
         TIME_WITH_FIELDNAME(verbose_, copy_x_back, this->fieldName(),
                             dist_x.copy_back();)
@@ -295,12 +316,12 @@ class lduLduBase : public OGL_Info,
         solverPerf.finalResidual() = this->get_res_norm();
         solverPerf.nIterations() = this->get_number_of_iterations();
         this->store_number_of_iterations();
-        auto time_for_res_norm_eval = this->get_res_norm_time();
+        auto time_for_res_norm_eval = this->get_res_norm_time() + SMALL;
         auto time_per_iter =
-            delta_t_solve / max(this->get_number_of_iterations(), 1);
+            delta_t_solve_ / max(this->get_number_of_iterations(), 1);
         scalar prev_rel_res_cost = time_per_iter / time_for_res_norm_eval;
-        exec_handler_.get_gko_mpi_host_comm()->broadcast(
-            exec_handler_.get_ref_exec(), &prev_rel_res_cost, 1, 0);
+        exec_handler_.get_host_comm()->broadcast(exec_handler_.get_ref_exec(),
+                                                 &prev_rel_res_cost, 1, 0);
         this->set_prev_rel_res_cost(prev_rel_res_cost);
         size_t dofs = repartitioner->get_orig_partition()->get_size();
         auto time_per_iter_and_dof = time_per_iter * 1000.0 / dofs;
@@ -328,16 +349,10 @@ class lduLduBase : public OGL_Info,
                 exec_handler_.get_exec_name() + typeName,
             field_name_);
 
-        // Solve system
-        if (Pstream::parRun()) {
-            TIME_WITH_FIELDNAME(
-                verbose_, solve_multi_gpu, field_name_,
-                auto res = solve_multi_gpu_impl(psi, source, solverPerf);)
-            return res;
-        } else {
-            FatalErrorInFunction << "Only parallel runs are supported for OGL"
-                                 << exit(FatalError);
-        }
+        TIME_WITH_FIELDNAME(
+            verbose_, solve_multi_gpu, field_name_,
+            auto res = solve_multi_gpu_impl(psi, source, solverPerf);)
+        return res;
 
         return solverPerf;
     };
diff --git a/src/CommunicationPattern.cpp b/src/CommunicationPattern.cpp
index 531501d1..8605998e 100644
--- a/src/CommunicationPattern.cpp
+++ b/src/CommunicationPattern.cpp
@@ -22,7 +22,7 @@ AllToAllPattern compute_scatter_from_owner_counts(
     const ExecutorHandler &exec_handler, label ranks_per_owner, label size)
 {
     auto exec = exec_handler.get_device_exec();
-    auto comm = *exec_handler.get_communicator().get();
+    auto comm = *exec_handler.get_host_comm().get();
 
     label total_ranks{comm.size()};
     label rank{comm.rank()};
@@ -68,7 +68,7 @@ AllToAllPattern compute_gather_to_owner_counts(
     label total_size, label padding_before, label padding_after)
 {
     auto exec = exec_handler.get_device_exec();
-    auto comm = *exec_handler.get_communicator().get();
+    auto comm = *exec_handler.get_host_comm().get();
 
     OGL_ASSERT_EQ(total_size, size + padding_before + padding_after);
     label total_ranks{comm.size()};
@@ -134,9 +134,9 @@ void communicate_values(const ExecutorHandler &exec_handler,
                         const scalar *send_buffer, scalar *recv_buffer)
 {
     auto exec = exec_handler.get_device_exec();
-    auto comm = *exec_handler.get_communicator().get();
-    label rank = exec_handler.get_rank();
+    auto comm = *exec_handler.get_host_comm().get();
 
+    // label rank = exec_handler.get_rank();
     // Foam::sleep(rank);
     // size_t send_size = comm_pattern.send_offsets.back();
     // std::vector<scalar> send_vec;;
@@ -236,7 +236,7 @@ std::vector<label> gather_labels_to_owner(const ExecutorHandler &exec_handler,
                                           label send_size, label offset)
 {
     auto exec = exec_handler.get_ref_exec();
-    auto comm = *exec_handler.get_communicator().get();
+    auto comm = *exec_handler.get_host_comm().get();
 
     std::vector<label> send_buffer_copy;
     // create a copy if offset is needed
@@ -274,7 +274,6 @@ std::ostream &operator<<(std::ostream &out, const CommunicationPattern &e)
 		    out << idx << " ";
 	    }
 	    out << "]\n";
-}
     out << "}\n";
     return out;
 }
@@ -296,15 +295,17 @@ gko::array<label> CommunicationPattern::compute_recv_gather_idxs(
     const ExecutorHandler &exec_handler) const
 {
     auto exec = exec_handler.get_ref_exec();
-    auto comm = *exec_handler.get_communicator().get();
+    auto host_comm = *exec_handler.get_host_comm().get();
+    auto device_comm = *exec_handler.get_device_comm().get();
     auto rs_idx = total_rank_send_idx();
     auto pattern = send_recv_pattern();
     auto recv_buffer =
         gko::array<label>(exec_handler.get_ref_exec(), rs_idx.size());
 
-    comm.all_to_all_v(exec, rs_idx.data(), pattern.send_counts.data(),
-                      pattern.send_offsets.data(), recv_buffer.get_data(),
-                      pattern.recv_counts.data(), pattern.recv_offsets.data());
+    device_comm.all_to_all_v(exec, rs_idx.data(), pattern.send_counts.data(),
+                             pattern.send_offsets.data(),
+                             recv_buffer.get_data(), pattern.recv_counts.data(),
+                             pattern.recv_offsets.data());
 
     return recv_buffer;
 }
@@ -312,24 +313,31 @@ gko::array<label> CommunicationPattern::compute_recv_gather_idxs(
 
 AllToAllPattern CommunicationPattern::send_recv_pattern() const
 {
-    auto comm = *exec_handler.get_communicator().get();
+    auto device_comm = *exec_handler.get_device_comm().get();
+    auto host_comm = *exec_handler.get_host_comm().get();
+
+    label device_ranks = device_comm.size();
+    label host_ranks = host_comm.size();
+    auto ratio = host_ranks / device_ranks;
 
-    std::vector<int> send_counts(comm.size());
-    std::vector<int> send_offsets(comm.size() + 1);
-    std::vector<int> recv_counts(comm.size());
-    std::vector<int> recv_offsets(comm.size() + 1);
+    std::vector<int> send_counts(device_ranks, 0);
+    std::vector<int> send_offsets(device_ranks + 1, 0);
+    std::vector<int> recv_counts(device_ranks, 0);
+    std::vector<int> recv_offsets(device_ranks + 1, 0);
 
-    label comm_ranks = target_ids.size();
     int tot_comm_size = 0;
-    for (label i = 0; i < comm_ranks; i++) {
-        auto comm_rank = target_ids.data()[i];
-        auto comm_size = target_sizes.data()[i];
+    for (label r = 0; r < target_ids.size(); r++) {
+        auto comm_rank = target_ids.data()[r];
+        auto comm_size = target_sizes.data()[r];
         tot_comm_size += comm_size;
-        send_counts[comm_rank] = comm_size;
-        recv_counts[comm_rank] = comm_size;
+
+        auto device_rank = label(comm_rank / ratio);
+        send_counts[device_rank] = comm_size;
+        recv_counts[device_rank] = comm_size;
     }
 
-    recv_offsets[comm.size()] = tot_comm_size;
+    recv_offsets[device_ranks] = tot_comm_size;
+
     std::partial_sum(recv_counts.begin(), recv_counts.end(),
                      recv_offsets.begin() + 1);
     recv_offsets[0] = 0;
diff --git a/src/MatrixWrapper/Distributed.cpp b/src/MatrixWrapper/Distributed.cpp
index 3f26d1fe..f64ee103 100644
--- a/src/MatrixWrapper/Distributed.cpp
+++ b/src/MatrixWrapper/Distributed.cpp
@@ -60,7 +60,7 @@ void generate_pairwise_update_data(
     // need to find original id, whether this rank sends or receives, the
     // comm_rank, interface_size and pointer to send to
     bool owner = repartitioner->is_owner(exec_handler);
-    label rank = exec_handler.get_rank();
+    label rank = exec_handler.get_host_rank();
     size_t offset = start_offset;
 
     for (size_t i = 0; i < in->get_id().size(); i++) {
@@ -147,7 +147,6 @@ void generate_reorder_map(
         &reorder_maps)
 {
     // NOTE early return if rank is empty
-    label rank = exec_handler.get_rank();
     if (maps.size() == 0) return;
     OGL_ASSERT_EQ(linops.size(), maps.size());
     for (size_t i = 0; i < linops.size(); i++) {
@@ -187,15 +186,14 @@ void RepartDistMatrix::write(const ExecutorHandler &exec_handler,
     if (write_global) {
         size_t rows = dist_mtx_->get_local_matrix()->get_size()[0];
         auto ref_exec = exec_handler.get_ref_exec();
-        auto comm = exec_handler.get_gko_mpi_host_comm();
+        auto comm = exec_handler.get_host_comm();
 
         auto partition = gko::share(
             gko::experimental::distributed::build_partition_from_local_size<
                 label, label>(ref_exec, *comm.get(), rows));
 
         // overwrite non_local column indices with global indices
-
-        label rank{exec_handler.get_rank()};
+        label rank{exec_handler.get_host_rank()};
         label offset = partition->get_range_bounds()[rank];
         label local_nnz = local->get_num_stored_elements();
 
@@ -259,9 +257,9 @@ void update_impl(
         &reorder_maps,
     bool fuse, std::map<label, scalar *> linops, label verbose)
 {
-    auto comm = exec_handler.get_communicator();
+    auto comm = exec_handler.get_host_comm();
     auto ref_exec = exec_handler.get_ref_exec();
-    auto rank = exec_handler.get_rank();
+    auto rank = exec_handler.get_host_rank();
     auto device_exec = exec_handler.get_device_exec();
     bool force_host_buffer = !exec_handler.get_non_orig_device_comm();
     word fieldname = host_A->get_field_name();
@@ -337,13 +335,12 @@ void update_impl(
     TIME_WITH_FIELDNAME(verbose, perform_pairwise_update, fieldname,
                         pairwise_communicate(););
 
-    auto reorder_data =
-        [reorder_maps, exec_handler]() {
-            for (auto [reorder_map, data_ptr] : reorder_maps) {
-                reorder_interface_impl<LocalMatrixType>(exec_handler,
-                                                        reorder_map, data_ptr);
-            }
-        };
+    auto reorder_data = [reorder_maps, exec_handler]() {
+        for (auto [reorder_map, data_ptr] : reorder_maps) {
+            reorder_interface_impl<LocalMatrixType>(exec_handler, reorder_map,
+                                                    data_ptr);
+        }
+    };
 
     TIME_WITH_FIELDNAME(verbose, reorder_matrix_data, fieldname,
                         reorder_data();)
@@ -372,9 +369,10 @@ std::shared_ptr<RepartDistMatrix> create_impl(
 {
     using dist_mtx =
         gko::experimental::distributed::Matrix<scalar, label, label>;
-    label rank = exec_handler.get_rank();
+    label rank = exec_handler.get_host_rank();
     auto exec = exec_handler.get_ref_exec();
-    auto comm = *exec_handler.get_communicator().get();
+    auto host_comm = *exec_handler.get_host_comm().get();
+    auto device_comm = *exec_handler.get_device_comm().get();
     bool owner = repartitioner->is_owner(exec_handler);
 
     auto [local_sparsity, non_local_sparsity] =
@@ -465,11 +463,11 @@ std::shared_ptr<RepartDistMatrix> create_impl(
 
     if (fuse) {
         dist_A = gko::share(dist_mtx::create(
-            device_exec, comm, global_dim, local_linops[0], non_local_linops[0],
-            recv_sizes, recv_offsets, recv_gather_idxs));
+            device_exec, device_comm, global_dim, local_linops[0],
+            non_local_linops[0], recv_sizes, recv_offsets, recv_gather_idxs));
     } else {
         dist_A = gko::share(dist_mtx::create(
-            device_exec, comm, global_dim,
+            device_exec, device_comm, global_dim,
             gko::share(CombinationMatrix<LocalMatrixType>::create(
                 device_exec, repart_dim, local_linops)),
             gko::share(CombinationMatrix<LocalMatrixType>::create(
@@ -495,7 +493,7 @@ std::shared_ptr<RepartDistMatrix> create_impl(
                                              fuse, linops, verbose););
 
     return std::make_shared<RepartDistMatrix>(
-        device_exec, comm, matrix_format, dist_A, repartitioner, fuse,
+        device_exec, host_comm, matrix_format, dist_A, repartitioner, fuse,
         all_to_all_update_data, pairwise_update_data, reorder_maps,
         compress_to_global, linops);
 }
diff --git a/src/MatrixWrapper/HostMatrix.cpp b/src/MatrixWrapper/HostMatrix.cpp
index 8ecf8796..aac4c055 100644
--- a/src/MatrixWrapper/HostMatrix.cpp
+++ b/src/MatrixWrapper/HostMatrix.cpp
@@ -51,8 +51,8 @@ HostMatrixWrapper::HostMatrixWrapper(
       interfaceBouCoeffs_(interfaceBouCoeffs)
 {
     auto ref_exec = exec.get_ref_exec();
-    auto comm = *exec.get_communicator().get();
-    label rank = exec.get_rank();
+    auto comm = *exec.get_host_comm().get();
+    label rank = comm.rank();
 
     using pair_dtype = std::pair<label, const scalar *>;
     // TODO this needs to be consistent with how sparsity are generated
@@ -181,7 +181,7 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_interface_sparsity(
         partition) const
 {
     // vector of neighbour cell idx connected to interface
-    auto rank = get_exec_handler().get_rank();
+    auto rank = get_exec_handler().get_host_rank();
     auto pattern = std::make_shared<SparsityPattern>();
 
     for (label i = 0; i < interfaces_.size(); i++) {
@@ -197,13 +197,11 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_interface_sparsity(
                 continue;
             }
 
-
             const auto &coupledPatch =
                 refCast<const coupledFvPatch>(iface->interface());
             const auto &patch =
                 refCast<const processorFvPatch>(iface->interface());
 
-
             const processorLduInterface &pldui =
                 refCast<const processorLduInterface>(iface->interface());
             const label neighbProcNo = pldui.neighbProcNo();
@@ -238,12 +236,13 @@ std::shared_ptr<SparsityPattern> HostMatrixWrapper::compute_interface_sparsity(
 #endif
             const labelUList &cols = addr_.patchAddr(neighbPatchId);
 
-            // FIXME the other comm_id is wrong
+            // TODO: check if the patch ids are correct
             pattern->insert_interface(
                 std::vector<label>(face_cells.cdata(),
                                    face_cells.cdata() + interface_size),
                 std::vector<label>(cols.cdata(), cols.cdata() + interface_size),
-                rank, rank, (i + 1) * -1, (i + 1) * -1);
+                rank, rank, -(local_to_global_interface_idx_[rank] + i),
+                -(local_to_global_interface_idx_[rank] + neighbPatchId));
         }
     }
     return pattern;
@@ -258,7 +257,7 @@ HostMatrixWrapper::compute_sparsity_patterns(
 {
     auto local_sparsity = compute_local_sparsity();
     auto non_local_sparsity = compute_interface_sparsity(partition);
-    auto rank = get_exec_handler().get_rank();
+    auto rank = get_exec_handler().get_host_rank();
     local_sparsity->move_interface(non_local_sparsity, rank,
                                    [](auto in) { return in; });
     return {local_sparsity, non_local_sparsity};
diff --git a/src/Repartitioner.cpp b/src/Repartitioner.cpp
index 41a8c3a7..40fbfb5d 100644
--- a/src/Repartitioner.cpp
+++ b/src/Repartitioner.cpp
@@ -27,8 +27,8 @@ Repartitioner::repartition_sparsity(
 
 
     auto exec = exec_handler.get_ref_exec();
-    auto comm = *exec_handler.get_communicator().get();
-    label rank = exec_handler.get_rank();
+    auto comm = *exec_handler.get_host_comm().get();
+    label rank = exec_handler.get_host_rank();
     label owner_rank = get_owner_rank(exec_handler);
     label ranks_per_gpu = ranks_per_gpu_;
 
@@ -154,9 +154,11 @@ Repartitioner::repartition_comm_pattern(
 
     // using comm_size_type = label;
     auto exec = exec_handler.get_ref_exec();
-    auto comm = src_comm_pattern->get_comm();
+    auto host_comm = exec_handler.get_host_comm();
+    auto device_comm = exec_handler.get_device_comm();
 
-    label rank = comm.rank();
+    label host_rank = host_comm->rank();
+    label device_rank = device_comm->rank();
     bool owner = is_owner(exec_handler);
 
     // Step 1. Check if communication partner is non-local after
@@ -195,24 +197,24 @@ Repartitioner::repartition_comm_pattern(
     // next the send_ixs need to be updated we send them piecewise since
     // the send_idxs are a vector of gko::arrays
     if (owner) {
-        label recv_ctr = comm_pattern.recv_counts[rank];
+        label recv_ctr = comm_pattern.recv_counts[host_rank];
         // retrieved from i-th neighbor
         for (int i = 1; i < ranks_per_gpu_; i++) {
             // how many gko::arrays to with send_indexes to receive
             // from i-th neighbor
-            label recv_count = comm_pattern.recv_counts[rank + i];
+            label recv_count = comm_pattern.recv_counts[host_rank + i];
 
             for (int j = 0; j < recv_count; j++) {
                 auto target_size = gathered_target_sizes[j + recv_ctr];
                 std::vector<label> recv_buffer(target_size);
 
-                comm.recv(exec, recv_buffer.data(), target_size, rank + i,
-                          rank);
+                host_comm->recv(exec, recv_buffer.data(), target_size,
+                                host_rank + i, host_rank);
 
                 // the new offset is
                 auto offset =
-                    get_orig_partition()->get_range_bounds()[rank + i] -
-                    get_orig_partition()->get_range_bounds()[rank];
+                    get_orig_partition()->get_range_bounds()[host_rank + i] -
+                    get_orig_partition()->get_range_bounds()[host_rank];
 
                 std::transform(recv_buffer.begin(), recv_buffer.end(),
                                recv_buffer.begin(),
@@ -226,8 +228,8 @@ Repartitioner::repartition_comm_pattern(
     } else {
         label owner = get_owner_rank(exec_handler);
         for (int i = 0; i < comm_pattern.send_counts[owner]; i++) {
-            comm.send(exec, send_idxs[i].data(), send_idxs[i].size(), owner,
-                      owner);
+            host_comm->send(exec, send_idxs[i].data(), send_idxs[i].size(),
+                            owner, owner);
         }
     }
 
@@ -291,6 +293,7 @@ Repartitioner::repartition_comm_pattern(
         send_idxs.emplace_back(merged_send_idxs[i]);
     }
 
+
     return std::make_shared<CommunicationPattern>(exec_handler,
                                                   merged_target_ids, send_idxs);
 }
diff --git a/test/unit/CommunicationPattern.cpp b/test/unit/CommunicationPattern.cpp
index db3af848..764efcf4 100644
--- a/test/unit/CommunicationPattern.cpp
+++ b/test/unit/CommunicationPattern.cpp
@@ -35,7 +35,7 @@ class CommunicationPatternEnvironment : public testing::Environment {
         exec = std::make_shared<ExecutorHandler>(time->thisDb(), dict, "dummy",
                                                  true);
 
-        auto comm = exec->get_gko_mpi_host_comm();
+        auto comm = exec->get_host_comm();
         if (comm->size() < 2) {
             std::cout << "At least 2 CPU processes should be used!"
                       << std::endl;
@@ -64,7 +64,7 @@ TEST(CommunicationPattern, compute_owner_rank_single_owner)
 {
     // Arrange
     auto exec = ((CommunicationPatternEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_host_comm();
+    auto comm = exec->get_host_comm();
 
     // Act
     auto owner_rank = compute_owner_rank(comm->rank(), comm->size());
@@ -77,7 +77,7 @@ TEST(CommunicationPattern, compute_owner_rank_two_owners)
 {
     // Arrange
     auto exec = ((CommunicationPatternEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_host_comm();
+    auto comm = exec->get_host_comm();
     auto comm_rank = comm->rank();
     auto comm_size = comm->size();
 
@@ -96,7 +96,7 @@ TEST(CommunicationPattern, compute_owner_rank_all_owners)
 {
     // Arrange
     auto exec = ((CommunicationPatternEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_host_comm();
+    auto comm = exec->get_host_comm();
     auto comm_rank = comm->rank();
 
     // Act
@@ -110,7 +110,7 @@ TEST(CommunicationPattern, compute_scatter_from_owner_counts_single_owner)
 {
     // Arrange
     auto exec = ((CommunicationPatternEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_host_comm();
+    auto comm = exec->get_host_comm();
     auto num_elements = 10;
     auto comm_size = comm->size();
     auto comm_rank = comm->rank();
@@ -151,7 +151,7 @@ TEST(CommunicationPattern, compute_scatter_from_owner_counts_two_owners)
 {
     // Arrange
     auto exec = ((CommunicationPatternEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_host_comm();
+    auto comm = exec->get_host_comm();
     auto num_elements = 10;
     auto comm_size = comm->size();
     auto comm_rank = comm->rank();
@@ -208,7 +208,7 @@ TEST(CommunicationPattern, compute_gather_to_owner_counts_single_owner)
 {
     // Arrange
     auto exec = ((CommunicationPatternEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_host_comm();
+    auto comm = exec->get_host_comm();
     auto num_elements = 10;
     auto comm_size = comm->size();
     auto comm_rank = comm->rank();
@@ -252,7 +252,7 @@ TEST(CommunicationPattern, compute_gather_to_owner_counts_two_owners)
 {
     // Arrange
     auto exec = ((CommunicationPatternEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_host_comm();
+    auto comm = exec->get_host_comm();
     auto num_elements = 10;
     auto comm_rank = comm->rank();
     auto comm_size = comm->size();
@@ -308,7 +308,7 @@ TEST(CommunicationPattern, compute_gather_to_owner_counts_all_owners)
 {
     // Arrange
     auto exec = ((CommunicationPatternEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_host_comm();
+    auto comm = exec->get_host_comm();
     auto num_elements = 10;
     auto comm_rank = comm->rank();
 
diff --git a/test/unit/MatrixWrapper/Distributed.cpp b/test/unit/MatrixWrapper/Distributed.cpp
index 43290f76..c34baf68 100644
--- a/test/unit/MatrixWrapper/Distributed.cpp
+++ b/test/unit/MatrixWrapper/Distributed.cpp
@@ -59,7 +59,7 @@ class Environment : public testing::Environment {
         exec = std::make_shared<ExecutorHandler>(runTime_->thisDb(), dict,
                                                  "dummy", true);
 
-        auto comm = exec->get_gko_mpi_host_comm();
+        auto comm = exec->get_host_comm();
         if (comm->size() != 4 || Pstream::nProcs() != 4) {
             std::cout << "This unit test expects to be run on 4 ranks"
                       << std::endl;
@@ -140,12 +140,9 @@ class DistMatL2D
     : public testing::TestWithParam<std::tuple<int, string, bool>> {
 public:
     ExecutorHandler exec = *((Environment *)global_env)->exec.get();
-    label rank = exec.get_rank();
+    label rank = exec.get_host_rank();
     const gko::experimental::mpi::communicator comm =
-        *(exec.get_communicator().get());
-
-    //
-    //
+        *(exec.get_host_comm().get());
 };
 
 
diff --git a/test/unit/MatrixWrapper/HostMatrix.cpp b/test/unit/MatrixWrapper/HostMatrix.cpp
index c4ca30ea..fc8fd197 100644
--- a/test/unit/MatrixWrapper/HostMatrix.cpp
+++ b/test/unit/MatrixWrapper/HostMatrix.cpp
@@ -49,7 +49,7 @@ class HostMatrixEnvironment : public testing::Environment {
         exec = std::make_shared<ExecutorHandler>(runTime_->thisDb(), dict,
                                                  "dummy", true);
 
-        auto comm = exec->get_gko_mpi_host_comm();
+        auto comm = exec->get_host_comm();
         if (comm->size() != 4 || Pstream::nProcs() != 4) {
             std::cout << "This unit test expects to be run on 4 ranks"
                       << std::endl;
@@ -87,8 +87,7 @@ class HostMatrixEnvironment : public testing::Environment {
         partition_ = gko::share(
             gko::experimental::distributed::build_partition_from_local_size<
                 label, label>(exec->get_ref_exec(),
-                              *exec->get_communicator().get(),
-                              exp_size[name_]));
+                              *exec->get_host_comm().get(), exp_size[name_]));
         std::cout << __FILE__ << __LINE__ << " done partition \n";
     }
 
@@ -118,7 +117,7 @@ TEST(HostMatrixTest, returnsCorrectSize)
     auto fvMatrix = ((HostMatrixEnvironment *)global_env)->fvMatrix;
     auto hostMatrix = ((HostMatrixEnvironment *)global_env)->hostMatrix;
     auto exec = ((HostMatrixEnvironment *)global_env)->exec;
-    auto rank = exec->get_rank();
+    auto rank = exec->get_host_rank();
     auto name = ((HostMatrixEnvironment *)global_env)->name_;
 
     EXPECT_EQ(mesh->C().size(), exp_size[name]);
@@ -133,14 +132,13 @@ TEST(HostMatrixTest, canCreateCommunicationPattern)
     std::shared_ptr<const HostMatrixWrapper> hostMatrix =
         ((HostMatrixEnvironment *)global_env)->hostMatrix;
     auto commPattern = hostMatrix->create_communication_pattern();
-    auto comm = commPattern->get_comm();
     auto exec = ((HostMatrixEnvironment *)global_env)->exec;
-    auto rank = exec->get_rank();
+    auto rank = exec->get_host_rank();
     auto name = ((HostMatrixEnvironment *)global_env)->name_;
 
     EXPECT_EQ(commPattern->send_idxs.size(), exp_send_idx_size[name][rank]);
-    EXPECT_EQ(commPattern->target_ids, exp_target_ids[name][comm.rank()]);
-    EXPECT_EQ(commPattern->target_sizes, exp_target_sizes[name][comm.rank()]);
+    EXPECT_EQ(commPattern->target_ids, exp_target_ids[name][rank]);
+    EXPECT_EQ(commPattern->target_sizes, exp_target_sizes[name][rank]);
 }
 
 TEST(HostMatrixTest, canGenerateLocalSparsityPattern)
@@ -162,8 +160,7 @@ TEST(HostMatrixTest, canGenerateNonLocalSparsityPattern)
 {
     auto hostMatrix = ((HostMatrixEnvironment *)global_env)->hostMatrix;
     auto exec = ((HostMatrixEnvironment *)global_env)->exec;
-    auto comm = exec->get_gko_mpi_device_comm();
-    auto rank = exec->get_rank();
+    auto rank = exec->get_host_rank();
     auto name = ((HostMatrixEnvironment *)global_env)->name_;
     auto partition = ((HostMatrixEnvironment *)global_env)->partition_;
 
diff --git a/test/unit/Repartitioner.H b/test/unit/Repartitioner.H
index 3f4771d6..5fabd972 100644
--- a/test/unit/Repartitioner.H
+++ b/test/unit/Repartitioner.H
@@ -36,10 +36,11 @@ public:
         db = std::make_shared<Foam::objectRegistry>(*time.get());
         Foam::dictionary dict;
         dict.add("executor", "reference");
+        dict.add("splitComm", 1);
         exec = std::make_shared<ExecutorHandler>(time->thisDb(), dict, "dummy",
                                                  true);
 
-        if (exec->get_communicator()->size() < 2) {
+        if (exec->get_host_comm()->size() < 2) {
             std::cout << "At least 2 CPU processes should be used!"
                       << std::endl;
             std::abort();
@@ -66,9 +67,9 @@ class RepartitionerFixture : public testing::Test {
 public:
     ExecutorHandler exec =
         *((RepartitionerEnvironment *)global_env)->exec.get();
-    label rank = exec.get_rank();
+    label rank = exec.get_host_rank();
     const gko::experimental::mpi::communicator comm =
-        *(exec.get_communicator().get());
+        *(exec.get_host_comm().get());
 };
 
 int main(int argc, char *argv[])