From 1443a1f5021d534da962fd7894d3823dfa044142 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=83=91=E5=90=AF=E8=88=AA?= <597323109@qq.com>
Date: Tue, 14 Jan 2025 09:38:34 +0800
Subject: [PATCH] Feature/opt boxing (#1282)

* add opt boxing

* add filter

* pass ut

* don't allow gather reduce with reshape

* fix reshape typeinfer bug

* add full search space

* fix compile bug

* remove fold boxing

* add latency in cpu options

* fix bug

* fix build

* Apply code-format changes

* fix build

* fix build

* fix ci

* fix lstm

* fix cycle

* fix ci

* fix boxing type infer

* fix dotnet test

---------

Co-authored-by: zhen8838 <zhen8838@users.noreply.github.com>
---
 .../CodeGen/CPU/CSourceBuiltn.cs              |    9 +-
 .../CPU/KernelCSourceConvertVisitor.cs        |    9 +-
 .../CodeGen/CPU/KernelUtility.cs              |    2 +-
 .../CPU/Templates/topo_aware_runtime.cshtml   |   66 +-
 .../Evaluator/CPU/Boxing.cs                   |  141 ++-
 .../Evaluator/CPU/CPUModule.cs                |    1 +
 .../Evaluator/CPU/ForceBoxing.cs              |   98 ++
 .../Evaluator/CPU/Im2col.cs                   |    2 +-
 .../Evaluator/CPU/PackedReduce.cs             |    5 +-
 modules/Nncase.Modules.CPU/IR/CPU/Boxing.cs   |    4 +-
 .../Nncase.Modules.CPU/IR/CPU/ForceBoxing.cs  |   29 +
 .../Nncase.Modules.CPU/IR/CPU/Functional.cs   |    9 +-
 .../Passes/Distributed/AutoDistributed.cs     | 1117 +++++++++++------
 ...tributedScheme.cs => DistributedSchema.cs} |    2 +-
 .../Passes/Rules/CPU/FoldMatmulReduce.cs      |   12 +-
 .../Passes/Rules/CPU/PackRule.cs              |    4 +-
 .../Passes/Tile/KernelToTIRVisitor.cs         |    3 +
 .../Nncase.Modules.CPU/Targets/CPUTarget.cs   |    7 +-
 .../Targets/CPUTargetOptions.cs               |   16 +-
 .../Targets/CPUTargetOptionsCommand.cs        |   24 +-
 .../Utilities/PackUtility.cs                  |  124 --
 modules/Nncase.Modules.CPU/packages.lock.json |    3 +-
 .../Nncase.Modules.StackVM/packages.lock.json |    3 +-
 python/_nncase.pyi                            |    2 +
 python/nncase/native/ffi.cpp                  |    8 +
 src/Native/include/nncase/compiler.h          |   24 +
 src/Nncase.Cli/packages.lock.json             |    3 +-
 src/Nncase.Compiler/Compiler.cs               |    2 +-
 src/Nncase.Compiler/Interop/CApi.cs           |   16 +
 src/Nncase.Compiler/packages.lock.json        |    3 +-
 src/Nncase.Core/CostModel/Cost.cs             |    2 +
 src/Nncase.Core/DistributedType.cs            |   14 +-
 src/Nncase.Core/Enum/ReduceOp.cs              |    3 +
 src/Nncase.Core/IR/Math/Reduce.cs             |    4 +-
 src/Nncase.Core/IR/Tensors/Functional.cs      |    4 +-
 src/Nncase.Core/IR/Tensors/GetItem.cs         |    2 +-
 src/Nncase.Core/ITarget.cs                    |    4 +
 .../Utilities/DistributedUtility.cs           |   37 -
 src/Nncase.Core/Utilities/IRUtility.cs        |  131 ++
 src/Nncase.EGraph/Nncase.EGraph.csproj        |    1 +
 src/Nncase.EGraph/Passes/EGraphExtractor.cs   |  521 +-------
 src/Nncase.EGraph/packages.lock.json          |   14 +
 src/Nncase.Evaluator/Math/Binary.cs           |    8 +-
 src/Nncase.Evaluator/Math/Compare.cs          |    6 +-
 src/Nncase.Evaluator/Math/MatMul.cs           |    4 +-
 src/Nncase.Evaluator/Math/Reduce.cs           |   18 +-
 src/Nncase.Evaluator/Math/ReduceArg.cs        |    2 +-
 src/Nncase.Evaluator/Math/Unary.cs            |    2 +-
 src/Nncase.Evaluator/NN/Activations.cs        |    6 +-
 src/Nncase.Evaluator/NN/Conv2D.cs             |    2 +-
 src/Nncase.Evaluator/Tensors/Cast.cs          |    2 +-
 src/Nncase.Evaluator/Tensors/Concat.cs        |    2 +-
 src/Nncase.Evaluator/Tensors/Reshape.cs       |  134 +-
 src/Nncase.Graph/Graphs/HyperGraph.cs         |  526 ++++++++
 .../Rules/Neutral/CombineTranspose.cs         |   38 +-
 .../Rules/Neutral/CombineUnary.cs             |    2 +-
 src/Nncase.Passes/packages.lock.json          |    3 +-
 src/Nncase.Quantization/packages.lock.json    |    3 +-
 .../Schedule/TilingUtilities.cs               |    2 +-
 src/Nncase.Schedule/packages.lock.json        |    3 +-
 src/Nncase.Studio/packages.lock.json          |    3 +-
 src/Nncase.Targets/packages.lock.json         |    3 +-
 .../packages.lock.json                        |    3 +-
 .../UnitTestIRUtilities.cs}                   |    6 +-
 ...eScheme.cs => UnitTestDistributeSchema.cs} |    9 +-
 .../UnitTestDistributedTypeInfer.cs           |  116 ++
 .../UnitTestDistributedUtilities.cs           |   24 -
 .../Rewrite/Fusion/UnitTestGraphPartition.cs  |   10 +-
 src/Nncase.Tests/Rewrite/RewriteBase.cs       |   40 +-
 .../Rewrite/UnitTestDataFlowRewrite.cs        |   66 +
 .../Rewrite/UnitTestEGraphRewriteFactory.cs   |    3 +-
 .../Rules/Neutral/UnitTestCombineTranspose.cs |    2 +-
 .../Rules/Neutral/UnitTestCombineUnary.cs     |    2 +-
 .../Targets/UnitTestCPUKernels.cs             |  271 ++--
 src/Nncase.Tests/packages.lock.json           |    3 +-
 75 files changed, 2368 insertions(+), 1441 deletions(-)
 create mode 100644 modules/Nncase.Modules.CPU/Evaluator/CPU/ForceBoxing.cs
 create mode 100644 modules/Nncase.Modules.CPU/IR/CPU/ForceBoxing.cs
 rename modules/Nncase.Modules.CPU/Passes/Distributed/{DistributedScheme.cs => DistributedSchema.cs} (75%)
 create mode 100644 src/Nncase.Core/Utilities/IRUtility.cs
 create mode 100644 src/Nncase.Graph/Graphs/HyperGraph.cs
 rename src/Nncase.Tests/{Rules/Packing/PackUtilityTest.cs => Core/UnitTestIRUtilities.cs} (93%)
 rename src/Nncase.Tests/Distributed/{UnitTestDistributeScheme.cs => UnitTestDistributeSchema.cs} (87%)
 create mode 100644 src/Nncase.Tests/Distributed/UnitTestDistributedTypeInfer.cs
 delete mode 100644 src/Nncase.Tests/Distributed/UnitTestDistributedUtilities.cs

diff --git a/modules/Nncase.Modules.CPU/CodeGen/CPU/CSourceBuiltn.cs b/modules/Nncase.Modules.CPU/CodeGen/CPU/CSourceBuiltn.cs
index 5037fc6e3..260c963d5 100644
--- a/modules/Nncase.Modules.CPU/CodeGen/CPU/CSourceBuiltn.cs
+++ b/modules/Nncase.Modules.CPU/CodeGen/CPU/CSourceBuiltn.cs
@@ -47,13 +47,8 @@ public static class CSourceBuiltn
 
     public static string TopoAwareRuntimeDef(CpuTargetOptions options, ulong dataAlign, ulong collective_pool_size)
     {
-        if (options.Hierarchies[0].Any(i => i > 1))
-        {
-            var content = RazorTemplateEngine.RenderAsync("~/CodeGen/CPU/Templates/topo_aware_runtime.cshtml", new CpuTargetOptionsModel(options, dataAlign, collective_pool_size)).Result;
-            return content;
-        }
-
-        return string.Empty;
+        var content = RazorTemplateEngine.RenderAsync("~/CodeGen/CPU/Templates/topo_aware_runtime.cshtml", new CpuTargetOptionsModel(options, dataAlign, collective_pool_size)).Result;
+        return content;
     }
 
     public static string TopologyDef(CpuTargetOptions options)
diff --git a/modules/Nncase.Modules.CPU/CodeGen/CPU/KernelCSourceConvertVisitor.cs b/modules/Nncase.Modules.CPU/CodeGen/CPU/KernelCSourceConvertVisitor.cs
index 7b69500ca..4213a0112 100644
--- a/modules/Nncase.Modules.CPU/CodeGen/CPU/KernelCSourceConvertVisitor.cs
+++ b/modules/Nncase.Modules.CPU/CodeGen/CPU/KernelCSourceConvertVisitor.cs
@@ -437,7 +437,7 @@ protected override CSymbol VisitCall(Call expr)
 
                     break;
                 case TIR.Memcopy copy:
-                    IndentScope.Writer.Write($"tensor_copy({VisitBuffer(args[0], local: true).Name}, {VisitBuffer(args[1], local: true).Name});\n");
+                    IndentScope.Writer.Write($"tensor_copy({VisitBuffer(args[1], local: true).Name}, {VisitBuffer(args[0], local: true).Name});\n");
                     break;
                 case TIR.CPU.Gather gather:
                     IndentScope.Writer.Write($"gather<{gather.Axis}>({VisitBuffer(args[0], local: true).Name}, {VisitBuffer(args[1], local: true).Name}, {VisitBuffer(args[2], local: true).Name});\n");
@@ -505,10 +505,11 @@ protected override CSymbol VisitCall(Call expr)
                     break;
                 case TIR.CPU.GatherReduceScatter grs:
                     {
-                        if (grs.InType.NdSBP.Any(s => s is SBPPartialSum))
+                        if (grs.InType.NdSBP.Any(s => s is SBPPartial))
                         {
-                            var reduceKind = "tar::reduce_kind::" + string.Join("_", grs.InType.NdSBP.Select((s, i) => (s is SBPPartialSum ? "r" : string.Empty) + TargetOptions.HierarchyNames[i]));
-                            IndentScope.Writer.IndWrite($"tac::tensor_reduce_sync<ops::add, {reduceKind}>({VisitBuffer(args[0], local: true).Name}, {VisitBuffer(args[1], local: true).Name});\n");
+                            var sbpPartial = (SBPPartial)grs.InType.NdSBP.Where(s => s is SBPPartial).Distinct().First();
+                            var reduceKind = "tar::reduce_kind::" + string.Join("_", grs.InType.NdSBP.Select((s, i) => (s is SBPPartial ? "r" : string.Empty) + TargetOptions.HierarchyNames[i]));
+                            IndentScope.Writer.IndWrite($"tac::tensor_reduce_sync<reduce_op::{sbpPartial.Op.ToC()}, {reduceKind}>({VisitBuffer(args[0], local: true).Name}, {VisitBuffer(args[1], local: true).Name});\n");
                         }
                         else
                         {
diff --git a/modules/Nncase.Modules.CPU/CodeGen/CPU/KernelUtility.cs b/modules/Nncase.Modules.CPU/CodeGen/CPU/KernelUtility.cs
index 84c249313..080be9d5c 100644
--- a/modules/Nncase.Modules.CPU/CodeGen/CPU/KernelUtility.cs
+++ b/modules/Nncase.Modules.CPU/CodeGen/CPU/KernelUtility.cs
@@ -80,7 +80,7 @@ public static string DistributedToC(DistributedType distributedType)
             }
         }
 
-        var implicitPolicy = ndSBP.Any(x => x is SBPPartialSum) ? "P<reduce_op::sum>" : "B";
+        var implicitPolicy = ndSBP.Any(x => x is SBPPartial) ? "P<reduce_op::sum>" : "B";
         sb.Append($">, {implicitPolicy}");
 
         for (int axis = 0; axis < distributedType.TensorType.Shape.Rank; axis++)
diff --git a/modules/Nncase.Modules.CPU/CodeGen/CPU/Templates/topo_aware_runtime.cshtml b/modules/Nncase.Modules.CPU/CodeGen/CPU/Templates/topo_aware_runtime.cshtml
index edfe0e290..ad6fb71d9 100644
--- a/modules/Nncase.Modules.CPU/CodeGen/CPU/Templates/topo_aware_runtime.cshtml
+++ b/modules/Nncase.Modules.CPU/CodeGen/CPU/Templates/topo_aware_runtime.cshtml
@@ -102,7 +102,7 @@ template <tar::reduce_kind Kind> class group_heirarchy_getter;
 @:};
 }
 
-template <template <class, class> class Op, tar::reduce_kind Kind>
+template <ntt::reduce_op Op, tar::reduce_kind Kind>
 class tensor_reduce_sync_impl {
   public:
     void reduce_group_sync() const noexcept {
@@ -154,6 +154,20 @@ class tensor_reduce_sync_impl {
       var cur_index = string.Join(", ", Enumerable.Range(0, hierarchy.Length).Select(i => "ntt::distributed::" + hierarchyNames[i] + "id()"));
     }
 
+    template <class TSliceIn, class TSliceOut>
+    void reduce_impl(TSliceIn &local, TSliceIn &remote, TSliceOut &dest) {
+        if constexpr (Op == ntt::reduce_op::max) {
+            ntt::binary<ntt::ops::max>(local, remote, dest);
+        } else if constexpr (Op == ntt::reduce_op::sum ||
+                             Op == ntt::reduce_op::mean) {
+            ntt::binary<ntt::ops::add>(local, remote, dest);
+        } else if constexpr (Op == ntt::reduce_op::min) {
+            ntt::binary<ntt::ops::min>(local, remote, dest);
+        } else if constexpr (Op == ntt::reduce_op::prod) {
+            ntt::binary<ntt::ops::mul>(local, remote, dest);
+        }
+    }
+
     template <class TIn, class TOut> void operator()(TIn &src, TOut &&dest) {
         // collect all tensors pointer for access tensor from other nodes.
         using TElem = typename TIn::element_type;
@@ -198,8 +212,8 @@ class tensor_reduce_sync_impl {
         auto next_index = ntt::make_ranked_shape(@(cur_index));
         index_group2global(next_index_g, next_index);
 
-        // reduce-scatter
-        for (auto i = 0; i < group_size - 1; i++) // communicate (group_size - 1) times
+        // reduce-scatter, communicate (group_size - 1) times
+        for (auto i = 0; i < group_size - 1; i++) 
         {
             // check when the last time.
             auto offset = (node_number_g + i + 2) % group_size;
@@ -216,21 +230,30 @@ class tensor_reduce_sync_impl {
             auto viewed_dest_tensor = dest.view(starts, new_shape);
 
             if (i == 0) {
-                auto src2_tensor = ntt::tensor_view<TElem, typename TIn::shape_type, typename TIn::strides_type>(
-                                        std::span<TElem, TIn::shape().length()>((TElem *)tar::src_ptr_tensor(next_index), src.shape().length()));
+                auto src2_tensor =
+                    ntt::tensor_view<TElem, typename TIn::shape_type,
+                                     typename TIn::strides_type>(
+                        std::span<TElem, TIn::shape().length()>(
+                            (TElem *)tar::src_ptr_tensor(next_index),
+                            src.shape().length()));
                 auto viewed_src2_tensor = src2_tensor.view(starts, new_shape);
-                ntt::binary<Op>(viewed_src1_tensor, viewed_src2_tensor, viewed_dest_tensor);
+                reduce_impl(viewed_src1_tensor, viewed_src2_tensor,
+                            viewed_dest_tensor);
             } else {
-                auto src2_tensor = ntt::tensor_view<TElem, typename TOutBase::shape_type, typename TOutBase::strides_type>(
-                                        std::span<TElem, TOutBase::shape().length()>((TElem *)tar::dest_ptr_tensor(next_index), dest.shape().length()));
+                auto src2_tensor =
+                    ntt::tensor_view<TElem, typename TOutBase::shape_type,
+                                     typename TOutBase::strides_type>(
+                        std::span<TElem, TOutBase::shape().length()>(
+                            (TElem *)tar::dest_ptr_tensor(next_index),
+                            dest.shape().length()));
                 auto viewed_src2_tensor = src2_tensor.view(starts, new_shape);
-                ntt::binary<Op>(viewed_src1_tensor, viewed_src2_tensor, viewed_dest_tensor);
+                reduce_impl(viewed_src1_tensor, viewed_src2_tensor,
+                            viewed_dest_tensor);
             }
 
-
             reduce_group_sync();
         }
-        
+
         // all gather
         auto dest_index_g = ntt::unravel_index((node_number_g + group_size - 1) % group_size, group_heirarchy);
         auto dest_index = ntt::make_ranked_shape(@(cur_index));
@@ -244,8 +267,7 @@ class tensor_reduce_sync_impl {
             for (size_t j = 0; j < Rank; j++) {
                 if (j == axis) {
                     starts[j] = offset * frac;
-                }
-                else {
+                } else {
                     starts[j] = 0;
                 }
             }
@@ -253,18 +275,22 @@ class tensor_reduce_sync_impl {
             new_shape[axis] = (offset == (group_size - 1)) ? frac + remain : frac;
             auto viewed_src_tensor = dest.view(starts, new_shape);
             auto viewed_dest_tensor = dest_tensor.view(starts, new_shape);
-            ntt::tensor_copy(std::move(viewed_src_tensor), std::move(viewed_dest_tensor));
+            ntt::tensor_copy(std::move(viewed_src_tensor),
+                             std::move(viewed_dest_tensor));
 
             reduce_group_sync();
         }
+
+        if (Op == ntt::reduce_op::mean) {
+          ntt::binary<ntt::ops::div>(dest, ntt::tensor<TElem, ntt::fixed_shape<1>>((TElem)group_size), dest);
+        }
     }
 };
 } // namespace detail
 
-template <template <class, class> class Op, tar::reduce_kind Kind, class TIn,
-          class TOut>
-void tensor_reduce_sync(TIn &input, TOut &&output){
-  detail::tensor_reduce_sync_impl<Op, Kind> impl;
-  impl(input, output);
-}
+template <ntt::reduce_op Op, tar::reduce_kind Kind, class TIn, class TOut>
+void tensor_reduce_sync(TIn &input, TOut &&output) {
+    detail::tensor_reduce_sync_impl<Op, Kind> impl;
+    impl(input, output);
 }
+} // namespace tac
diff --git a/modules/Nncase.Modules.CPU/Evaluator/CPU/Boxing.cs b/modules/Nncase.Modules.CPU/Evaluator/CPU/Boxing.cs
index 18d356ad8..ee8d1810f 100644
--- a/modules/Nncase.Modules.CPU/Evaluator/CPU/Boxing.cs
+++ b/modules/Nncase.Modules.CPU/Evaluator/CPU/Boxing.cs
@@ -13,56 +13,79 @@ namespace Nncase.Evaluator.IR.CPU;
 
 public sealed class BoxingEvaluator : ITypeInferencer<Boxing>, ICostEvaluator<Boxing>, IEvaluator<Boxing>
 {
-    private const int _burstLength = 256;
-
-    public IRType Visit(ITypeInferenceContext context, Boxing target)
+    public static IRType VisitType(IRType inType, IRType outType, bool isReshape = false)
     {
-        var check = (DistributedType inv, DistributedType outv) =>
+        IRType VisitD2D(DistributedType inv, DistributedType outv)
+        {
+            if (inv == outv)
             {
-                // TODO: add more invalid cases
-                if (inv.NdSBP.Distinct().Count() == 1 && outv.NdSBP.Distinct().Count() == 1 && inv.NdSBP[0] == outv.NdSBP[0])
+                return new InvalidType("Same DistributedType");
+            }
+
+            if (inv.TensorType != outv.TensorType)
+            {
+                if (!inv.NdSBP.Any(sbp => sbp is SBPPartial))
                 {
-                    return (IRType)new InvalidType("Same NDSBP");
+                    return outv;
                 }
+            }
 
-                if (inv.NdSBP.Any(sbp => sbp is SBPPartialSum))
+            // TODO: add more invalid cases
+            if (inv.NdSBP.Distinct().Count() == 1 && outv.NdSBP.Distinct().Count() == 1 && inv.NdSBP[0] == outv.NdSBP[0])
+            {
+                return new InvalidType("Same NDSBP");
+            }
+
+            for (int i = 0; i < inv.NdSBP.Count; i++)
+            {
+                switch (inv.NdSBP[i], outv.NdSBP[i])
                 {
-                    DistributedUtility.TryGetDividedTensorType(inv, out var inType);
-                    DistributedUtility.TryGetDividedTensorType(outv, out var outType);
+                    case (SBPPartial, SBPSplit):
+                        return new InvalidType("partial to split");
+                    case (not SBPPartial, SBPPartial):
+                        return new InvalidType("split/broadcast to partial");
+                }
+            }
 
-                    var nonPartialSumPos = Enumerable.Range(0, inv.NdSBP.Count).Where(i => inv.NdSBP[i] is not SBPPartialSum);
-                    if (nonPartialSumPos.Any(i => inv.NdSBP[i] is SBPSplit && outv.NdSBP[i] is SBPBroadCast))
-                    {
-                        return new InvalidType("Not supported input is Splite output is BroadCast");
-                    }
+            return outv;
+        }
 
-                    var partialSumPos = Enumerable.Range(0, inv.NdSBP.Count).Where(i => inv.NdSBP[i] is SBPPartialSum);
-                    if (partialSumPos.Any(i => inv.NdSBP[i] is SBPPartialSum && outv.NdSBP[i] is SBPSplit))
-                    {
-                        return new InvalidType("Not supported input is Partial output is Split");
-                    }
+        IRType VisitD2T(DistributedType inv, TensorType outv)
+        {
+            if (inv.NdSBP.Any(s => s is SBPPartial))
+            {
+                return new InvalidType("Not supported input is Partial output is Unshard");
+            }
 
-                    if (target.IsReshape)
-                    {
-                        return new InvalidType("partial not support reshape");
-                    }
+            return outv;
+        }
 
-                    return outv;
-                }
-                else
-                {
-                    return outv;
-                }
-            };
+        IRType VisitT2D(TensorType inv, DistributedType outv)
+        {
+            if (outv.NdSBP.Any(s => s is SBPPartial))
+            {
+                return new InvalidType("Not supported input is Unshard output is Partial");
+            }
 
-        return (context.GetArgumentType(target, Boxing.Input), target.NewType) switch
+            return outv;
+        }
+
+        return (inType, outType) switch
         {
             (InvalidType inv, _) => inv,
-            (DistributedType inv, DistributedType outv) => check(inv, outv),
-            _ => target.NewType,
+            (_, InvalidType inv) => inv,
+            (DistributedType d, DistributedType d1) => VisitD2D(d, d1),
+            (TensorType t, DistributedType d) => VisitT2D(t, d),
+            (DistributedType d, TensorType t) => VisitD2T(d, t),
+            _ => new InvalidType($"not support boxing {inType} to {outType}"),
         };
     }
 
+    public IRType Visit(ITypeInferenceContext context, Boxing target)
+    {
+        return VisitType(context.GetArgumentType(target, Boxing.Input), target.NewType);
+    }
+
     public Cost Visit(ICostEvaluateContext context, Boxing target)
     {
         var inType = context.GetArgumentType<IRType>(target, Boxing.Input);
@@ -70,31 +93,25 @@ public Cost Visit(ICostEvaluateContext context, Boxing target)
         var cost = new Cost() { [CostFactorNames.MemoryLoad] = 0, [CostFactorNames.MemoryStore] = 0 };
         switch (inType, returnType)
         {
-            case (TensorType tensorType, DistributedType distTensorType):
+            case (TensorType _, DistributedType distTensorType):
                 switch (context.CompileOptions.TargetOptions)
                 {
-                    case Targets.CpuTargetOptions { UnifiedMemoryArch: true }:
-                        break;
                     default:
                         cost = new Cost()
                         {
-                            [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(tensorType),
-                            [CostFactorNames.MemoryStore] = (UInt128)((float)CostUtility.GetMemoryAccess(distTensorType) / DistributedUtility.GetDividedTensorEfficiency(distTensorType, _burstLength)),
+                            [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(distTensorType),
                         };
                         break;
                 }
 
                 break;
-            case (DistributedType distTensorType, TensorType tensorType):
+            case (DistributedType distTensorType, TensorType _):
                 switch (context.CompileOptions.TargetOptions)
                 {
-                    case Targets.CpuTargetOptions { UnifiedMemoryArch: true }:
-                        break;
                     default:
                         cost = new Cost()
                         {
-                            [CostFactorNames.MemoryLoad] = (UInt128)((float)CostUtility.GetMemoryAccess(distTensorType) / DistributedUtility.GetDividedTensorEfficiency(distTensorType, _burstLength)),
-                            [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(tensorType),
+                            [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(distTensorType),
                         };
                         break;
                 }
@@ -105,12 +122,14 @@ public Cost Visit(ICostEvaluateContext context, Boxing target)
                 {
                     var fullLoadStore = new Cost()
                     {
-                        [CostFactorNames.MemoryStore] = (UInt128)((float)CostUtility.GetMemoryAccess(a) / DistributedUtility.GetDividedTensorEfficiency(a, _burstLength)),
-                        [CostFactorNames.MemoryLoad] = (UInt128)((float)CostUtility.GetMemoryAccess(b) / DistributedUtility.GetDividedTensorEfficiency(b, _burstLength)),
+                        [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(a),
+                        [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(b),
                     };
 
                     float scatterPart = 1;
                     float gatherPart = 1;
+                    float reducePart = 1;
+                    float latency = 0;
                     for (int i = 0; i < a.Placement.Rank; i++)
                     {
                         switch (a.NdSBP[i], b.NdSBP[i])
@@ -142,12 +161,14 @@ public Cost Visit(ICostEvaluateContext context, Boxing target)
                                     [CostFactorNames.CPUCycles] = 1,
                                 };
                                 break;
-                            case (SBPPartialSum, SBP sbpout):
+                            case (SBPPartial, SBP sbpout):
                                 switch (sbpout)
                                 {
-                                    case SBPPartialSum:
+                                    case SBPPartial:
                                         break;
-                                    case SBPBroadCast or SBPSplit:
+                                    case SBPBroadCast:
+                                        latency = MathF.Max(latency, ((ICpuTargetOptions)context.CompileOptions.TargetOptions).HierarchyLatencies[i]);
+                                        reducePart *= a.Placement.Hierarchy[i];
                                         gatherPart *= a.Placement.Hierarchy[i];
                                         if (i == 0)
                                         {
@@ -155,10 +176,12 @@ public Cost Visit(ICostEvaluateContext context, Boxing target)
                                         }
 
                                         break;
+                                    case SBPSplit:
+                                        throw new NotSupportedException("split to partial");
                                 }
 
                                 break;
-                            case (SBPBroadCast, SBPPartialSum):
+                            case (SBPBroadCast, SBPPartial):
                                 // note this case only for tests.
                                 cost += new Cost()
                                 {
@@ -177,12 +200,20 @@ public Cost Visit(ICostEvaluateContext context, Boxing target)
                             [CostFactorNames.MemoryStore] = (UInt128)((gatherPart - 1) * (float)CostUtility.GetMemoryAccess(DistributedUtility.GetDividedTensorType(a)) / gatherPart),
                         };
 
-                        if (a.Placement.HierarchyKind == HierarchyKind.SMT && (a.NdSBP[1] is SBPPartialSum))
+                        if (a.Placement.HierarchyKind == HierarchyKind.SMT && (a.NdSBP[1] is SBPPartial))
                         {
                             cost[CostFactorNames.MemoryStore] *= 8;
                         }
                     }
 
+                    if (reducePart > 1f)
+                    {
+                        cost += new Cost()
+                        {
+                            [CostFactorNames.Comm] = (UInt128)((reducePart - 1) * latency),
+                        };
+                    }
+
                     if (scatterPart > 1f)
                     {
                         cost += new Cost()
@@ -196,15 +227,15 @@ public Cost Visit(ICostEvaluateContext context, Boxing target)
             case (DistributedType a, DistributedType b) when a.TensorType != b.TensorType && a.Placement == b.Placement:
                 cost = new Cost()
                 {
-                    [CostFactorNames.MemoryStore] = (UInt128)((float)CostUtility.GetMemoryAccess(a) / DistributedUtility.GetDividedTensorEfficiency(a, _burstLength)),
-                    [CostFactorNames.MemoryLoad] = (UInt128)((float)CostUtility.GetMemoryAccess(b) / DistributedUtility.GetDividedTensorEfficiency(b, _burstLength)),
+                    [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(a),
+                    [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(b),
                 };
                 break;
             case (DistributedType a, DistributedType b) when a.Placement != b.Placement:
                 cost = new Cost()
                 {
-                    [CostFactorNames.MemoryStore] = (UInt128)((float)CostUtility.GetMemoryAccess(a) / DistributedUtility.GetDividedTensorEfficiency(a, _burstLength)),
-                    [CostFactorNames.MemoryLoad] = (UInt128)((float)CostUtility.GetMemoryAccess(b) / DistributedUtility.GetDividedTensorEfficiency(b, _burstLength)),
+                    [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(a),
+                    [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(b),
                 };
                 break;
             case (DistributedType a, DistributedType b) when a == b:
diff --git a/modules/Nncase.Modules.CPU/Evaluator/CPU/CPUModule.cs b/modules/Nncase.Modules.CPU/Evaluator/CPU/CPUModule.cs
index ade842f7f..4d130900e 100644
--- a/modules/Nncase.Modules.CPU/Evaluator/CPU/CPUModule.cs
+++ b/modules/Nncase.Modules.CPU/Evaluator/CPU/CPUModule.cs
@@ -14,6 +14,7 @@ internal class CPUModule : IApplicationPart
     public void ConfigureServices(IRegistrator registrator)
     {
         registrator.RegisterManyInterface<BoxingEvaluator>(reuse: Reuse.Singleton);
+        registrator.RegisterManyInterface<ForceBoxingEvaluator>(reuse: Reuse.Singleton);
         registrator.RegisterManyInterface<LoadEvaluator>(reuse: Reuse.Singleton);
         registrator.RegisterManyInterface<StoreEvaluator>(reuse: Reuse.Singleton);
         registrator.RegisterManyInterface<PackEvaluator>(reuse: Reuse.Singleton);
diff --git a/modules/Nncase.Modules.CPU/Evaluator/CPU/ForceBoxing.cs b/modules/Nncase.Modules.CPU/Evaluator/CPU/ForceBoxing.cs
new file mode 100644
index 000000000..a04ec41fa
--- /dev/null
+++ b/modules/Nncase.Modules.CPU/Evaluator/CPU/ForceBoxing.cs
@@ -0,0 +1,98 @@
+// Copyright (c) Canaan Inc. All rights reserved.
+// Licensed under the Apache license. See LICENSE file in the project root for full license information.
+#pragma warning disable SA1010, SA1008
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Nncase.CostModel;
+using Nncase.IR;
+using Nncase.IR.CPU;
+using Nncase.Utilities;
+
+namespace Nncase.Evaluator.IR.CPU;
+
+public sealed class ForceBoxingEvaluator : ITypeInferencer<ForceBoxing>, ICostEvaluator<ForceBoxing>, IEvaluator<ForceBoxing>
+{
+    public static IRType VisitType(IRType inType, IRType outType)
+    {
+        IRType VisitD2D(DistributedType inv, DistributedType outv)
+        {
+            // TODO: add more invalid cases
+            if (inv.NdSBP.Distinct().Count() == 1 && outv.NdSBP.Distinct().Count() == 1 && inv.NdSBP[0] == outv.NdSBP[0])
+            {
+                return new InvalidType("Same NDSBP");
+            }
+
+            if (inv.NdSBP.Any(sbp => sbp is SBPPartial))
+            {
+                var nonPartialSumPos = Enumerable.Range(0, inv.NdSBP.Count).Where(i => inv.NdSBP[i] is not SBPPartial);
+                if (nonPartialSumPos.Any(i => inv.NdSBP[i] is SBPSplit && outv.NdSBP[i] is SBPBroadCast))
+                {
+                    return new InvalidType("Not supported input is Split output is BroadCast");
+                }
+
+                var partialSumPos = Enumerable.Range(0, inv.NdSBP.Count).Where(i => inv.NdSBP[i] is SBPPartial);
+                if (partialSumPos.Any(i => inv.NdSBP[i] is SBPPartial && outv.NdSBP[i] is SBPSplit))
+                {
+                    return new InvalidType("Not supported input is Partial output is Split");
+                }
+
+                return outv;
+            }
+
+            if (outv.NdSBP.Any(sbp => sbp is SBPPartial))
+            {
+                var nonPartialSumPos = Enumerable.Range(0, outv.NdSBP.Count).Where(i => outv.NdSBP[i] is not SBPPartial);
+                if (nonPartialSumPos.Any(i => inv.NdSBP[i] is SBPSplit && outv.NdSBP[i] is SBPBroadCast))
+                {
+                    return new InvalidType("Not supported input is Split output is BroadCast");
+                }
+
+                var partialSumPos = Enumerable.Range(0, outv.NdSBP.Count).Where(i => outv.NdSBP[i] is SBPPartial);
+                if (partialSumPos.Any(i => inv.NdSBP[i] is SBPSplit && outv.NdSBP[i] is SBPPartial))
+                {
+                    return new InvalidType("Not supported input is Split output is Partial");
+                }
+
+                return outv;
+            }
+
+            return outv;
+        }
+
+        return (inType, outType) switch
+        {
+            (InvalidType inv, _) => inv,
+            (_, InvalidType inv) => inv,
+            (DistributedType d, DistributedType d1) => VisitD2D(d, d1),
+            _ => new InvalidType($"not support Forceboxing {inType} to {outType}"),
+        };
+    }
+
+    public IRType Visit(ITypeInferenceContext context, ForceBoxing target)
+    {
+        return VisitType(context.GetArgumentType(target, ForceBoxing.Input), target.NewType);
+    }
+
+    public Cost Visit(ICostEvaluateContext context, ForceBoxing target)
+    {
+        var inType = context.GetArgumentType<IRType>(target, ForceBoxing.Input);
+        var returnType = context.GetReturnType<IRType>();
+        var cost = new Cost() { [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inType), [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(returnType) };
+        return cost;
+    }
+
+    public IValue Visit(IEvaluateContext context, ForceBoxing target)
+    {
+        var inTenor = context.GetArgumentValueAsTensor(target, ForceBoxing.Input);
+        var input = inTenor.ToOrtTensor();
+        var output = input - input;
+        var repeat = target.NewType.NdSBP.Select((x, i) => (x is SBPPartial) ? target.NewType.Placement.Hierarchy[i] : 1).Aggregate(1, (x, i) => x * i);
+        for (int i = 0; i < repeat; i++)
+        {
+            output += input;
+        }
+
+        return Value.FromTensor(Tensor.FromBytes(inTenor.ElementType, output.BytesBuffer.ToArray(), target.NewType.TensorType.Shape));
+    }
+}
diff --git a/modules/Nncase.Modules.CPU/Evaluator/CPU/Im2col.cs b/modules/Nncase.Modules.CPU/Evaluator/CPU/Im2col.cs
index b838d8c2e..b07d0788a 100644
--- a/modules/Nncase.Modules.CPU/Evaluator/CPU/Im2col.cs
+++ b/modules/Nncase.Modules.CPU/Evaluator/CPU/Im2col.cs
@@ -130,7 +130,7 @@ private IRType Visit(DistributedType dt, Im2col target)
                     }
 
                     break;
-                case SBPPartialSum:
+                case SBPPartial:
                     return new InvalidType($"can't be partial sum");
                 default:
                     ndsbp[i] = sbp;
diff --git a/modules/Nncase.Modules.CPU/Evaluator/CPU/PackedReduce.cs b/modules/Nncase.Modules.CPU/Evaluator/CPU/PackedReduce.cs
index 4a7a67c08..acf04ded9 100644
--- a/modules/Nncase.Modules.CPU/Evaluator/CPU/PackedReduce.cs
+++ b/modules/Nncase.Modules.CPU/Evaluator/CPU/PackedReduce.cs
@@ -108,8 +108,6 @@ private IRType Visit(ITypeInferenceContext context, PackedReduce target, Distrib
         }
 
         var axes = target.Axes.ToArray();
-        var invalid = new InvalidType($"{input}, not support");
-
         var ndsbp = new SBP[input.Placement.Rank];
 
         for (int i = 0; i < input.Placement.Rank; i++)
@@ -117,7 +115,8 @@ private IRType Visit(ITypeInferenceContext context, PackedReduce target, Distrib
             switch (input.NdSBP[i])
             {
                 case SBPSplit { Axis: int ix } when axes.Contains(ix):
-                    return invalid;
+                    ndsbp[i] = SBP.P(target.ReduceOp);
+                    break;
                 default:
                     ndsbp[i] = input.NdSBP[i];
                     break;
diff --git a/modules/Nncase.Modules.CPU/IR/CPU/Boxing.cs b/modules/Nncase.Modules.CPU/IR/CPU/Boxing.cs
index bb39b781f..016b14036 100644
--- a/modules/Nncase.Modules.CPU/IR/CPU/Boxing.cs
+++ b/modules/Nncase.Modules.CPU/IR/CPU/Boxing.cs
@@ -24,8 +24,6 @@ public sealed partial class Boxing : Op
 
     public IRType NewType { get; }
 
-    public bool IsReshape { get; }
-
     /// <inheritdoc/>
-    public override string DisplayProperty() => $"NewType: {NewType}, IsReshape: {IsReshape}";
+    public override string DisplayProperty() => $"NewType: {NewType}";
 }
diff --git a/modules/Nncase.Modules.CPU/IR/CPU/ForceBoxing.cs b/modules/Nncase.Modules.CPU/IR/CPU/ForceBoxing.cs
new file mode 100644
index 000000000..8913960db
--- /dev/null
+++ b/modules/Nncase.Modules.CPU/IR/CPU/ForceBoxing.cs
@@ -0,0 +1,29 @@
+// Copyright (c) Canaan Inc. All rights reserved.
+// Licensed under the Apache license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Nncase.PatternMatch;
+
+namespace Nncase.IR.CPU;
+
+/// <summary>
+/// Force Boxing, only can change broadcast to partial, just use for test ccl.
+/// </summary>
+[PatternFunctionalGenerator]
+public sealed partial class ForceBoxing : Op
+{
+    /// <summary>
+    /// Gets input.
+    /// </summary>
+    public static readonly ParameterInfo Input = new(typeof(ForceBoxing), 0, "input", ParameterKind.Input);
+
+    public DistributedType NewType { get; }
+
+    /// <inheritdoc/>
+    public override string DisplayProperty() => $"NewType: {NewType}";
+}
diff --git a/modules/Nncase.Modules.CPU/IR/CPU/Functional.cs b/modules/Nncase.Modules.CPU/IR/CPU/Functional.cs
index 13e18c583..3b1a44dc8 100644
--- a/modules/Nncase.Modules.CPU/IR/CPU/Functional.cs
+++ b/modules/Nncase.Modules.CPU/IR/CPU/Functional.cs
@@ -12,9 +12,14 @@ namespace Nncase.IR.F;
 
 public partial class CPU
 {
-    public static Call Boxing(Expr input, IRType type, bool isReshape = false)
+    public static Call Boxing(Expr input, IRType type)
     {
-        return new Call(new Boxing(type, isReshape), input);
+        return new Call(new Boxing(type), input);
+    }
+
+    public static Call ForceBoxing(Expr input, DistributedType type)
+    {
+        return new Call(new ForceBoxing(type), input);
     }
 
     public static Call Load(Expr input)
diff --git a/modules/Nncase.Modules.CPU/Passes/Distributed/AutoDistributed.cs b/modules/Nncase.Modules.CPU/Passes/Distributed/AutoDistributed.cs
index 74423538a..7fa4d5173 100644
--- a/modules/Nncase.Modules.CPU/Passes/Distributed/AutoDistributed.cs
+++ b/modules/Nncase.Modules.CPU/Passes/Distributed/AutoDistributed.cs
@@ -1,33 +1,29 @@
 // Copyright (c) Canaan Inc. All rights reserved.
 // Licensed under the Apache license. See LICENSE file in the project root for full license information.
 
+using System.Diagnostics.CodeAnalysis;
 using System.Reactive;
 using System.Runtime.CompilerServices;
 using Google.OrTools.Sat;
 using NetFabric.Hyperlinq;
-using Nncase.CodeGen;
+using Nncase.Graphs;
 using Nncase.IR;
 using Nncase.IR.CPU;
-using Nncase.IR.Tensors;
-using Nncase.PatternMatch;
 using Nncase.Targets;
 using Nncase.Utilities;
-using static Nncase.PatternMatch.Utility;
+using QuikGraph;
+using QuikGraph.Graphviz;
 
 [assembly: InternalsVisibleTo("Nncase.Tests")]
 
 namespace Nncase.Passes.Distributed;
 
-public interface IEquality
-{
-}
-
-public record EqualityNode(Expr Expr) : IEquality
-{
-}
-
-public record EqualityClass(bool Tuple, List<IEquality> Children) : IEquality
+internal enum SearchGraphKind : int
 {
+    Root,
+    DistributedCluster,
+    StandaloneCluster,
+    Bucket,
 }
 
 public sealed class AutoDistributedMetaData : IRMetadata
@@ -36,7 +32,7 @@ public sealed class AutoDistributedMetaData : IRMetadata
 }
 
 /// <summary>
-/// auto distributed the xpu fusion.
+/// auto distributed the function.
 /// </summary>
 [RuleGenerator]
 public sealed partial class AutoDistributedPass : FunctionPass
@@ -49,11 +45,14 @@ public sealed partial class AutoDistributedPass : FunctionPass
 
     public AutoDistributedPass(bool bidirectional, string moduleKind, CompileOptions compileOptions)
     {
+        Bidirectional = bidirectional;
         _compileOptions = compileOptions;
         _bidirectional = bidirectional;
         _moduleKind = moduleKind;
     }
 
+    public bool Bidirectional { get; }
+
     protected override Task<BaseFunction> RunCoreAsync(BaseFunction input, RunPassContext context)
     {
         if (input.Metadata is AutoDistributedMetaData { Skip: true })
@@ -66,9 +65,71 @@ protected override Task<BaseFunction> RunCoreAsync(BaseFunction input, RunPassCo
     }
 }
 
-internal sealed class AutoDistributedRewriter : ExprVisitor<Dictionary<IRType, List<Expr>>, Unit>
+internal sealed class SearchableNode
 {
-    private readonly Dictionary<Expr, IEquality> _equalMemo = new();
+    public SearchableNode(Expr expr, IRType type, bool isBidirect = false)
+    {
+        Expr = expr;
+        IRType = type;
+        IsBidirect = isBidirect;
+    }
+
+    public Expr Expr { get; }
+
+    public IRType IRType { get; }
+
+    public bool IsBidirect { get; }
+}
+
+internal sealed record CrossEdge : IEdge<SearchableNode>
+{
+    public CrossEdge(SearchableNode root, SearchableNode input, int inputIndex, DistributedSearchGraph inputGraph)
+    {
+        Root = root;
+        Input = input;
+        InputIndex = inputIndex;
+        InputGraph = inputGraph;
+    }
+
+    public SearchableNode Root { get; }
+
+    public SearchableNode Input { get; }
+
+    public int InputIndex { get; }
+
+    public DistributedSearchGraph InputGraph { get; }
+
+    public SearchableNode Source => Root;
+
+    public SearchableNode Target => Input;
+}
+
+internal sealed class DistributedSearchGraph : TieredAdjacencyGraph<SearchableNode, CrossEdge>
+{
+    public DistributedSearchGraph([NotNull] AdjacencyGraph<SearchableNode, CrossEdge> wrappedGraph, SearchGraphKind kind)
+    : base(wrappedGraph)
+    {
+        Kind = kind;
+    }
+
+    public DistributedSearchGraph([NotNull] TieredAdjacencyGraph<SearchableNode, CrossEdge> parentGraph, SearchGraphKind kind)
+        : base(parentGraph)
+    {
+        Kind = kind;
+    }
+
+    public SearchGraphKind Kind { get; }
+}
+
+internal sealed class AutoDistributedRewriter : ExprVisitor<Unit, Unit>
+{
+    private readonly Dictionary<Expr, DistributedSearchGraph> _reshardMemo;
+
+    private readonly Dictionary<Expr, DistributedSearchGraph> _inferedMemo;
+
+    private readonly AdjacencyGraph<SearchableNode, CrossEdge> _rootGraph;
+
+    private readonly DistributedSearchGraph _rootSearchGraph;
 
     private readonly string _moduleKind;
 
@@ -76,10 +137,12 @@ internal sealed class AutoDistributedRewriter : ExprVisitor<Dictionary<IRType, L
 
     public AutoDistributedRewriter(CompileOptions compileOptions, CpuTargetOptions targetOptions, string moduleKind = "cpu", bool bidirectional = false)
     {
-        Placements = targetOptions.Hierarchies.Select(h => new Placement(h, targetOptions.HierarchyNames, targetOptions.HierarchyKind)).ToArray();
+        Placements = targetOptions.Hierarchies.Select(h => new Placement(h, targetOptions.HierarchyNames)).ToArray();
+        Bidirectional = bidirectional;
         CompileOptions = compileOptions;
         TargetOptions = targetOptions;
-        if (Path.Exists(TargetOptions.DistributedScheme) && System.Text.Json.JsonSerializer.Deserialize<DistributedScheme>(File.ReadAllText(TargetOptions.DistributedScheme)) is DistributedScheme scheme)
+        _moduleKind = moduleKind;
+        if (Path.Exists(TargetOptions.DistributedScheme) && System.Text.Json.JsonSerializer.Deserialize<DistributedSchema>(File.ReadAllText(TargetOptions.DistributedScheme)) is DistributedSchema scheme)
         {
             Scheme = scheme.Outputs.ToDictionary(n => n.Name, n => (new IRArray<SBP>(n.NdSBP), new Placement(n.Hierarchy, n.HierarchyName, targetOptions.HierarchyKind)));
         }
@@ -88,13 +151,18 @@ public AutoDistributedRewriter(CompileOptions compileOptions, CpuTargetOptions t
             Scheme = new Dictionary<string, (IRArray<SBP> NdSBP, Placement Placement)>();
         }
 
+        _reshardMemo = new(ReferenceEqualityComparer.Instance);
+        _inferedMemo = new(ReferenceEqualityComparer.Instance);
+        _rootGraph = new(true);
+        _rootSearchGraph = new(_rootGraph, SearchGraphKind.Root);
         _moduleKind = moduleKind;
-
         _bidirectional = bidirectional;
     }
 
     public IRArray<Placement> Placements { get; }
 
+    public bool Bidirectional { get; }
+
     public CompileOptions CompileOptions { get; }
 
     public CpuTargetOptions TargetOptions { get; }
@@ -111,23 +179,9 @@ public static void MemoryExtractConstrains(CpModel model, IReadOnlyDictionary<EN
         })) < (2L * 512L * 1024L * 1024L));
     }
 
-    public static IReadOnlyList<Expr> GetLeafCandidateBoxings(Expr expr, IEnumerable<Placement> placements)
+    public static IReadOnlyList<DistributedType> GetLeafCandidateDistTypes(TensorType tensorType, IEnumerable<Placement> placements)
     {
-        return placements.Select(
-            placement =>
-                Utilities.DistributedUtility.GetLeafCandidateNDSBPs((TensorType)expr.CheckedType, placement).
-                Select(ndsbp =>
-                    IR.F.CPU.Boxing(expr, new DistributedType((TensorType)expr.CheckedType, ndsbp, placement)))).
-            SelectMany(e => e).ToArray();
-    }
-
-    public static IReadOnlyList<IRArray<SBP>> GetDiverseCandidateSBPs(DistributedType distributedType, IEnumerable<Placement> placements)
-    {
-        return placements.Select(
-            placement =>
-                Utilities.DistributedUtility.GetLeafCandidateNDSBPs(distributedType.TensorType, placement).
-                Where(ndsbp => ndsbp != distributedType.NdSBP)).
-            SelectMany(e => e).ToArray();
+        return placements.Select(placement => DistributedUtility.GetLeafCandidateNDSBPs(tensorType, placement).Select(ndsbp => new DistributedType(tensorType, ndsbp, placement))).SelectMany(e => e).ToArray();
     }
 
     public void SingleNodeMemoryExtractConstrains(CpModel model, IReadOnlyDictionary<ENode, BoolVar> vars)
@@ -140,7 +194,7 @@ public void SingleNodeMemoryExtractConstrains(CpModel model, IReadOnlyDictionary
                 var type = DistributedUtility.GetDividedTensorType((DistributedType)k.Expr.CheckedType);
                 var size = TensorUtilities.GetProduct(type.Shape.ToValueArray()) * type.DType.SizeInBytes;
 
-                if (k.Expr is Call { Target: IR.CPU.Boxing boxing } call && boxing.NewType is DistributedType distributedType && call.Arguments[0].CheckedType is DistributedType inType && inType.NdSBP.Any(sbp => sbp is SBPPartialSum) && distributedType != call.Arguments[0].CheckedType)
+                if (k.Expr is Call { Target: IR.CPU.Boxing boxing } call && boxing.NewType is DistributedType distributedType && call.Arguments[0].CheckedType is DistributedType inType && inType.NdSBP.Any(sbp => sbp is SBPPartial) && distributedType != call.Arguments[0].CheckedType)
                 {
                     type = DistributedUtility.GetDividedTensorType(inType);
                     size += TensorUtilities.GetProduct(type.Shape.ToValueArray()) * type.DType.SizeInBytes;
@@ -151,18 +205,42 @@ public void SingleNodeMemoryExtractConstrains(CpModel model, IReadOnlyDictionary
         }
     }
 
-    public void FilterByScheme(Expr expr, Dictionary<IRType, List<Expr>> result)
+    public void FilterByScheme(Expr expr, DistributedSearchGraph cluster)
     {
+        bool Matched(SearchableNode node, (IRArray<SBP> NdSBP, Placement Placement) tp)
+        {
+            return node.IRType is DistributedType dtype && dtype.NdSBP == tp.NdSBP && dtype.Placement == tp.Placement;
+        }
+
         foreach (var name in expr.Metadata.OutputNames ?? Array.Empty<string>())
         {
             if (Scheme.TryGetValue(name, out var tp))
             {
-                var keys = result.Keys.ToArray();
-                foreach (var key in keys)
+                if (cluster.Kind is SearchGraphKind.DistributedCluster)
                 {
-                    if (!(key is DistributedType dtype && dtype.NdSBP == tp.NdSBP && dtype.Placement == tp.Placement))
+                    if (!cluster.Clusters.OfType<DistributedSearchGraph>().Any(b => Matched(b.Vertices.First(), tp)))
+                    {
+                        return;
+                    }
+
+                    var removes = new List<DistributedSearchGraph>();
+                    foreach (var bucket in cluster.Clusters.OfType<DistributedSearchGraph>())
+                    {
+                        bucket.RemoveVertexIf(v => !Matched(v, tp));
+                        if (bucket.VertexCount == 0)
+                        {
+                            removes.Add(bucket);
+                        }
+                    }
+
+                    foreach (var r in removes)
                     {
-                        result.Remove(key);
+                        cluster.RemoveCluster(r);
+                    }
+
+                    foreach (var bucket in cluster.Clusters.OfType<DistributedSearchGraph>().Where(b => Matched(b.Vertices.First(), tp)))
+                    {
+                        bucket.RemoveVertexIf(v => _rootSearchGraph.TryGetOutEdges(v, out var edges) && !edges.Any());
                     }
                 }
             }
@@ -171,536 +249,749 @@ public void FilterByScheme(Expr expr, Dictionary<IRType, List<Expr>> result)
 
     public BaseFunction Rewirte(BaseFunction input)
     {
-        if (input is Function || input is Fusion)
+        if (input is Function { Body: Expr body } function)
         {
-            var body = input is Function ? ((Function)input).Body : ((Fusion)input).Body;
-            var typeEquivalents = Visit(body);
+            Visit(body);
+            var rootCluster = TryInstertTerminator(body);
 
-            if (body is IR.Tuple tp)
+            using (var stream = Diagnostics.DumpScope.Current.IsEnabled(Diagnostics.DumpFlags.PassIR) ? Diagnostics.DumpScope.Current.OpenFile("DistributedSearchGraph.dot") : Stream.Null)
             {
-                var outputs = new List<Expr>();
-                var equ = _equalMemo[tp];
+                Dump(stream, new Dictionary<SearchableNode, bool>() { }, new Dictionary<SearchableNode, CostModel.Cost>() { });
+            }
 
-                void Dfs(IEquality equality)
-                {
-                    switch (equality)
-                    {
-                        case EqualityNode n:
-                            outputs.Add(n.Expr);
-                            break;
-                        case EqualityClass tp:
-                            foreach (var item in tp.Children)
-                            {
-                                Dfs(item);
-                            }
+            var post = SolveAndExtract(rootCluster);
+            return function.With(body: post);
+        }
 
-                            break;
-                    }
-                }
+        return input;
+    }
 
-                Dfs(equ);
+    protected override Unit DefaultVisitLeaf(Expr expr)
+    {
+        return default;
+    }
 
-                using (new ExprPinner(outputs.ToArray()))
-                {
-                    BranchCut();
-                }
+    protected override Unit VisitLeafCall(Call expr)
+    {
+        bool isSupported;
+        var argClusters = new DistributedSearchGraph[expr.Arguments.Length];
+        if (expr.Target is not Op op)
+        {
+            isSupported = false;
+            foreach (var (param, i) in expr.Arguments.AsValueEnumerable().Select((p, i) => (p, i)))
+            {
+                argClusters[i] = VisitLeafArgument(ParameterKind.Input, expr.Arguments[i], isSupported);
             }
-            else
+        }
+        else
+        {
+            isSupported = PassUtility.IsCpuSupported(op, expr, expr.Arguments.ToArray(), _moduleKind);
+            foreach (var param in op.Parameters)
             {
-                var outputs = typeEquivalents.Select(g => InstertTerminator(g.Value[0]))
-                .Select(e => new EqualityNode(e))
-                .OfType<IEquality>().ToList();
+                argClusters[param.Index] = VisitLeafArgument(param.ParameterKind, expr.Arguments[param.Index], isSupported);
+            }
+        }
 
-                if (outputs.Any())
-                {
-                    _equalMemo.Add(body, new EqualityClass(false, outputs));
+        var callCluster = _rootSearchGraph.CreateCluster<DistributedSearchGraph>(isSupported ? SearchGraphKind.DistributedCluster : SearchGraphKind.StandaloneCluster);
 
-                    using (new ExprPinner(outputs.Select(e => ((EqualityNode)e).Expr).ToArray()))
-                    {
-                        BranchCut();
-                    }
+        // 1. inference
+        var bucketMemo = new Dictionary<IRType, DistributedSearchGraph>();
+        foreach (var combBuckets in argClusters.Select(c => c.Clusters.OfType<DistributedSearchGraph>()).CartesianProduct())
+        {
+            var tempArgs = combBuckets.Select<DistributedSearchGraph, Expr>(bucket => bucket.Vertices.First() switch
+            {
+                SearchableNode { Expr: Const attr } => attr,
+                SearchableNode n => new Var(n.IRType),
+            }).ToArray();
+            var newCalls = BuildEquivalentCalls(expr.Target, tempArgs);
+            foreach (var (newCall, used) in newCalls)
+            {
+                var checkType = newCall.CheckedType;
+                if (checkType is InvalidType)
+                {
+                    continue;
                 }
-                else
+
+                if (!bucketMemo.TryGetValue(checkType, out var dbucket))
                 {
-                    return input;
+                    dbucket = callCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+                    bucketMemo.Add(checkType, dbucket);
                 }
-            }
 
-            var graph = new EGraph();
-            foreach (var (exprKey, buckets) in ExprMemo.Where(kv => kv.Key is not Op))
-            {
-                foreach (var (typeKey, bucket) in buckets.Where(kv => kv.Value.Any()))
+                var dnode = new SearchableNode(newCall.Target, checkType);
+                dbucket.AddVertex(dnode);
+
+                foreach (var ((arg, _), i) in combBuckets.Zip(used).Where(p => p.Second is true).Select((arg, i) => (arg, i)))
                 {
-                    Unions(graph, bucket);
+                    _rootSearchGraph.AddEdge(new(dnode, arg.Vertices.First(), i, arg));
                 }
             }
+        }
 
-            var equivalents = _equalMemo[body];
-            EClass Ddfs(IEquality equival)
-            {
-                switch (equival)
-                {
-                    case EqualityNode n:
-                        return graph.Add(n.Expr);
-                    case EqualityClass tp:
-                        var eids = tp.Children.Select(Ddfs).ToArray();
-                        if (tp.Tuple)
-                        {
-                            return graph.AddENode(new IR.Tuple(), eids);
-                        }
-                        else
-                        {
-                            foreach (var cls in eids.Skip(1))
-                            {
-                                graph.Union(eids[0], cls);
-                            }
+        _inferedMemo.Add(expr, callCluster);
 
-                            graph.Rebuild();
-                            return eids[0];
-                        }
+        if (!isSupported)
+        {
+            return default;
+        }
 
-                    default:
-                        throw new NotSupportedException();
+        // 3. add bidirectional connections.
+        if (Bidirectional)
+        {
+            foreach (var (lType, lBucket) in bucketMemo.Where(kv => kv.Key is DistributedType))
+            {
+                foreach (var (rType, rBucket) in bucketMemo.Where(kv => kv.Key is DistributedType distributedType && distributedType != lType))
+                {
+                    if (Evaluator.IR.CPU.BoxingEvaluator.VisitType(lType, rType) is not InvalidType)
+                    {
+                        var rnode = new SearchableNode(new Boxing(rType), rType, true);
+                        rBucket.AddVertex(rnode);
+                        callCluster.AddEdge(new(rnode, lBucket.Vertices.First(), 0, lBucket));
+                    }
                 }
             }
+        }
 
-            var root = Ddfs(equivalents);
-            if (Diagnostics.DumpScope.Current.IsEnabled(Diagnostics.DumpFlags.EGraphCost))
+        // 4. add not infered type in search space.
+        var addedBuckets = bucketMemo.Values.ToArray();
+        foreach (var nType in GetLeafCandidateDistTypes(expr.CheckedTensorType, Placements))
+        {
+            if (!bucketMemo.TryGetValue(nType, out var bucket))
             {
-                using (var stream = Diagnostics.DumpScope.Current.OpenFile("egraph.dot"))
+                bucket = callCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+                var node = new SearchableNode(new Boxing(nType), nType);
+                bucket.AddVertex(node);
+                var linked = false;
+                foreach (var addedBucket in addedBuckets)
                 {
-                    EGraphPrinter.DumpEgraphAsDot(graph, stream);
+                    var addedNode = addedBucket.Vertices.First();
+                    if (Evaluator.IR.CPU.BoxingEvaluator.VisitType(addedNode.IRType, nType) is not InvalidType)
+                    {
+                        callCluster.AddEdge(new(node, addedNode, 0, addedBucket));
+                        linked |= true;
+                    }
                 }
-            }
-
-            var constrains = new EGraphExtractConstrains[] { SingleNodeMemoryExtractConstrains };
-            var post = graph.Extract(root, CompileOptions, null, constrains);
 
-            if (input is Function)
-            {
-                return ((Function)input).With(body: post);
-            }
-            else
-            {
-                return ((Fusion)input).With(body: post);
+                if (!linked)
+                {
+                    bucket.RemoveVertex(node);
+                    callCluster.RemoveCluster(bucket);
+                }
+                else
+                {
+                    bucketMemo.Add(nType, bucket);
+                }
             }
         }
 
-        return input;
+        // 5. filter
+        FilterByScheme(expr, callCluster);
+        return default;
     }
 
-    protected override Dictionary<IRType, List<Expr>> DefaultVisitLeaf(Expr expr)
+    /// <summary>
+    /// some times we didn't use all args.
+    /// </summary>
+    private IEnumerable<(Call Call, bool[] Used)> BuildEquivalentCalls(Expr target, Expr[] tempArgs)
     {
-        return new();
+        IEnumerable<(Call Call, bool[] Used)> calls = [(new Call(target, tempArgs), Enumerable.Repeat(true, tempArgs.Length).ToArray())];
+        if (target is IR.Tensors.Reshape reshape && tempArgs[0].CheckedType is DistributedType distType && tempArgs[1] is TensorConst tc)
+        {
+            var newShape = tc.Value.ToArray<int>();
+            var newTensorType = new TensorType(distType.TensorType.DType, newShape);
+            calls = calls.Concat(DistributedUtility.GetLeafCandidateNDSBPs(newTensorType, distType.Placement).Select(ndsbp => (new Call(new Boxing(new DistributedType(newTensorType, ndsbp, distType.Placement)), tempArgs[0]), new[] { true, false })));
+        }
+
+        return calls;
     }
 
-    protected override Dictionary<IRType, List<Expr>> VisitLeafIf(If expr)
+    private IReadOnlyList<IRArray<SBP>> GetDiverseCandidateSBPs(DistributedType distributedType, IEnumerable<Placement> placements)
     {
-        return new() { { expr.CheckedType, new() { expr } } };
+        return placements.Select(
+            placement =>
+                DistributedUtility.GetLeafCandidateNDSBPs(distributedType.TensorType, placement).
+                Where(ndsbp => ndsbp != distributedType.NdSBP)).
+            SelectMany(e => e).ToArray();
     }
 
-    protected override Dictionary<IRType, List<Expr>> VisitLeafTuple(IR.Tuple expr)
+    private DistributedSearchGraph VisitLeafArgument(ParameterKind parameterKind, Expr expr, bool isSupported)
     {
-        if (ReferenceEquals(expr, VisitRoot))
+        DistributedSearchGraph argCluster;
+        switch (parameterKind, expr)
         {
-            var fileds = new List<IEquality>();
-            foreach (var i in Enumerable.Range(0, expr.Fields.Length))
-            {
-                var boxings = Visit(expr.Fields[i]).Values.
-                    Select(l => l.Select(e => e.CheckedType is DistributedType dt ? (dt.NdSBP.Any(s => s is SBPPartialSum) ? IR.F.CPU.Boxing(IR.F.CPU.Boxing(e, new DistributedType(dt.TensorType, dt.NdSBP.Select(s => s is SBPPartialSum ? SBP.B : s).ToArray(), dt.Placement)), dt.TensorType) : IR.F.CPU.Boxing(e, dt.TensorType)) : e).ToArray()).
-                    SelectMany(e => e).Select(e => new EqualityNode(e)).OfType<IEquality>().ToList();
-                fileds.Add(new EqualityClass(false, boxings));
-            }
+            case (ParameterKind.Input, Expr e):
+                if (isSupported)
+                {
+                    argCluster = TryAddOriginator(e);
+                }
+                else
+                {
+                    argCluster = TryInstertTerminator(e);
+                }
 
-            _equalMemo.Add(expr, new EqualityClass(true, fileds));
-            return new Dictionary<IRType, List<Expr>> { }; // return empty.
+                break;
+            case (ParameterKind.Attribute, Expr e):
+                argCluster = TryInstertTerminator(e);
+                break;
+            default:
+                throw new InvalidOperationException();
         }
 
-        return expr.Fields.ToArray().
-                Select(Visit).
-                CartesianProduct().
-                Select(e => new IR.Tuple(e.Select(e => e.Value[0]).ToArray())).
-                GroupBy(tp => tp.CheckedType).
-                ToDictionary(g => g.Key, g => g.ToList<Expr>());
+        FilterByScheme(expr, argCluster);
+        return argCluster ?? throw new InvalidOperationException("the argument cluster can't be null.");
     }
 
-    protected override Dictionary<IRType, List<Expr>> VisitLeafCall(Call expr)
+    private bool IsDistributed(IRType type) => type switch
+    {
+        DistributedType => true,
+        TupleType t => t.All(IsDistributed),
+        _ => false,
+    };
+
+    private DistributedSearchGraph CreateOriginatorCluster(Expr expr, bool init)
     {
-        if (expr.Target is Fusion fusion)
+        if (expr is IR.Tuple tp)
         {
-            foreach (var idx in Enumerable.Range(0, fusion.Parameters.Length))
+            var distCluster = _rootSearchGraph.CreateCluster<DistributedSearchGraph>(SearchGraphKind.DistributedCluster);
+            var buckets = new DistributedSearchGraph[tp.Fields.Length];
+            foreach (var (f, fGraph, i) in tp.Fields.AsValueEnumerable().Select((f, i) => (f, Visit(f), i)))
             {
-                VisitLeafArgument(ParameterKind.Input, expr.Arguments[idx], false);
+                buckets[i] = TryAddOriginator(f).Clusters.OfType<DistributedSearchGraph>().First();
             }
 
-            var rewriter = new AutoDistributedRewriter(CompileOptions, TargetOptions);
-            var post = rewriter.Rewirte(fusion);
-            var ret = expr.Arguments.ToArray().
-                    Select(Visit).
-                    CartesianProduct().
-                    Select(args => args.ToArray()).
-                    Select(args => args.Select(kv => kv.Value[0]).Select(arg => arg.CheckedType switch
-                    {
-                        DistributedType d => GetDiverseCandidateSBPs(d, Placements).Select(ndsbp => IR.F.CPU.Boxing(arg, new DistributedType(d.TensorType, ndsbp, d.Placement))).Concat(new[] { arg }).ToArray(),
-                        _ => new[] { arg },
-                    }).ToList().CartesianProduct().Select(arg => expr.With(target: post, arguments: arg.ToArray())).ToArray()).
-                    SelectMany(i => i).
-                    GroupBy(c => c.CheckedType).
-                    ToDictionary(g => g.Key, g => g.OrderByDescending(e => e.Users.Count()).ToList<Expr>());
+            var tpnode = new SearchableNode(new IR.Tuple(), new TupleType(buckets.Select(g => g.Vertices.First().IRType).ToArray()));
+            var bucket = distCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+            bucket.AddVertex(tpnode);
+            for (int i = 0; i < tp.Fields.Length; i++)
+            {
+                _rootSearchGraph.AddEdge(new(tpnode, buckets[i].Vertices.First(), i, buckets[i]));
+            }
 
-            return ret;
+            return distCluster;
         }
+        else if (expr is TensorConst tc && tc.ValueType is TensorType tensorType)
+        {
+            var distCluster = _rootSearchGraph.CreateCluster<DistributedSearchGraph>(SearchGraphKind.DistributedCluster);
+            foreach (var dType in GetLeafCandidateDistTypes(tensorType, Placements))
+            {
+                var bucket = distCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+                var dnode = new SearchableNode(new TensorConst(tc.Value, dType.NdSBP, dType.Placement), dType);
+                bucket.AddVertex(dnode);
+            }
 
-        if (expr.Target is not Op op)
+            return distCluster;
+        }
+        else
         {
-            return new Dictionary<IRType, List<Expr>> { { expr.CheckedType, new() { expr } } };
+            if (init)
+            {
+                var standCluster = _rootSearchGraph.CreateCluster<DistributedSearchGraph>(SearchGraphKind.StandaloneCluster);
+                var bucket = standCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+                var node = new SearchableNode(expr, expr.CheckedType);
+                bucket.AddVertex(node);
+                return standCluster;
+            }
+            else
+            {
+                var distCluster = _rootSearchGraph.CreateCluster<DistributedSearchGraph>(SearchGraphKind.DistributedCluster);
+                var inferCluster = _inferedMemo[expr];
+                foreach (var dType in GetLeafCandidateDistTypes((TensorType)inferCluster.Vertices.First().IRType, Placements))
+                {
+                    var bucket = distCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+                    var dnode = new SearchableNode(new Boxing(dType), dType);
+                    bucket.AddVertex(dnode);
+                    _rootSearchGraph.AddEdge(new(dnode, inferCluster.Vertices.First(), 0, inferCluster.Clusters.OfType<DistributedSearchGraph>().First()));
+                }
+
+                return distCluster;
+            }
         }
+    }
 
-        var isSupported = PassUtility.IsCpuSupported(op, expr, expr.Arguments.ToArray(), _moduleKind);
-        foreach (var param in op.Parameters)
+    private DistributedSearchGraph TryAddOriginator(Expr expr)
+    {
+        if (!_inferedMemo.TryGetValue(expr, out var inferCluster))
         {
-            VisitLeafArgument(param.ParameterKind, expr.Arguments[param.Index], isSupported);
+            inferCluster = CreateOriginatorCluster(expr, true);
+            _inferedMemo.Add(expr, inferCluster);
         }
 
-        Dictionary<IRType, List<Expr>> results;
-        if (_bidirectional && isSupported)
+        if (inferCluster.Kind is SearchGraphKind.DistributedCluster)
         {
-            results = expr.Arguments.ToArray().
-                                Select(Visit).
-                                CartesianProduct().
-                                Select(args => args.ToArray()).
-                                Select(args => args.Select(kv => kv.Value[0]).Select(arg => arg.CheckedType switch
-                                {
-                                    DistributedType d => GetDiverseCandidateSBPs(d, Placements).Select(ndsbp => IR.F.CPU.Boxing(arg, new DistributedType(d.TensorType, ndsbp, d.Placement))).Concat(new[] { arg }).ToArray(),
-                                    _ => new[] { arg },
-                                }).ToList().CartesianProduct().Select(arg => BuildEquivalCalls(op, arg.ToArray())).SelectMany(i => i).ToArray()).
-                                SelectMany(i => i).
-                                GroupBy(c => c.CheckedType).
-                                ToDictionary(g => g.Key, g => g.OrderByDescending(e => e.Users.Count()).ToList<Expr>());
+            return inferCluster;
         }
-        else
+
+        // unshard to standalone
+        if (!_reshardMemo.TryGetValue(expr, out var distCluster))
         {
-            results = expr.Arguments.ToArray().
-                Select(Visit).
-                CartesianProduct().
-                Select(args => args.ToArray()).
-                Select(args => isSupported ? BuildEquivalCalls(op, args.Select(kv => kv.Value[0]).ToArray()).ToArray() :
-                                BuildNotSupportedCalls(op, args.Select(kv => kv.Value[0]).ToArray())).
-                SelectMany(i => i).
-                GroupBy(c => c.CheckedType).
-                ToDictionary(g => g.Key, g => g.OrderByDescending(e => e.Users.Count()).ToList<Expr>());
+            distCluster = CreateOriginatorCluster(expr, false);
+            _reshardMemo.Add(expr, distCluster);
         }
 
-        if (results.Count == 0)
+        if (distCluster.Kind != SearchGraphKind.DistributedCluster)
         {
-            return expr.Arguments.ToArray().
-                    Select(Visit).
-                    CartesianProduct().
-                    Select(args => args.ToArray()).
-                    Select(args => new[] { new Call(op, args.Select(kv => kv.Value[0]).Select(arg => arg.CheckedType switch
-                    {
-                        DistributedType d => d.NdSBP.All(sbp => sbp is SBPBroadCast) ? arg : IR.F.CPU.Boxing(arg, d with { NdSBP = new(Enumerable.Repeat(SBP.B, d.NdSBP.Count)) }),
-                        _ => arg,
-                    }).ToArray()), }).
-                    SelectMany(i => i).
-                    GroupBy(c => c.CheckedType).
-                    ToDictionary(g => g.Key, g => g.OrderByDescending(e => e.Users.Count()).ToList<Expr>());
-        }
-
-        // TODO: refactor here
-        if (expr.Target is not ScatterND && expr.Target is not Boxing && (expr.CheckedType is TensorType or DistributedType) && expr.CheckedShape.IsFixed && !expr.CheckedShape.ToValueArray().Contains(0) && results.Count == 1 && results.First().Key is DistributedType dt && dt.NdSBP.All(sbp => sbp is SBPBroadCast))
-        {
-            return expr.Arguments.ToArray().
-                    Select(Visit).
-                    CartesianProduct().
-                    Select(args => args.ToArray()).
-                    Select(args => args.Select(kv => kv.Value[0]).Select(arg => arg.CheckedType switch
-                    {
-                        DistributedType d => GetDiverseCandidateSBPs(d, Placements).Select(ndsbp => IR.F.CPU.Boxing(arg, new DistributedType(d.TensorType, ndsbp, d.Placement))).Concat(new[] { arg }).ToArray(),
-                        _ => new[] { arg },
-                    }).ToList().CartesianProduct().Select(arg => BuildEquivalCalls(op, arg.ToArray())).SelectMany(i => i).ToArray()).
-                    SelectMany(i => i).
-                    GroupBy(c => c.CheckedType).
-                    ToDictionary(g => g.Key, g => g.OrderByDescending(e => e.Users.Count()).ToList<Expr>());
+            throw new InvalidOperationException("The inference and reshard cluster cannot be distributed either.");
         }
 
-        FilterByScheme(expr, results);
-        return results;
+        return distCluster;
     }
 
-    private Dictionary<IRType, List<Expr>> VisitLeafArgument(ParameterKind parameterKind, Expr expr, bool isSupported)
+    private DistributedSearchGraph CreateTerminatorCluster(Expr expr, bool init)
     {
-        var updateBuckets = (Dictionary<IRType, List<Expr>> buckets, IEnumerable<Expr> equivalents) =>
+        var standCluster = _rootSearchGraph.CreateCluster<DistributedSearchGraph>(SearchGraphKind.StandaloneCluster);
+
+        if (expr is IR.Tuple tp)
         {
-            foreach (var eq in equivalents)
+            var buckets = new DistributedSearchGraph[tp.Fields.Length];
+            foreach (var (f, fGraph, i) in tp.Fields.AsValueEnumerable().Select((f, i) => (f, Visit(f), i)))
             {
-                if (!buckets.TryGetValue(eq.CheckedType, out var bucket))
-                {
-                    bucket = new();
-                    buckets.Add(eq.CheckedType, bucket);
-                }
-
-                bucket.Add(eq);
+                buckets[i] = TryInstertTerminator(f).Clusters.OfType<DistributedSearchGraph>().First();
             }
 
-            FilterByScheme(expr, buckets);
-        };
-
-        var buckets = ExprMemo[expr];
-        if (!buckets.Any())
-        {
-            switch (parameterKind, expr)
+            var tpnode = new SearchableNode(new IR.Tuple(), new TupleType(buckets.Select(g => g.Vertices.First().IRType).ToArray()));
+            var bucket = standCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+            bucket.AddVertex(tpnode);
+            for (int i = 0; i < tp.Fields.Length; i++)
             {
-                case (ParameterKind.Input, Expr e) when e is Const or Var:
-                    updateBuckets(buckets, isSupported ? GetLeafCandidateBoxings(e, Placements) : new[] { e });
-                    break;
-                case (ParameterKind.Input, Expr e) when e is IR.Tuple tp:
-                    foreach (var f in tp.Fields)
-                    {
-                        VisitLeafArgument(parameterKind, f, isSupported);
-                    }
-
-                    foreach (var (k, v) in VisitLeafTuple(tp))
-                    {
-                        buckets.Add(k, v);
-                    }
-
-                    break;
-                case (ParameterKind.Attribute, Var e):
-                    updateBuckets(buckets, new[] { e });
-                    break;
-                case (ParameterKind.Attribute, TensorConst e):
-                    updateBuckets(buckets, new[] { e.With() }); // remove all old users.
-                    break;
-                case (ParameterKind.Attribute, None e):
-                    updateBuckets(buckets, new[] { e.With() });
-                    break;
-                default:
-                    throw new InvalidOperationException();
+                _rootSearchGraph.AddEdge(new(tpnode, buckets[i].Vertices.First(), i, buckets[i]));
             }
         }
-        else if (parameterKind == ParameterKind.Input)
+        else
         {
-            if (isSupported)
+            if (init)
             {
-                if (!buckets.Keys.Any(IsDistributed))
-                {
-                    var results = buckets.Select(kv => GetLeafCandidateBoxings(kv.Value[0], Placements)).SelectMany(i => i).ToArray();
-                    updateBuckets(buckets, results);
-                }
+                var bucket = standCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+                var node = new SearchableNode(expr, expr.CheckedType);
+                bucket.AddVertex(node);
             }
             else
             {
-                if (buckets.Keys.All(IsDistributed))
+                var onode = new SearchableNode(new Boxing(expr.CheckedType), expr.CheckedType);
+                var inputBuckets = _inferedMemo[expr].Clusters.OfType<DistributedSearchGraph>().ToArray();
+
+                var bucket = standCluster.CreateCluster<DistributedSearchGraph>(SearchGraphKind.Bucket);
+                bucket.AddVertex(onode);
+                foreach (var inputBucket in inputBuckets)
                 {
-                    var results = buckets.Select(kv => InstertTerminator(kv.Value[0])).ToArray();
-                    updateBuckets(buckets, results);
+                    if (Evaluator.IR.CPU.BoxingEvaluator.VisitType(inputBucket.Vertices.First().IRType, onode.IRType) is not InvalidType)
+                    {
+                        _rootSearchGraph.AddEdge(new(onode, inputBucket.Vertices.First(), 0, inputBucket));
+                    }
                 }
             }
         }
 
-        return buckets;
+        return standCluster;
     }
 
-    private Call[] BuildNotSupportedCalls(Op target, Expr[] args)
+    private DistributedSearchGraph TryInstertTerminator(Expr expr)
     {
-        if (target.Parameters.Where(p => p.ParameterKind == ParameterKind.Input).Any(p => IsDistributed(args[p.Index].CheckedType)))
+        if (!_inferedMemo.TryGetValue(expr, out var inferCluster))
         {
-            return Array.Empty<Call>();
+            inferCluster = CreateTerminatorCluster(expr, true);
+            _inferedMemo.Add(expr, inferCluster);
+            return inferCluster;
         }
 
-        return new[] { new Call(target, args) };
-    }
+        if (inferCluster.Kind is SearchGraphKind.StandaloneCluster)
+        {
+            return inferCluster;
+        }
 
-    private IEnumerable<Call> BuildEquivalCalls(Op target, Expr[] args)
-    {
-        if (!target.Parameters.Where(p => p.ParameterKind == ParameterKind.Input).All(p => IsDistributed(args[p.Index].CheckedType)))
+        // unshard to standalone
+        if (!_reshardMemo.TryGetValue(expr, out var standCluster))
+        {
+            standCluster = CreateTerminatorCluster(expr, false);
+            _reshardMemo.Add(expr, standCluster);
+            return standCluster;
+        }
+
+        if (standCluster.Kind != SearchGraphKind.StandaloneCluster)
         {
-            return Array.Empty<Call>();
+            throw new InvalidOperationException("The inference and reshard cluster cannot be distributed either.");
         }
 
-        var calls = new List<Call>();
-        var call = new Call(target, args);
-        var valid = call.InferenceType();
-        if (!valid)
+        return standCluster;
+    }
+
+    private void Dump(Stream stream, IReadOnlyDictionary<SearchableNode, bool> pickMemo, IReadOnlyDictionary<SearchableNode, CostModel.Cost> costMemo)
+    {
+        using var writer = new StreamWriter(stream);
+        writer.Write(_rootSearchGraph.ToGraphviz(alg =>
         {
-            // 1. dispose current call
-            using var pinner = new ExprPinner(args);
-            call.Dispose();
+            alg.GraphFormat.RankDirection = QuikGraph.Graphviz.Dot.GraphvizRankDirection.LR;
+            alg.FormatCluster += (_, arg) =>
+            {
+                if (arg.Cluster is DistributedSearchGraph tg)
+                {
+                    arg.GraphFormat.LabelLocation = QuikGraph.Graphviz.Dot.GraphvizLabelLocation.T;
+                    arg.GraphFormat.LabelJustification = QuikGraph.Graphviz.Dot.GraphvizLabelJustification.L;
+                    arg.GraphFormat.Label = tg.Kind.ToString();
+                    if (tg.Kind is SearchGraphKind.Bucket)
+                    {
+                        arg.GraphFormat.Label += ": " + tg.Vertices.First().IRType.ToString();
+                    }
+                }
+            };
 
-            if (target is Reshape)
+            alg.FormatVertex += (_, arg) =>
             {
-                // the reshape need force boxing.
-                var newShape = ((TensorConst)args[1]).Value.ToArray<int>();
-                var inType = (DistributedType)args[0].CheckedType;
-                var tensorType = inType.TensorType with { Shape = newShape };
-                foreach (var boxing in Utilities.DistributedUtility.GetLeafCandidateNDSBPs(tensorType, inType.Placement).
-                    Select(ndsbp => IR.F.CPU.Boxing(args[0], new DistributedType(tensorType, ndsbp, inType.Placement), true)))
+                var row0 = new QuikGraph.Graphviz.Dot.GraphvizRecordCell();
+                var col1 = new QuikGraph.Graphviz.Dot.GraphvizRecordCell();
+                row0.Cells.Add(col1);
+
+                col1.Cells.Add(new() { Text = CompilerServices.Print(arg.Vertex.Expr) });
+                if (arg.Vertex.Expr is IR.Tuple && arg.Vertex.IRType is TupleType tpTuple)
                 {
-                    if (boxing.CheckedType is InvalidType)
+                    for (int i = 0; i < tpTuple.Fields.Count; i++)
                     {
-                        boxing.Dispose();
+                        col1.Cells.Add(new() { Text = i.ToString(), Port = $"P{i}" });
                     }
-                    else
+                }
+                else if (arg.Vertex.Expr is Op op)
+                {
+                    for (int i = 0; i < op.Parameters.Count(); i++)
+                    {
+                        col1.Cells.Add(new() { Text = i.ToString(), Port = $"P{i}" });
+                    }
+                }
+
+                arg.VertexFormat.Record.Cells.Add(row0);
+                arg.VertexFormat.Shape = QuikGraph.Graphviz.Dot.GraphvizVertexShape.Record;
+                arg.VertexFormat.Style = QuikGraph.Graphviz.Dot.GraphvizVertexStyle.Filled;
+                if (costMemo.TryGetValue(arg.Vertex, out var cost))
+                {
+                    var row1 = new QuikGraph.Graphviz.Dot.GraphvizRecordCell();
+                    foreach (var (k, v) in cost.Factors)
+                    {
+                        row1.Cells.Add(new() { Text = $"{k}: {v}" });
+                    }
+
+                    row1.Cells.Add(new() { Text = $"Score: {cost.Score}" });
+                    col1.Cells.Add(row1);
+                }
+
+                if (pickMemo.TryGetValue(arg.Vertex, out var picked) && picked == true)
+                {
+                    arg.VertexFormat.FillColor = QuikGraph.Graphviz.Dot.GraphvizColor.SkyBlue;
+                }
+            };
+
+            alg.FormatEdge += (_, arg) =>
+            {
+                arg.EdgeFormat.Direction = QuikGraph.Graphviz.Dot.GraphvizEdgeDirection.Back;
+                arg.EdgeFormat.TailPort = $"P{arg.Edge.InputIndex}";
+            };
+        }));
+    }
+
+    private Expr SolveAndExtract(DistributedSearchGraph rootCluster)
+    {
+        // 0. create bool var for all node.
+        var cpmodel = new CpModel();
+        var varMemo = new Dictionary<SearchableNode, BoolVar>();
+        var costMemo = new Dictionary<SearchableNode, CostModel.Cost>();
+        foreach (var cluster in _rootSearchGraph.Clusters.OfType<DistributedSearchGraph>())
+        {
+            foreach (var bucket in cluster.Clusters.OfType<DistributedSearchGraph>())
+            {
+                foreach (var enode in bucket.Vertices)
+                {
+                    CostModel.Cost cost;
+                    switch (enode.Expr)
                     {
-                        calls.Add(boxing);
+                        case Const or Var or If or IR.Tuple or BaseFunction:
+                            cost = new CostModel.Cost() { [CostModel.CostFactorNames.CPUCycles] = 1 };
+                            break;
+                        case Op op:
+                            {
+                                if (!_rootSearchGraph.TryGetOutEdges(enode, out var edges))
+                                {
+                                    throw new NotSupportedException("graph doesn't contain the vertex.");
+                                }
+
+                                var tempArgs = edges.OrderBy(e => e.InputIndex).Select<CrossEdge, Expr>(e => e.Target switch
+                                {
+                                    SearchableNode { Expr: Const attr } => attr,
+                                    SearchableNode n => new Var(n.IRType),
+                                }).ToArray();
+
+                                var context = new DistributedCostEvaluateContext(op, enode.IRType, tempArgs, CompileOptions);
+                                cost = CompilerServices.EvaluateOpCost(op, context);
+                            }
+
+                            break;
+                        default:
+                            throw new NotSupportedException($"extract not support {enode.Expr.GetType()}");
                     }
+
+                    costMemo.Add(enode, cost);
+
+                    var boolVar = cpmodel.NewBoolVar(string.Empty);
+                    varMemo.Add(enode, boolVar);
                 }
             }
-            else
+        }
+
+        // 1. must pick one in root enode.
+        cpmodel.AddBoolOr(rootCluster.Vertices.Select(n => varMemo[n]).ToArray());
+
+        // 2. when pick node, must pick one child node.
+        foreach (var n in _rootSearchGraph.Vertices)
+        {
+            var ns = new[] { varMemo[n].Not() };
+
+            if (_rootSearchGraph.TryGetOutEdges(n, out var allEdges))
             {
-                // todo expand search space.
-                // calls.AddRange(Utilities.DistributedUtility.GetLeafCandidateNDSBPs(tensorType, inType.Placement).
-                // Select(ndsbp => IR.F.CPU.Boxing(args[0], new DistributedType(tensorType, ndsbp, inType.Placement))));
+                foreach (var argEdges in allEdges.GroupBy(g => g.InputIndex))
+                {
+                    cpmodel.AddBoolOr(ns.Concat(argEdges.SelectMany(e => e.InputGraph.Vertices).Select(cn => varMemo[cn])));
+                }
             }
         }
-        else
+
+        // 3. no cycle
+        foreach (var cluster in _rootSearchGraph.Clusters.OfType<DistributedSearchGraph>())
         {
-            calls.Add(call);
-            if (call.CheckedType is DistributedType distType)
+            foreach (var sourceBucket in cluster.Clusters.OfType<DistributedSearchGraph>())
             {
-                // boxing for partialsum
-                var partialBoxings = Utilities.DistributedUtility.GetPartialCandidateNDSBPs(distType).
-                    Select(ndsbp => (ndsbp, IR.F.CPU.Boxing(call, distType with { NdSBP = ndsbp }))).Select(p =>
-                    {
-                        var lastSbp = p.ndsbp;
-                        var reduced = p.Item2;
-                        return Utilities.DistributedUtility.GetLeafCandidateNDSBPs(distType.TensorType, distType.Placement).Where(ndsbp => lastSbp != ndsbp).Select(ndsbp => IR.F.CPU.Boxing(reduced, distType with { NdSBP = ndsbp })).ToArray();
-                    }).SelectMany(i => i).ToArray();
-                calls.AddRange(partialBoxings);
-
-                using var pinner = new ExprPinner(calls.ToArray());
-                var getExtraBoxings = (Expr expr) => Placements.
-                    Where(p => p != distType.Placement).
-                    Select(p => Utilities.DistributedUtility.GetLeafCandidateNDSBPs(distType.TensorType, p).
-                        Select(ndsbp => IR.F.CPU.Boxing(expr, new DistributedType(distType.TensorType, ndsbp, p)))).
-                    SelectMany(b => b);
-
-                // boxing for other placements
-                var extraBoxings = partialBoxings.Any() ? partialBoxings.Select(getExtraBoxings).SelectMany(i => i) : getExtraBoxings(call);
-                foreach (var boxing in extraBoxings)
+                foreach (var destBucket in cluster.Clusters.OfType<DistributedSearchGraph>().Where(b => !ReferenceEquals(b, sourceBucket)))
                 {
-                    if (boxing.CheckedType is InvalidType)
+                    foreach (var (src, dest) in sourceBucket.Vertices.Where(v => v.IsBidirect).Zip(destBucket.Vertices.Where(v => v.IsBidirect)))
                     {
-                        boxing.Dispose();
-                    }
-                    else
-                    {
-                        calls.Add(boxing);
+                        cpmodel.AddBoolAnd([varMemo[src].Not(), varMemo[dest].Not()]);
                     }
                 }
             }
         }
 
-        return calls;
+        // 3. add pick weights for all enode.
+        cpmodel.Minimize(LinearExpr.WeightedSum(_rootSearchGraph.Vertices.Select(n => varMemo[n]), _rootSearchGraph.Vertices.Select(n => checked((long)costMemo[n].Score))));
+
+        if (cpmodel.Validate().Any())
+        {
+            throw new InvalidDataException("the sat model invalid: " + cpmodel.Validate());
+        }
+
+        var solver = new CpSolver();
+        int max_time = 120;
+        if (System.Environment.GetEnvironmentVariable("SOLVE_MAX_TIME") is string s_solve_max_time)
+        {
+            try
+            {
+                var solve_max_time = int.Parse(s_solve_max_time);
+                max_time = solve_max_time;
+            }
+            catch (System.Exception)
+            {
+            }
+        }
+
+        int processorCount = Math.Max(System.Environment.ProcessorCount / 2, 1);
+        if (System.Environment.GetEnvironmentVariable("SOLVE_PROCESSOR_COUNT") is string s_solve_processor_count)
+        {
+            try
+            {
+                var solve_processor_count = int.Parse(s_solve_processor_count);
+                processorCount = solve_processor_count;
+            }
+            catch (System.Exception)
+            {
+            }
+        }
+
+        solver.StringParameters = $"max_time_in_seconds:{max_time},num_workers:{processorCount}";
+
+        var enableDump = Diagnostics.DumpScope.Current.IsEnabled(Diagnostics.DumpFlags.EGraphCost);
+        CpSolverStatus status;
+        using (var dumpStream = enableDump ? Diagnostics.DumpScope.Current.OpenFile("Costs/Solve.txt") : Stream.Null)
+        {
+            using var writer = new StreamWriter(dumpStream);
+            var cb = new PrintCostCallBack(varMemo, costMemo, writer, enableDump);
+            status = solver.Solve(cpmodel, cb);
+            writer.WriteLine($"Status : {status}");
+        }
+
+        if (status is not (CpSolverStatus.Optimal or CpSolverStatus.Feasible))
+        {
+            throw new InvalidProgramException("SatExtract Failed!");
+        }
+
+        var picks = _rootSearchGraph.Vertices.ToDictionary(e => e, e => solver.BooleanValue(varMemo[e]));
+        using (var stream = enableDump ? Diagnostics.DumpScope.Current.OpenFile("Costs/Pick.dot") : Stream.Null)
+        {
+            Dump(stream, picks, costMemo);
+        }
+
+        return new ExprBuildVisitor(_rootSearchGraph, picks).Visit(rootCluster.Clusters.OfType<DistributedSearchGraph>());
     }
 
-    private IReadOnlyList<Expr> GetReBoxings(Expr expr)
+    private HyperGraph<DistributedSearchGraph, SearchableNode> ToHyperGraph(DistributedSearchGraph root, DistributedSearchGraph rootCluster)
     {
-        if (expr is IR.Tuple tuple)
+        var hgraph = new HyperGraph<DistributedSearchGraph, SearchableNode>();
+        var visited = new HashSet<DistributedSearchGraph>();
+        var queue = new Queue<DistributedSearchGraph>();
+        var rootBuckets = rootCluster.Clusters.OfType<DistributedSearchGraph>().ToArray();
+        if (rootBuckets.Length != 1)
         {
-            var candidates = tuple.Fields.ToArray().
-                Select(GetReBoxings).
-                CartesianProduct();
-            return candidates.Any() ? candidates.
-                Select(fs => new IR.Tuple(fs.ToArray())).
-                ToArray() : Array.Empty<Expr>();
+            throw new InvalidOperationException("The root Cluster should contains only one bucket!");
         }
 
-        var type = (DistributedType)expr.CheckedType;
-        var tensorType = type.TensorType;
-        var candidateNdsbps = new List<SBP>[type.Placement.Rank];
-        for (int i = 0; i < type.Placement.Rank; i++)
+        queue.Enqueue(rootBuckets[0]);
+        visited.Add(rootBuckets[0]);
+        while (queue.Any())
         {
-            candidateNdsbps[i] = new List<SBP> { SBP.B };
-            for (int axis = 0; axis < tensorType.Shape.Rank; axis++)
+            var front = queue.Dequeue();
+            foreach (var node in front.Vertices)
             {
-                if (tensorType.Shape[axis] is { IsFixed: true, Value: int s } && Utilities.DistributedUtility.IsDivideExactly(s, type.Placement.Hierarchy[i]))
+                root.TryGetOutEdges(node, out var edges);
+                foreach (var edge in edges)
                 {
-                    candidateNdsbps[i].Add(SBP.S(axis));
+                    var canonical = edge.InputGraph;
+                    hgraph.Connect(front, canonical, node);
+                    if (!visited.Contains(canonical))
+                    {
+                        visited.Add(canonical);
+                        queue.Enqueue(canonical);
+                    }
                 }
             }
         }
 
-        return candidateNdsbps.CartesianProduct().
-            Select(ndsbp => new IRArray<SBP>(ndsbp)).
-            Where(ndsbp => ndsbp != type.NdSBP).
-            Select(ndsbp => new DistributedType(tensorType, new IRArray<SBP>(ndsbp), type.Placement)).
-            Select(disttype => IR.F.CPU.Boxing(expr, disttype)).ToArray();
+        return hgraph;
     }
+}
 
-    private bool IsDistributed(IRType type) => type switch
+internal sealed class ExprBuildVisitor
+{
+    private readonly Dictionary<SearchableNode, bool> _picks;
+    private readonly DistributedSearchGraph _rootSearchGraph;
+    private readonly Dictionary<SearchableNode, Expr> _memo;
+
+    public ExprBuildVisitor(DistributedSearchGraph rootSearchGraph, Dictionary<SearchableNode, bool> picks)
     {
-        DistributedType => true,
-        TupleType t => t.All(IsDistributed),
-        _ => false,
-    };
+        _rootSearchGraph = rootSearchGraph;
+        _picks = picks;
+        _memo = new();
+    }
 
-    private Expr InstertTerminator(Expr expr)
+    public Expr Visit(IEnumerable<DistributedSearchGraph> rootBuckets)
     {
-        Expr CreateFinalBoxing(Expr e, DistributedType type)
+        var rootPicks = rootBuckets.SelectMany(b => b.Vertices).Where(v => _picks.TryGetValue(v, out var pick) && pick).ToArray();
+        if (rootPicks.Length != 1)
+        {
+            throw new InvalidProgramException("the one cluster only can pick one vertex!");
+        }
+
+        var root = rootPicks[0];
+        if (!_memo.TryGetValue(root, out var expr))
         {
-            if (type.NdSBP.Any(s => s is SBPPartialSum))
+            _rootSearchGraph.TryGetOutEdges(root, out var edges);
+            var children = edges.GroupBy(e => e.InputIndex).Select(g => Visit(g.Select(e => e.InputGraph))).ToArray();
+            switch (root.Expr)
             {
-                var boxingP2B = IR.F.CPU.Boxing(e, new DistributedType(type.TensorType, type.NdSBP.Select(s => s is SBPPartialSum ? SBP.B : s).ToArray(), type.Placement));
-                return IR.F.CPU.Boxing(boxingP2B, type.TensorType);
+                case Var or TensorConst or TupleConst or None:
+                    expr = root.Expr;
+                    break;
+                case BaseFunction func:
+                    expr = new Call(target: func, arguments: children);
+                    break;
+                case Op op:
+                    expr = new Call(target: op, arguments: children);
+                    break;
+                case IR.Tuple tp:
+                    expr = tp.With(fields: children);
+                    break;
+                case IR.If @if:
+                    expr = @if.With(condition: children[^3], then: children[^2], @else: children[^1], paramList: children[..^3].ToArray());
+                    break;
+                default:
+                    throw new NotSupportedException(root.Expr.GetType().Name);
             }
 
-            return IR.F.CPU.Boxing(e, type.TensorType);
+            _memo.Add(root, expr);
         }
 
-        return (expr, expr.CheckedType) switch
-        {
-            (IR.Tuple tp, TupleType tptype) => new IR.Tuple(tp.Fields.ToArray().Select(InstertTerminator).ToArray()),
-            (Expr e, DistributedType type) => CreateFinalBoxing(e, type),
-            (Expr e, TensorType type) => e,
-            (Expr e, AnyType type) => e,
-            (_, _) => throw new NotSupportedException(),
-        };
+        return expr;
+    }
+}
+
+internal sealed class DistributedCostEvaluateContext : Evaluator.ICostEvaluateContext
+{
+    public DistributedCostEvaluateContext(Op op, IRType returnType, Expr[] args, CompileOptions compileOptions)
+    {
+        Op = op;
+        ReturnType = returnType;
+        Args = args;
+        CompileOptions = compileOptions;
+    }
+
+    public Op Op { get; }
+
+    public IRType ReturnType { get; }
+
+    public Expr[] Args { get; }
+
+    public CompileOptions CompileOptions { get; }
+
+    public T GetArgument<T>(Op op, ParameterInfo parameter)
+        where T : BaseFunction
+    {
+        throw new NotSupportedException();
     }
 
-    private EClass Unions(EGraph graph, IEnumerable<Expr> equivalents)
+    public T GetArgumentType<T>(Op op, ParameterInfo parameter)
+        where T : IRType
     {
-        var eids = equivalents.Select(graph.Add).ToArray();
-        foreach (var cls in eids.Skip(1))
+        if (op.GetType() == parameter.OwnerType)
         {
-            graph.Union(eids[0], cls);
+            return (T?)Args[parameter.Index].CheckedType ?? throw new InvalidOperationException("Run type infer first.");
         }
+        else
+        {
+            throw new ArgumentOutOfRangeException($"Operator {op} doesn't have parameter: {parameter.Name}.");
+        }
+    }
 
-        graph.Rebuild();
-        return eids[0];
+    public T GetReturnType<T>()
+         where T : IRType
+    {
+        return (T)ReturnType;
+    }
+}
+
+internal sealed class PrintCostCallBack : CpSolverSolutionCallback
+{
+    private readonly IReadOnlyDictionary<SearchableNode, BoolVar> _vars;
+    private readonly Dictionary<SearchableNode, CostModel.Cost> _costModel;
+    private readonly StreamWriter _dumpWriter;
+    private readonly bool _enableDump;
+    private int _count;
+
+    public PrintCostCallBack(IReadOnlyDictionary<SearchableNode, BoolVar> vars, Dictionary<SearchableNode, CostModel.Cost> costModel, StreamWriter writer, bool enableDump)
+    {
+        _vars = vars;
+        _costModel = costModel;
+        _dumpWriter = writer;
+        _enableDump = enableDump;
     }
 
-    private void BranchCut()
+    public override void OnSolutionCallback()
     {
-        bool changed = true;
-        while (changed)
+        if (_enableDump)
         {
-            changed = false;
-            foreach (var (e, bukets) in ExprMemo)
+            var cost = CostModel.Cost.Zero;
+            foreach (var (n, v) in _vars)
             {
-                foreach (var (_, buket) in bukets.Where(kv => kv.Value.Any()))
+                if (_costModel[n] != CostModel.Cost.Zero && BooleanValue(v))
                 {
-                    if (!buket[0].Users.Any())
-                    {
-                        foreach (var item in buket)
-                        {
-                            if (item.Users.Any())
-                            {
-                                throw new InvalidOperationException("this item can't have more than zero users!");
-                            }
-
-                            using (new ExprPinner(item.Operands.ToArray()))
-                            {
-                                item.Dispose();
-                            }
-                        }
-
-                        buket.Clear();
-                        changed = true;
-                    }
+                    cost += _costModel[n];
                 }
             }
+
+            _dumpWriter.WriteLine($"Solution {_count++} @ {WallTime()}:");
+            _dumpWriter.WriteLine(cost.ToString());
+            _dumpWriter.Flush();
         }
     }
 }
diff --git a/modules/Nncase.Modules.CPU/Passes/Distributed/DistributedScheme.cs b/modules/Nncase.Modules.CPU/Passes/Distributed/DistributedSchema.cs
similarity index 75%
rename from modules/Nncase.Modules.CPU/Passes/Distributed/DistributedScheme.cs
rename to modules/Nncase.Modules.CPU/Passes/Distributed/DistributedSchema.cs
index 7e7a4c3d5..3cb03dac2 100644
--- a/modules/Nncase.Modules.CPU/Passes/Distributed/DistributedScheme.cs
+++ b/modules/Nncase.Modules.CPU/Passes/Distributed/DistributedSchema.cs
@@ -3,7 +3,7 @@
 
 namespace Nncase.Passes.Distributed;
 
-public record class DistributedScheme(string Version, string Model, DistributedScheme.Node[] Outputs)
+public record class DistributedSchema(string Version, string Model, DistributedSchema.Node[] Outputs)
 {
     public record class Node(string Name, IR.SBP[] NdSBP, int[] Hierarchy, string HierarchyName)
     {
diff --git a/modules/Nncase.Modules.CPU/Passes/Rules/CPU/FoldMatmulReduce.cs b/modules/Nncase.Modules.CPU/Passes/Rules/CPU/FoldMatmulReduce.cs
index a800d4a6c..d99c28473 100644
--- a/modules/Nncase.Modules.CPU/Passes/Rules/CPU/FoldMatmulReduce.cs
+++ b/modules/Nncase.Modules.CPU/Passes/Rules/CPU/FoldMatmulReduce.cs
@@ -15,7 +15,7 @@ public sealed partial class FoldPackedMatmulReduce : IRewriteRule
     public IPattern Pattern { get; } =
         IsBoxing(
             target_name: "boxing",
-            op => op.NewType is DistributedType dt && dt.NdSBP.All(s => s != SBP.P),
+            op => op.NewType is DistributedType dt && dt.NdSBP.All(s => s is not SBPPartial),
             IsPackedMatMul(
                 "mm",
                 "call",
@@ -25,7 +25,7 @@ public sealed partial class FoldPackedMatmulReduce : IRewriteRule
 
     public Expr? GetReplace(Call call, PackedMatMul mm, Expr lhs, Expr rhs)
     {
-        if (call.CheckedType is DistributedType dt && dt.NdSBP.Any(s => s == SBP.P))
+        if (call.CheckedType is DistributedType dt && dt.NdSBP.Any(s => s is SBPPartial))
         {
             var newMatmul = new IR.CPU.PackedMatMul(mm.LhsPackedAxes, mm.LhsPadedNums, mm.RhsPackedAxes, mm.RhsPadedNums, mm.TransposeA, mm.TransposeB, true);
             return new Call(newMatmul, lhs, rhs);
@@ -41,7 +41,7 @@ public sealed partial class SwapUnpackReduce : IRewriteRule
     public IPattern Pattern { get; } =
         IsBoxing(
             target_name: "boxing",
-            op => op.NewType is DistributedType dt && dt.NdSBP.All(s => s != SBP.P),
+            op => op.NewType is DistributedType dt && dt.NdSBP.All(s => s is not SBPPartial),
             IsUnpack(
                 target_name: "unpack",
                 _ => true,
@@ -54,10 +54,10 @@ public sealed partial class SwapUnpackReduce : IRewriteRule
 
     public Expr? GetReplace(Call call, Boxing boxing, Unpack unpack)
     {
-        if (call.CheckedType is DistributedType dt && dt.NdSBP.Any(s => s == SBP.P))
+        if (call.CheckedType is DistributedType dt && dt.NdSBP.Any(s => s is SBPPartial))
         {
-            var newType = new DistributedType(dt.TensorType, dt.NdSBP.Select(s => s is SBPPartialSum ? SBP.B : s).ToArray(), dt.Placement);
-            var newBoxing = IR.F.CPU.Boxing(call, newType, boxing.IsReshape);
+            var newType = new DistributedType(dt.TensorType, dt.NdSBP.Select(s => s is SBPPartial ? SBP.B : s).ToArray(), dt.Placement);
+            var newBoxing = IR.F.CPU.Boxing(call, newType);
             return IR.F.CPU.Unpack(newBoxing, [.. unpack.Lanes], [.. unpack.Axes]);
         }
 
diff --git a/modules/Nncase.Modules.CPU/Passes/Rules/CPU/PackRule.cs b/modules/Nncase.Modules.CPU/Passes/Rules/CPU/PackRule.cs
index e74db57b8..244d2d8d0 100644
--- a/modules/Nncase.Modules.CPU/Passes/Rules/CPU/PackRule.cs
+++ b/modules/Nncase.Modules.CPU/Passes/Rules/CPU/PackRule.cs
@@ -768,12 +768,12 @@ public override List<Expr> GetReplaceCandidates(IMatchResult result, RunPassCont
         var inShape = input.CheckedShape.ToValueArray();
 
         // 1. find the mapping transforms
-        if (!PackUtility.TryGetShapeMapMatrix(inShape, newShape, out var mat))
+        if (!IRUtility.TryGetShapeMapMatrix(inShape, newShape, out var mat))
         {
             return new List<Expr> { };
         }
 
-        var (forwardDict, backwardDict) = PackUtility.ShapeMapMatrixAsDict(mat);
+        var (forwardDict, backwardDict) = IRUtility.ShapeMapMatrixAsDict(mat);
 
         void AddCandidate(int[] packedAxes, int[] lanes)
         {
diff --git a/modules/Nncase.Modules.CPU/Passes/Tile/KernelToTIRVisitor.cs b/modules/Nncase.Modules.CPU/Passes/Tile/KernelToTIRVisitor.cs
index 8d9666599..44249ca11 100644
--- a/modules/Nncase.Modules.CPU/Passes/Tile/KernelToTIRVisitor.cs
+++ b/modules/Nncase.Modules.CPU/Passes/Tile/KernelToTIRVisitor.cs
@@ -96,6 +96,9 @@ protected override Unit VisitLeafCall(Call expr)
             case IR.CPU.Boxing boxing:
                 GenerateBoxing(boxing, arguments, ret, expr);
                 break;
+            case IR.CPU.ForceBoxing forceBoxing:
+                _mainBody.Add(T.Memcopy(ret, arguments[0]));
+                break;
             case Binary binary:
                 GenerateBinary(binary, arguments, ret, expr);
                 break;
diff --git a/modules/Nncase.Modules.CPU/Targets/CPUTarget.cs b/modules/Nncase.Modules.CPU/Targets/CPUTarget.cs
index 7506ad5b1..757c0c1bb 100644
--- a/modules/Nncase.Modules.CPU/Targets/CPUTarget.cs
+++ b/modules/Nncase.Modules.CPU/Targets/CPUTarget.cs
@@ -114,12 +114,7 @@ public void RegisterTargetDependentAfterQuantPass(IPassManager passManager, Comp
             });
         }
 
-        // need refactor tiling.
-        passManager.Add<Passes.Distributed.AutoDistributedPass>(true, "cpu");
-        passManager.AddWithName<DataflowPass>("FoldBoxing").Configure(p =>
-        {
-            p.Add<Passes.Rules.Neutral.FoldConstCall>();
-        });
+        passManager.Add<Passes.Distributed.AutoDistributedPass>(true, Kind);
 
         passManager.Add<CPUFunctionPartitionPass>();
 
diff --git a/modules/Nncase.Modules.CPU/Targets/CPUTargetOptions.cs b/modules/Nncase.Modules.CPU/Targets/CPUTargetOptions.cs
index 28dd50284..d0f14b2ef 100644
--- a/modules/Nncase.Modules.CPU/Targets/CPUTargetOptions.cs
+++ b/modules/Nncase.Modules.CPU/Targets/CPUTargetOptions.cs
@@ -83,11 +83,23 @@ public class CpuTargetOptions : ICpuTargetOptions
     [CommandLine.AllowMultiplePerToken]
     public int[] HierarchySizes { get; set; } = new[] { 1 * (int)MathF.Pow(2, 30) };
 
+    [DisplayName("--hierarchy-latencies")]
+    [Description("the latency of hierarchies.")]
+    [DefaultValue("() => new int[] { 10000 }")]
+    [CommandLine.AllowMultiplePerToken]
+    public int[] HierarchyLatencies { get; set; } = new[] { 10000 };
+
+    [DisplayName("--hierarchy-bandwiths")]
+    [Description("the bandwidth of hierarchies.")]
+    [DefaultValue("() => new int[] { 1 }")]
+    [CommandLine.AllowMultiplePerToken]
+    public int[] HierarchyBandWidths { get; set; } = new[] { 1 };
+
     [DisplayName("--memory-capacities")]
     [Description("the memory capacity of single core. eg. `32 64` for sram,main")]
-    [DefaultValue("() => new int[] { 65536, 2147483647 }")]
+    [DefaultValue("() => new int[] { 524288, 2147483647 }")]
     [CommandLine.AllowMultiplePerToken]
-    public int[] MemoryCapacities { get; set; } = new[] { 65536, int.MaxValue };
+    public int[] MemoryCapacities { get; set; } = new[] { 524288, int.MaxValue };
 
     [DisplayName("--memory-bandwidths")]
     [Description("the memory bandwidth of single core. eg. `64 8` for sram,main")]
diff --git a/modules/Nncase.Modules.CPU/Targets/CPUTargetOptionsCommand.cs b/modules/Nncase.Modules.CPU/Targets/CPUTargetOptionsCommand.cs
index 6761b93f2..3bb2fb877 100644
--- a/modules/Nncase.Modules.CPU/Targets/CPUTargetOptionsCommand.cs
+++ b/modules/Nncase.Modules.CPU/Targets/CPUTargetOptionsCommand.cs
@@ -73,10 +73,26 @@ public CpuTargetOptionsCommand(string name)
             AllowMultipleArgumentsPerToken = true,
         };
         Add(HierarchySizesOption);
+        HierarchyLatenciesOption = new Option<IEnumerable<int>>(
+            name: "--hierarchy-latencies",
+            description: "the latency of hierarchies.",
+            getDefaultValue: () => new int[] { 10000 })
+        {
+            AllowMultipleArgumentsPerToken = true,
+        };
+        Add(HierarchyLatenciesOption);
+        HierarchyBandWidthsOption = new Option<IEnumerable<int>>(
+            name: "--hierarchy-bandwiths",
+            description: "the bandwidth of hierarchies.",
+            getDefaultValue: () => new int[] { 1 })
+        {
+            AllowMultipleArgumentsPerToken = true,
+        };
+        Add(HierarchyBandWidthsOption);
         MemoryCapacitiesOption = new Option<IEnumerable<int>>(
             name: "--memory-capacities",
             description: "the memory capacity of single core. eg. `32 64` for sram,main",
-            getDefaultValue: () => new int[] { 65536, 2147483647 })
+            getDefaultValue: () => new int[] { 524288, 2147483647 })
         {
             AllowMultipleArgumentsPerToken = true,
         };
@@ -119,6 +135,10 @@ public CpuTargetOptionsCommand(string name)
 
     public Option<IEnumerable<int>> HierarchySizesOption { get; }
 
+    public Option<IEnumerable<int>> HierarchyLatenciesOption { get; }
+
+    public Option<IEnumerable<int>> HierarchyBandWidthsOption { get; }
+
     public Option<IEnumerable<int>> MemoryCapacitiesOption { get; }
 
     public Option<IEnumerable<int>> MemoryBandWidthsOption { get; }
@@ -150,6 +170,8 @@ public CpuTargetOptions GetBoundValue(InvocationContext context)
             Hierarchies = context.ParseResult.GetValueForOption(_cmd.HierarchiesOption)!.ToArray(),
             HierarchyNames = context.ParseResult.GetValueForOption(_cmd.HierarchyNamesOption)!,
             HierarchySizes = context.ParseResult.GetValueForOption(_cmd.HierarchySizesOption)!.ToArray(),
+            HierarchyLatencies = context.ParseResult.GetValueForOption(_cmd.HierarchyLatenciesOption)!.ToArray(),
+            HierarchyBandWidths = context.ParseResult.GetValueForOption(_cmd.HierarchyBandWidthsOption)!.ToArray(),
             MemoryCapacities = context.ParseResult.GetValueForOption(_cmd.MemoryCapacitiesOption)!.ToArray(),
             MemoryBandWidths = context.ParseResult.GetValueForOption(_cmd.MemoryBandWidthsOption)!.ToArray(),
             DistributedScheme = context.ParseResult.GetValueForOption(_cmd.DistributedSchemeOption)!,
diff --git a/modules/Nncase.Modules.CPU/Utilities/PackUtility.cs b/modules/Nncase.Modules.CPU/Utilities/PackUtility.cs
index 88de3b7d5..d24f9b7b6 100644
--- a/modules/Nncase.Modules.CPU/Utilities/PackUtility.cs
+++ b/modules/Nncase.Modules.CPU/Utilities/PackUtility.cs
@@ -46,128 +46,4 @@ public static Expr SliceForPack(Expr input, int[] shape, int[] padNums)
 
         return isPadded ? IR.F.Tensors.Slice(input, Enumerable.Repeat(0, shape.Length).ToArray(), ends, shape.Length) : input;
     }
-
-    /// <summary>
-    /// find the reshape's shape transform matrix.
-    /// </summary>
-    /// <param name="inShape"> input shape.</param>
-    /// <param name="newShape">new shape.</param>
-    /// <param name="mat">mat.</param>
-    /// <returns>bool.</returns>
-    public static bool TryGetShapeMapMatrix(int[] inShape, int[] newShape, out int[,] mat)
-    {
-        int ProdIn(int[,] cmat, int i)
-        {
-            var prod = 1;
-            for (int j = 0; j < inShape.Length; j++)
-            {
-                var v = cmat[i, j] * inShape[j];
-                if (v != 0)
-                {
-                    prod *= v;
-                }
-            }
-
-            return prod;
-        }
-
-        int ProdOut(int[,] cmat, int j)
-        {
-            var prod = 1;
-            for (int i = 0; i < newShape.Length; i++)
-            {
-                var v = cmat[i, j] * newShape[i];
-                if (v != 0)
-                {
-                    prod *= v;
-                }
-            }
-
-            return prod;
-        }
-
-        mat = new int[newShape.Length, inShape.Length];
-        int i = 0, j = 0;
-        var paths = new List<(int, int)>();
-        while (i < newShape.Length && j < inShape.Length)
-        {
-            if (paths.IndexOf((i, j)) != -1)
-            {
-                return false;
-            }
-
-            mat[i, j] = 1;
-            paths.Add((i, j));
-            var inDim = ProdIn(mat, i);
-            var outDim = ProdOut(mat, j);
-            switch (inDim - outDim)
-            {
-                case 0:
-                    i++; j++;
-                    break;
-                case < 0:
-                    j++;
-                    break;
-                case > 0:
-                    if (inDim % newShape[i] == 0)
-                    {
-                        i++;
-                    }
-                    else
-                    {
-                        mat[i, j] = 0;
-                        j--;
-                        paths.RemoveAt(paths.Count - 1);
-                    }
-
-                    break;
-            }
-        }
-
-        return i == newShape.Length && j == inShape.Length;
-    }
-
-    /// <summary>
-    /// convert the mapping matrix as a dictionary.
-    /// the key is in dim, value is not dim.
-    /// </summary>
-    /// <param name="mat">mat.</param>
-    /// <returns>dict.</returns>
-    public static (Dictionary<int, List<int>> Forward, Dictionary<int, List<int>> Backward) ShapeMapMatrixAsDict(int[,] mat)
-    {
-        var forward = new Dictionary<int, List<int>>();
-        var backward = new Dictionary<int, List<int>>();
-        for (int i = 0; i < mat.GetLength(0); i++)
-        {
-            for (int j = 0; j < mat.GetLength(1); j++)
-            {
-                if (mat[i, j] == 0)
-                {
-                    continue;
-                }
-
-                if (!forward.TryGetValue(j, out var l1))
-                {
-                    l1 = new() { i };
-                    forward.Add(j, l1);
-                }
-                else
-                {
-                    l1.Add(i);
-                }
-
-                if (!backward.TryGetValue(i, out var l2))
-                {
-                    l2 = new() { j };
-                    backward.Add(i, l2);
-                }
-                else
-                {
-                    l2.Add(j);
-                }
-            }
-        }
-
-        return (forward, backward);
-    }
 }
diff --git a/modules/Nncase.Modules.CPU/packages.lock.json b/modules/Nncase.Modules.CPU/packages.lock.json
index da1939c61..d9d3e1b58 100644
--- a/modules/Nncase.Modules.CPU/packages.lock.json
+++ b/modules/Nncase.Modules.CPU/packages.lock.json
@@ -170,7 +170,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/modules/Nncase.Modules.StackVM/packages.lock.json b/modules/Nncase.Modules.StackVM/packages.lock.json
index 38b0c1fda..e8b2a0b12 100644
--- a/modules/Nncase.Modules.StackVM/packages.lock.json
+++ b/modules/Nncase.Modules.StackVM/packages.lock.json
@@ -158,7 +158,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/python/_nncase.pyi b/python/_nncase.pyi
index d72c42123..ae69363e2 100644
--- a/python/_nncase.pyi
+++ b/python/_nncase.pyi
@@ -27,6 +27,8 @@ class CpuTargetOptions:
     Hierarchies: List[List[int]]
     HierarchyNames: str
     HierarchySizes: List[int]
+    HierarchyLatencies: List[int]
+    HierarchyBandWidths: List[int]
     MemoryCapacities: List[int]
     MemoryBandWidths: List[int]
     DistributedScheme: str
diff --git a/python/nncase/native/ffi.cpp b/python/nncase/native/ffi.cpp
index e099c0571..9f9cd8e90 100644
--- a/python/nncase/native/ffi.cpp
+++ b/python/nncase/native/ffi.cpp
@@ -288,6 +288,14 @@ PYBIND11_MODULE(_nncase, m) {
         "HierarchySizes",
         []() {},
         py::overload_cast<std::vector<int>>(&cpu_target_options::hierarchy_sizes)) 
+      .def_property(
+        "HierarchyLatencies",
+        []() {},
+        py::overload_cast<std::vector<int>>(&cpu_target_options::hierarchy_latencies)) 
+      .def_property(
+        "HierarchyBandWidths",
+        []() {},
+        py::overload_cast<std::vector<int>>(&cpu_target_options::hierarchy_band_widths)) 
       .def_property(
         "MemoryCapacities",
         []() {},
diff --git a/src/Native/include/nncase/compiler.h b/src/Native/include/nncase/compiler.h
index 79e764197..5ab4c0f32 100644
--- a/src/Native/include/nncase/compiler.h
+++ b/src/Native/include/nncase/compiler.h
@@ -255,6 +255,8 @@ typedef struct {
     void (*cpu_target_options_set_hierarchies)(clr_object_handle_t handle, int32_t* value, size_t shape0, size_t* shape1);
     void (*cpu_target_options_set_hierarchy_names)(clr_object_handle_t handle, const char* value, size_t length);
     void (*cpu_target_options_set_hierarchy_sizes)(clr_object_handle_t handle, int32_t* value, size_t shape0);
+    void (*cpu_target_options_set_hierarchy_latencies)(clr_object_handle_t handle, int32_t* value, size_t shape0);
+    void (*cpu_target_options_set_hierarchy_band_widths)(clr_object_handle_t handle, int32_t* value, size_t shape0);
     void (*cpu_target_options_set_memory_capacities)(clr_object_handle_t handle, int32_t* value, size_t shape0);
     void (*cpu_target_options_set_memory_band_widths)(clr_object_handle_t handle, int32_t* value, size_t shape0);
     void (*cpu_target_options_set_distributed_scheme)(clr_object_handle_t handle, const char* value, size_t length);
@@ -578,6 +580,28 @@ class cpu_target_options : public clr_object_base {
         nncase_clr_api()->cpu_target_options_set_hierarchy_sizes(obj_.get(), values.data(), shape0);
     }
 
+    void hierarchy_latencies(std::vector<int> value) {
+        std::vector<int> values;
+        size_t shape0;
+        shape0 = value.size();
+        for (size_t i0 = 0; i0 < shape0; i0++) {
+          values.push_back(value[i0]);
+        }
+
+        nncase_clr_api()->cpu_target_options_set_hierarchy_latencies(obj_.get(), values.data(), shape0);
+    }
+
+    void hierarchy_band_widths(std::vector<int> value) {
+        std::vector<int> values;
+        size_t shape0;
+        shape0 = value.size();
+        for (size_t i0 = 0; i0 < shape0; i0++) {
+          values.push_back(value[i0]);
+        }
+
+        nncase_clr_api()->cpu_target_options_set_hierarchy_band_widths(obj_.get(), values.data(), shape0);
+    }
+
     void memory_capacities(std::vector<int> value) {
         std::vector<int> values;
         size_t shape0;
diff --git a/src/Nncase.Cli/packages.lock.json b/src/Nncase.Cli/packages.lock.json
index 97b64d5d5..b83346286 100644
--- a/src/Nncase.Cli/packages.lock.json
+++ b/src/Nncase.Cli/packages.lock.json
@@ -711,7 +711,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/src/Nncase.Compiler/Compiler.cs b/src/Nncase.Compiler/Compiler.cs
index f49c61e6e..e7950b0c0 100644
--- a/src/Nncase.Compiler/Compiler.cs
+++ b/src/Nncase.Compiler/Compiler.cs
@@ -166,7 +166,7 @@ public void TargetIndependentPass(IPassManager passManager)
             p.Add<Passes.Rules.WithMarker.FoldTransposeBinaryActTranspose>();
             p.Add<Passes.Rules.WithMarker.CombineReshapePad>();
             p.Add<Passes.Rules.WithMarker.CombinePadTranspose>();
-            p.Add<Passes.Rules.Neutral.CombineTransposeUnary>();
+            p.Add<Passes.Rules.Neutral.CombineUnaryTranspose>();
             if (_compileSession.CompileOptions.ShapeBucketOptions.Enable)
             {
                 p.Add<Passes.Rules.WithMarker.CombineTransposePad>();
diff --git a/src/Nncase.Compiler/Interop/CApi.cs b/src/Nncase.Compiler/Interop/CApi.cs
index a406d2ace..2946c732f 100644
--- a/src/Nncase.Compiler/Interop/CApi.cs
+++ b/src/Nncase.Compiler/Interop/CApi.cs
@@ -110,6 +110,8 @@ public unsafe struct CApiMT
     public delegate* unmanaged<IntPtr, int*, nuint, nuint*, void> CpuTargetOptionsSetHierarchiesPtr;
     public delegate* unmanaged<IntPtr, byte*, nuint, void> CpuTargetOptionsSetHierarchyNamesPtr;
     public delegate* unmanaged<IntPtr, int*, nuint, void> CpuTargetOptionsSetHierarchySizesPtr;
+    public delegate* unmanaged<IntPtr, int*, nuint, void> CpuTargetOptionsSetHierarchyLatenciesPtr;
+    public delegate* unmanaged<IntPtr, int*, nuint, void> CpuTargetOptionsSetHierarchyBandWidthsPtr;
     public delegate* unmanaged<IntPtr, int*, nuint, void> CpuTargetOptionsSetMemoryCapacitiesPtr;
     public delegate* unmanaged<IntPtr, int*, nuint, void> CpuTargetOptionsSetMemoryBandWidthsPtr;
     public delegate* unmanaged<IntPtr, byte*, nuint, void> CpuTargetOptionsSetDistributedSchemePtr;
@@ -202,6 +204,8 @@ public static void Initialize(CApiMT* mt)
         mt->CpuTargetOptionsSetHierarchiesPtr = &CpuTargetOptionsSetHierarchies;
         mt->CpuTargetOptionsSetHierarchyNamesPtr = &CpuTargetOptionsSetHierarchyNames;
         mt->CpuTargetOptionsSetHierarchySizesPtr = &CpuTargetOptionsSetHierarchySizes;
+        mt->CpuTargetOptionsSetHierarchyLatenciesPtr = &CpuTargetOptionsSetHierarchyLatencies;
+        mt->CpuTargetOptionsSetHierarchyBandWidthsPtr = &CpuTargetOptionsSetHierarchyBandWidths;
         mt->CpuTargetOptionsSetMemoryCapacitiesPtr = &CpuTargetOptionsSetMemoryCapacities;
         mt->CpuTargetOptionsSetMemoryBandWidthsPtr = &CpuTargetOptionsSetMemoryBandWidths;
         mt->CpuTargetOptionsSetDistributedSchemePtr = &CpuTargetOptionsSetDistributedScheme;
@@ -867,6 +871,18 @@ private static void CpuTargetOptionsSetHierarchySizes(IntPtr handle, int* value,
         Get<CpuTargetOptions>(handle).HierarchySizes = To1DArray(value, shape0);
     }
 
+    [UnmanagedCallersOnly]
+    private static void CpuTargetOptionsSetHierarchyLatencies(IntPtr handle, int* value, nuint shape0)
+    {
+        Get<CpuTargetOptions>(handle).HierarchyLatencies = To1DArray(value, shape0);
+    }
+
+    [UnmanagedCallersOnly]
+    private static void CpuTargetOptionsSetHierarchyBandWidths(IntPtr handle, int* value, nuint shape0)
+    {
+        Get<CpuTargetOptions>(handle).HierarchyBandWidths = To1DArray(value, shape0);
+    }
+
     [UnmanagedCallersOnly]
     private static void CpuTargetOptionsSetMemoryCapacities(IntPtr handle, int* value, nuint shape0)
     {
diff --git a/src/Nncase.Compiler/packages.lock.json b/src/Nncase.Compiler/packages.lock.json
index 5c7a0d700..6bfb830ed 100644
--- a/src/Nncase.Compiler/packages.lock.json
+++ b/src/Nncase.Compiler/packages.lock.json
@@ -692,7 +692,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/src/Nncase.Core/CostModel/Cost.cs b/src/Nncase.Core/CostModel/Cost.cs
index e60414a61..a197eb5ad 100644
--- a/src/Nncase.Core/CostModel/Cost.cs
+++ b/src/Nncase.Core/CostModel/Cost.cs
@@ -15,6 +15,8 @@ public static class CostFactorNames
     public static readonly string MemoryStore = "MemoryStore";
 
     public static readonly string CPUCycles = "CPUCycles";
+
+    public static readonly string Comm = "Comm";
 }
 
 /// <summary>
diff --git a/src/Nncase.Core/DistributedType.cs b/src/Nncase.Core/DistributedType.cs
index 0d3415a6f..582237579 100644
--- a/src/Nncase.Core/DistributedType.cs
+++ b/src/Nncase.Core/DistributedType.cs
@@ -21,14 +21,14 @@ public enum HierarchyKind : byte
 }
 
 [JsonDerivedType(typeof(SBPSplit), "S")]
-[JsonDerivedType(typeof(SBPPartialSum), "P")]
+[JsonDerivedType(typeof(SBPPartial), "P")]
 [JsonDerivedType(typeof(SBPBroadCast), "B")]
 public abstract record SBP
 {
-    public static SBPPartialSum P => SBPPartialSum.Instance;
-
     public static SBPBroadCast B => SBPBroadCast.Instance;
 
+    public static SBPPartial P(ReduceOp op = ReduceOp.Sum) => new SBPPartial(op);
+
     public static SBPSplit S(int axis) => new SBPSplit(axis);
 }
 
@@ -37,11 +37,9 @@ public sealed record SBPSplit(int Axis) : SBP
     public override string ToString() => $"S({Axis})";
 }
 
-public sealed record SBPPartialSum : SBP
+public sealed record SBPPartial(ReduceOp Op) : SBP
 {
-    public static readonly SBPPartialSum Instance = new SBPPartialSum();
-
-    public override string ToString() => "P";
+    public override string ToString() => $"P({Op})";
 }
 
 public sealed record SBPBroadCast : SBP
@@ -60,7 +58,7 @@ public sealed record Placement(IRArray<int> Hierarchy, string Name, HierarchyKin
     // }
     public int Rank => Hierarchy.Count;
 
-    public override string ToString() => $"@ [{string.Join(',', Hierarchy.Zip(Name).Select(t => t.First.ToString() + '@' + t.Second.ToString()))}]";
+    public override string ToString() => $"[{string.Join(',', Hierarchy.Zip(Name).Select(t => t.Second.ToString() + ':' + t.First.ToString()))}]";
 }
 
 public sealed record DistributedType(TensorType TensorType, IRArray<SBP> NdSBP, Placement Placement) : IRType
diff --git a/src/Nncase.Core/Enum/ReduceOp.cs b/src/Nncase.Core/Enum/ReduceOp.cs
index 646f5180a..92ed17abd 100644
--- a/src/Nncase.Core/Enum/ReduceOp.cs
+++ b/src/Nncase.Core/Enum/ReduceOp.cs
@@ -1,11 +1,14 @@
 // Copyright (c) Canaan Inc. All rights reserved.
 // Licensed under the Apache license. See LICENSE file in the project root for full license information.
 
+using System.Text.Json.Serialization;
+
 namespace Nncase;
 
 /// <summary>
 /// Reduce operator.
 /// </summary>
+[JsonConverter(typeof(JsonStringEnumConverter))]
 public enum ReduceOp : byte
 {
     /// <summary>
diff --git a/src/Nncase.Core/IR/Math/Reduce.cs b/src/Nncase.Core/IR/Math/Reduce.cs
index afc814a2c..c9dae41ac 100644
--- a/src/Nncase.Core/IR/Math/Reduce.cs
+++ b/src/Nncase.Core/IR/Math/Reduce.cs
@@ -18,9 +18,9 @@ public sealed partial class Reduce : Op
     public static readonly ParameterInfo Input = new(typeof(Reduce), 0, "input", ParameterKind.Input);
 
     /// <summary>
-    /// Gets axis.
+    /// Gets axes.
     /// </summary>
-    public static readonly ParameterInfo Axis = new(typeof(Reduce), 1, "axis", IsIntegral() & HasRank(1));
+    public static readonly ParameterInfo Axes = new(typeof(Reduce), 1, "axes", IsIntegral() & HasRank(1));
 
     /// <summary>
     /// Gets initial value.
diff --git a/src/Nncase.Core/IR/Tensors/Functional.cs b/src/Nncase.Core/IR/Tensors/Functional.cs
index d20f69cbf..2a89fc9c2 100644
--- a/src/Nncase.Core/IR/Tensors/Functional.cs
+++ b/src/Nncase.Core/IR/Tensors/Functional.cs
@@ -109,8 +109,8 @@ public static Call Prod(Expr input)
 
     public static Call Range(Expr begin, Expr end, Expr step) => new Call(new Range(), begin, end, step);
 
-    public static Call Reduce(ReduceOp reduceOp, Expr input, Expr axis, Expr initValue, Expr keepDims) =>
-        new Call(new Reduce(reduceOp), input, axis, initValue, keepDims);
+    public static Call Reduce(ReduceOp reduceOp, Expr input, Expr axes, Expr initValue, Expr keepDims) =>
+        new Call(new Reduce(reduceOp), input, axes, initValue, keepDims);
 
     public static Call ReduceArg(ReduceArgOp reduceArgOp, PrimType destType, Expr input, Expr axis, Expr keepDims, Expr selectLastIndex) =>
         new Call(new ReduceArg(reduceArgOp, destType), input, axis, keepDims, selectLastIndex);
diff --git a/src/Nncase.Core/IR/Tensors/GetItem.cs b/src/Nncase.Core/IR/Tensors/GetItem.cs
index 1c006d4fb..32bd9666b 100644
--- a/src/Nncase.Core/IR/Tensors/GetItem.cs
+++ b/src/Nncase.Core/IR/Tensors/GetItem.cs
@@ -21,7 +21,7 @@ public sealed partial class GetItem : Op
     /// <summary>
     /// Gets input.
     /// </summary>
-    public static readonly ParameterInfo Input = new(typeof(GetItem), 0, "input", IsTensor() | (IsTuple() & !IsUnit()));
+    public static readonly ParameterInfo Input = new(typeof(GetItem), 0, "input", IsTensor() | (IsTuple() & !IsUnit()), ParameterKind.Input);
 
     /// <summary>
     /// Gets index.
diff --git a/src/Nncase.Core/ITarget.cs b/src/Nncase.Core/ITarget.cs
index 58bc54ff1..8d5f1af5e 100644
--- a/src/Nncase.Core/ITarget.cs
+++ b/src/Nncase.Core/ITarget.cs
@@ -19,6 +19,10 @@ public interface ICpuTargetOptions : ITargetOptions
 
     int[] MemoryBandWidths { get; }
 
+    public int[] HierarchyLatencies { get; }
+
+    public int[] HierarchyBandWidths { get; }
+
     bool UnifiedMemoryArch { get; }
 }
 
diff --git a/src/Nncase.Core/Utilities/DistributedUtility.cs b/src/Nncase.Core/Utilities/DistributedUtility.cs
index b25313791..68392d33a 100644
--- a/src/Nncase.Core/Utilities/DistributedUtility.cs
+++ b/src/Nncase.Core/Utilities/DistributedUtility.cs
@@ -33,43 +33,6 @@ public static IReadOnlyList<IRArray<SBP>> GetLeafCandidateNDSBPs(TensorType tens
         return ndsbps.CartesianProduct().Select(ndsbp => ndsbp.ToArray()).Where(ndsbp => IsDistributable(tensorType, ndsbp, placement)).Select(ndsbp => new IRArray<SBP>(ndsbp)).ToArray();
     }
 
-    public static IReadOnlyList<IRArray<SBP>> GetPartialCandidateNDSBPs(DistributedType distributedType)
-    {
-        IRArray<SBP> ndsbp = distributedType.NdSBP;
-        TensorType tensorType = distributedType.TensorType;
-        Placement placement = distributedType.Placement;
-        if (!ndsbp.Any(sbp => sbp is SBPPartialSum))
-        {
-            return Array.Empty<IRArray<SBP>>();
-        }
-
-        var candidateNdsbps = new List<SBP>[placement.Rank];
-        for (int i = 0; i < placement.Rank; i++)
-        {
-            candidateNdsbps[i] = new List<SBP>();
-            var innerSplitedAxes = distributedType.NdSBP.Skip(i + 1).OfType<SBPSplit>().Select(sbp => sbp.Axis).ToList();
-            if (ndsbp[i] is SBPPartialSum)
-            {
-                candidateNdsbps[i].Add(SBP.B);
-
-                // note separate reduce boxing and reshard boxing.
-                // for (int axis = 0; axis < tensorType.Shape.Rank; axis++)
-                // {
-                //     if (tensorType.Shape[axis] is { IsFixed: true, Value: int s } && placement.Hierarchy[i] > 1 && IsDivideExactly(s, placement.Hierarchy[i]) && !innerSplitedAxes.Contains(axis))
-                //     {
-                //         candidateNdsbps[i].Add(SBP.S(axis));
-                //     }
-                // }
-            }
-            else
-            {
-                candidateNdsbps[i].Add(ndsbp[i]);
-            }
-        }
-
-        return candidateNdsbps.CartesianProduct().Select(ndsbp => ndsbp.ToArray()).Where(ndsbp => IsDistributable(tensorType, ndsbp, placement)).Select(ndsbp => new IRArray<SBP>(ndsbp)).ToArray();
-    }
-
     public static bool IsDistributable(TensorType tensorType, ReadOnlySpan<SBP> ndsbp, Placement placement)
     {
         if (!tensorType.Shape.IsFixed)
diff --git a/src/Nncase.Core/Utilities/IRUtility.cs b/src/Nncase.Core/Utilities/IRUtility.cs
new file mode 100644
index 000000000..1ce420031
--- /dev/null
+++ b/src/Nncase.Core/Utilities/IRUtility.cs
@@ -0,0 +1,131 @@
+// Copyright (c) Canaan Inc. All rights reserved.
+// Licensed under the Apache license. See LICENSE file in the project root for full license information.
+
+namespace Nncase.Utilities;
+
+public static class IRUtility
+{
+    /// <summary>
+    /// find the reshape's shape transform matrix.
+    /// </summary>
+    /// <param name="inShape"> input shape.</param>
+    /// <param name="newShape">new shape.</param>
+    /// <param name="mat">mat [new shape dim, old shpe dim].</param>
+    /// <returns>bool.</returns>
+    public static bool TryGetShapeMapMatrix(int[] inShape, int[] newShape, out int[,] mat)
+    {
+        int ProdIn(int[,] cmat, int i)
+        {
+            var prod = 1;
+            for (int j = 0; j < inShape.Length; j++)
+            {
+                var v = cmat[i, j] * inShape[j];
+                if (v != 0)
+                {
+                    prod *= v;
+                }
+            }
+
+            return prod;
+        }
+
+        int ProdOut(int[,] cmat, int j)
+        {
+            var prod = 1;
+            for (int i = 0; i < newShape.Length; i++)
+            {
+                var v = cmat[i, j] * newShape[i];
+                if (v != 0)
+                {
+                    prod *= v;
+                }
+            }
+
+            return prod;
+        }
+
+        mat = new int[newShape.Length, inShape.Length];
+        int i = 0, j = 0;
+        var paths = new List<(int, int)>();
+        while (i < newShape.Length && j < inShape.Length)
+        {
+            if (paths.IndexOf((i, j)) != -1)
+            {
+                return false;
+            }
+
+            mat[i, j] = 1;
+            paths.Add((i, j));
+            var inDim = ProdIn(mat, i);
+            var outDim = ProdOut(mat, j);
+            switch (inDim - outDim)
+            {
+                case 0:
+                    i++; j++;
+                    break;
+                case < 0:
+                    j++;
+                    break;
+                case > 0:
+                    if (inDim % newShape[i] == 0)
+                    {
+                        i++;
+                    }
+                    else
+                    {
+                        mat[i, j] = 0;
+                        j--;
+                        paths.RemoveAt(paths.Count - 1);
+                    }
+
+                    break;
+            }
+        }
+
+        return i == newShape.Length && j == inShape.Length;
+    }
+
+    /// <summary>
+    /// convert the mapping matrix as a dictionary.
+    /// the key is in dim, value is not dim.
+    /// </summary>
+    /// <param name="mat">mat.</param>
+    /// <returns>dict.</returns>
+    public static (Dictionary<int, List<int>> Forward, Dictionary<int, List<int>> Backward) ShapeMapMatrixAsDict(int[,] mat)
+    {
+        var forward = new Dictionary<int, List<int>>();
+        var backward = new Dictionary<int, List<int>>();
+        for (int i = 0; i < mat.GetLength(0); i++)
+        {
+            for (int j = 0; j < mat.GetLength(1); j++)
+            {
+                if (mat[i, j] == 0)
+                {
+                    continue;
+                }
+
+                if (!forward.TryGetValue(j, out var l1))
+                {
+                    l1 = new() { i };
+                    forward.Add(j, l1);
+                }
+                else
+                {
+                    l1.Add(i);
+                }
+
+                if (!backward.TryGetValue(i, out var l2))
+                {
+                    l2 = new() { j };
+                    backward.Add(i, l2);
+                }
+                else
+                {
+                    l2.Add(j);
+                }
+            }
+        }
+
+        return (forward, backward);
+    }
+}
diff --git a/src/Nncase.EGraph/Nncase.EGraph.csproj b/src/Nncase.EGraph/Nncase.EGraph.csproj
index 105753eb6..15a4bcfc1 100644
--- a/src/Nncase.EGraph/Nncase.EGraph.csproj
+++ b/src/Nncase.EGraph/Nncase.EGraph.csproj
@@ -13,6 +13,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\Nncase.Core\Nncase.Core.csproj" />
+    <ProjectReference Include="..\Nncase.Graph\Nncase.Graph.csproj" />
     <ProjectReference Include="..\Nncase.Evaluator\Nncase.Evaluator.csproj" />
   </ItemGroup>
 </Project>
diff --git a/src/Nncase.EGraph/Passes/EGraphExtractor.cs b/src/Nncase.EGraph/Passes/EGraphExtractor.cs
index b05dd322e..0bd18d0ec 100644
--- a/src/Nncase.EGraph/Passes/EGraphExtractor.cs
+++ b/src/Nncase.EGraph/Passes/EGraphExtractor.cs
@@ -9,6 +9,7 @@
 using Google.OrTools.Sat;
 using Nncase.CostModel;
 using Nncase.Diagnostics;
+using Nncase.Graphs;
 using Nncase.IR;
 
 namespace Nncase.Passes;
@@ -54,7 +55,7 @@ public Expr Extract(EClass root, IEGraph eGraph, EGraphExtractConstrains[] const
         // 3. no cycle
         {
             var hgraph = ToHyperGraph(root);
-            var class_cycles = FindCycles(hgraph);
+            var class_cycles = hgraph.FindCycles();
             foreach (var cycle in class_cycles)
             {
                 if (cycle.Count == 1)
@@ -164,9 +165,9 @@ public Expr Extract(EClass root, IEGraph eGraph, EGraphExtractConstrains[] const
         return new SatExprBuildVisitor(picks).Visit(root);
     }
 
-    private static HyperGraph ToHyperGraph(EClass root)
+    private static HyperGraph<EClass, ENode> ToHyperGraph(EClass root)
     {
-        var hgraph = new HyperGraph();
+        var hgraph = new HyperGraph<EClass, ENode>();
         var visited = new HashSet<EClass>();
         var queue = new Queue<EClass>();
         queue.Enqueue(root);
@@ -191,321 +192,6 @@ private static HyperGraph ToHyperGraph(EClass root)
 
         return hgraph;
     }
-
-    private static List<List<EClass>> FindCycles(HyperGraph hgraph)
-    {
-        var edges = hgraph.AdjacencyEdges();
-        var circuits = new List<List<EClass>>();
-        foreach (var n in hgraph.Nodes())
-        {
-            if (hgraph.Neighbors(n).Contains(n))
-            {
-                circuits.Add(new() { n });
-            }
-        }
-
-        var stack = new List<int>();
-        bool[] blocked = new bool[hgraph.NumEdges()];
-        var blockMap = new Dictionary<int, Dictionary<int, bool>>();
-        List<List<int>> adjList;
-        int s = 0;
-
-        void Unblock(int u)
-        {
-            blocked[u] = false;
-            if (blockMap.ContainsKey(u))
-            {
-                foreach (var w in blockMap[u].Keys)
-                {
-                    blockMap[u].Remove(w);
-                    if (blocked[w])
-                    {
-                        Unblock(w);
-                    }
-                }
-            }
-        }
-
-        bool Circuit(int v)
-        {
-            bool found = false;
-
-            stack.Add(v);
-            blocked[v] = true;
-
-            for (int i = 0; i < adjList[v].Count; i++)
-            {
-                int w = adjList[v][i];
-                if (w == s)
-                {
-                    Output(s, stack);
-                    found = true;
-                }
-                else if (!blocked[w])
-                {
-                    found = Circuit(w);
-                }
-            }
-
-            if (found)
-            {
-                Unblock(v);
-            }
-            else
-            {
-                for (int i = 0; i < adjList[v].Count; i++)
-                {
-                    int w = adjList[v][i];
-                    if (!blockMap.ContainsKey(w))
-                    {
-                        blockMap[w] = new Dictionary<int, bool>();
-                    }
-
-                    blockMap[w][w] = true;
-                }
-            }
-
-            stack.RemoveAt(stack.Count - 1);
-            return found;
-        }
-
-        void Output(int start, List<int> stack)
-        {
-            // var cycle = new List<int>(stack);
-            // {
-            //     start,
-            // };
-            circuits.Add(stack.Select(hgraph.GetIdByNode).ToList());
-        }
-
-        void Subgraph(int minId)
-        {
-            for (int i = 0; i < edges!.Count; i++)
-            {
-                if (i < minId || edges[i] == null)
-                {
-                    edges[i] = new List<int>();
-                }
-
-                edges[i] = edges[i].Where(j => j >= minId).ToList();
-            }
-        }
-
-        (int LeastVertex, List<List<int>> AdjList) AdjacencyStructureSCC(int from)
-        {
-            Subgraph(from);
-            List<List<int>> g = edges;
-
-            var (components, _) = StronglyConnectedComponents(edges);
-
-            var ccs = components.Where(scc => scc.Count > 1).ToList();
-
-            int leastVertex = int.MaxValue;
-            int leastVertexComponent = -1;
-            for (int i = 0; i < ccs.Count; i++)
-            {
-                for (int j = 0; j < ccs[i].Count; j++)
-                {
-                    if (ccs[i][j] < leastVertex)
-                    {
-                        leastVertex = ccs[i][j];
-                        leastVertexComponent = i;
-                    }
-                }
-            }
-
-            var cc = leastVertexComponent >= 0 ? ccs[leastVertexComponent] : null;
-
-            if (cc == null)
-            {
-                return (-1, new());
-            }
-
-            var adjList = edges.Select((l, index) =>
-            {
-                if (cc.IndexOf(index) == -1)
-                {
-                    return new();
-                }
-
-                return l.Where(i => cc.IndexOf(i) != -1).ToList();
-            }).ToList();
-
-            return (leastVertex, adjList);
-        }
-
-        while (s < edges.Count)
-        {
-            var (leastVertex, leastAdjList) = AdjacencyStructureSCC(s);
-            s = leastVertex;
-            adjList = leastAdjList;
-
-            if (adjList.Any())
-            {
-                for (int i = 0; i < adjList.Count; i++)
-                {
-                    for (int j = 0; j < adjList[i].Count; j++)
-                    {
-                        int vertexId = adjList[i][j];
-                        blocked[vertexId] = false;
-                        if (!blockMap.ContainsKey(vertexId))
-                        {
-                            blockMap[vertexId] = new Dictionary<int, bool>();
-                        }
-                    }
-                }
-
-                Circuit(s);
-                s++;
-            }
-            else
-            {
-                s = edges.Count;
-            }
-        }
-
-        return circuits;
-    }
-
-    private static (List<List<int>> Components, List<List<int>> AdjacencyList) StronglyConnectedComponents(List<List<int>> adjList)
-    {
-        int numVertices = adjList.Count;
-        int[] index = new int[numVertices];
-        int[] lowValue = new int[numVertices];
-        bool[] active = new bool[numVertices];
-        int[] child = new int[numVertices];
-        int[] scc = new int[numVertices];
-        var sccLinks = new List<int>[numVertices];
-
-        // Initialize tables
-        for (int i = 0; i < numVertices; ++i)
-        {
-            index[i] = -1;
-            lowValue[i] = 0;
-            active[i] = false;
-            child[i] = 0;
-            scc[i] = -1;
-            sccLinks[i] = new List<int>();
-        }
-
-        int count = 0;
-        var components = new List<List<int>>();
-        var sccAdjList = new List<List<int>>();
-
-        void StrongConnect(int v)
-        {
-            var s = new Stack<int>();
-            var t = new Stack<int>();
-            s.Push(v);
-            t.Push(v);
-            index[v] = lowValue[v] = count;
-            active[v] = true;
-            count++;
-
-            while (t.Count > 0)
-            {
-                v = t.Peek();
-                var e = adjList[v];
-                if (child[v] < e.Count)
-                {
-                    int i;
-                    for (i = child[v]; i < e.Count; ++i)
-                    {
-                        int u = e[i];
-                        if (index[u] < 0)
-                        {
-                            index[u] = lowValue[u] = count;
-                            active[u] = true;
-                            count++;
-                            s.Push(u);
-                            t.Push(u);
-                            break;
-                        }
-                        else if (active[u])
-                        {
-                            lowValue[v] = Math.Min(lowValue[v], lowValue[u]);
-                        }
-
-                        if (scc[u] >= 0)
-                        {
-                            sccLinks[v].Add(scc[u]);
-                        }
-                    }
-
-                    child[v] = i;
-                }
-                else
-                {
-                    if (lowValue[v] == index[v])
-                    {
-                        var component = new List<int>();
-                        var links = Enumerable.Range(0, s.Count).Select(i => new List<int>()).ToArray();
-                        int linkCount = 0;
-                        for (int i = s.Count - 1; i >= 0; --i)
-                        {
-                            int w = s.Pop();
-                            active[w] = false;
-                            component.Add(w);
-                            links[i] = sccLinks[w];
-                            linkCount += sccLinks[w].Count;
-                            scc[w] = components.Count;
-                            if (w == v)
-                            {
-                                break;
-                            }
-                        }
-
-                        components.Add(component);
-                        var allLinks = new List<int>(linkCount);
-                        for (int i = 0; i < links.Length; i++)
-                        {
-                            for (int j = 0; j < links[i].Count; j++)
-                            {
-                                allLinks.Add(links[i][j]);
-                            }
-                        }
-
-                        sccAdjList.Add(allLinks);
-                    }
-
-                    t.Pop();
-                }
-            }
-        }
-
-        // Run strong connect starting from each vertex
-        for (int i = 0; i < numVertices; ++i)
-        {
-            if (index[i] < 0)
-            {
-                StrongConnect(i);
-            }
-        }
-
-        // Compact sccAdjList
-        for (int i = 0; i < sccAdjList.Count; i++)
-        {
-            var e = sccAdjList[i];
-            if (e.Count == 0)
-            {
-                continue;
-            }
-
-            e.Sort();
-            var newE = new List<int> { e[0] };
-            for (int j = 1; j < e.Count; j++)
-            {
-                if (e[j] != e[j - 1])
-                {
-                    newE.Add(e[j]);
-                }
-            }
-
-            sccAdjList[i] = newE;
-        }
-
-        return (components, sccAdjList);
-    }
 }
 
 internal sealed class PrintCostCallBack : CpSolverSolutionCallback
@@ -601,202 +287,3 @@ public Expr Visit(EClass root)
         return expr;
     }
 }
-
-internal sealed class HyperGraph
-{
-    private readonly Dictionary<int, Dictionary<int, HashSet<ENode>>> _edges;
-    private readonly HashSet<int> _nodes;
-    private readonly Dictionary<EClass, int> _ids_to_nodes;
-    private readonly Dictionary<int, EClass> _nodes_to_ids;
-    private int _num_nodes;
-
-    public HyperGraph()
-    {
-        _edges = new();
-        _nodes = new();
-        _ids_to_nodes = new();
-        _nodes_to_ids = new();
-        _num_nodes = 0;
-    }
-
-    public bool Contains(EClass id)
-    {
-        return _ids_to_nodes.ContainsKey(id);
-    }
-
-    public List<List<int>> AdjacencyEdges()
-    {
-        return _edges.Select(kv => kv.Value.Keys.ToList()).ToList();
-    }
-
-    public Dictionary<EClass, HashSet<ENode>>? Edges(EClass eclass)
-    {
-        if (Contains(eclass))
-        {
-            var result = new Dictionary<EClass, HashSet<ENode>>();
-            foreach (var (to, enodes) in _edges[_ids_to_nodes[eclass]])
-            {
-                result.Add(_nodes_to_ids[to], enodes);
-            }
-
-            return result;
-        }
-
-        return null;
-    }
-
-    public int NumEdges()
-    {
-        return _edges.Select(kv => kv.Value.Keys.Count).Sum();
-    }
-
-    public HashSet<EClass> Nodes()
-    {
-        return new(_nodes.Select(x => _nodes_to_ids[x]));
-    }
-
-    public void AddNode(EClass k)
-    {
-        var node_id = _num_nodes;
-        _ids_to_nodes.Add(k, node_id);
-        _nodes_to_ids.Add(node_id, k);
-        _edges.Add(node_id, new());
-        _nodes.Add(node_id);
-        _num_nodes += 1;
-    }
-
-    public void Connect(EClass cfrom, EClass cto, ENode enode)
-    {
-        if (!Contains(cfrom))
-        {
-            AddNode(cfrom);
-        }
-
-        if (!Contains(cto))
-        {
-            AddNode(cto);
-        }
-
-        var from = _ids_to_nodes[cfrom];
-        var to = _ids_to_nodes[cto];
-        if (!_edges[from].ContainsKey(to))
-        {
-            _edges[from].Add(to, new HashSet<ENode>(new[] { enode }));
-        }
-        else
-        {
-            _edges[from][to].Add(enode);
-        }
-    }
-
-    public EClass[] Neighbors(EClass u)
-    {
-        if (Contains(u))
-        {
-            return _edges[_ids_to_nodes[u]].Keys.Select(x => _nodes_to_ids[x]).ToArray();
-        }
-
-        return Array.Empty<EClass>();
-    }
-
-    public int GetNodeById(EClass id)
-    {
-        return _ids_to_nodes[id];
-    }
-
-    public EClass GetIdByNode(int node)
-    {
-        return _nodes_to_ids[node];
-    }
-
-    public void RemoveNodeRaw(int node)
-    {
-        if (_nodes.Contains(node))
-        {
-            _edges.Remove(node);
-            foreach (var (_, v) in _edges)
-            {
-                v.Remove(node);
-            }
-
-            _nodes.Remove(node);
-        }
-    }
-
-    public void RemoveNode(EClass node)
-    {
-        var node_id = _ids_to_nodes[node];
-        if (Contains(node))
-        {
-            _edges.Remove(node_id);
-            foreach (var (_, v) in _edges)
-            {
-                v.Remove(node_id);
-            }
-
-            _nodes.Remove(node_id);
-        }
-    }
-
-    public int Size()
-    {
-        return _nodes.Count;
-    }
-
-    public HyperGraph SubGraph(IEnumerable<EClass> nodes)
-    {
-        var graph = new HyperGraph();
-        var node_set = new HashSet<EClass>(nodes);
-        foreach (var n in node_set)
-        {
-            var nedges = _edges[_ids_to_nodes[n]];
-            foreach (var (neighbor, enodes) in nedges)
-            {
-                if (!node_set.Contains(_nodes_to_ids[neighbor]))
-                {
-                    continue;
-                }
-
-                foreach (var enode in enodes)
-                {
-                    graph.Connect(n, _nodes_to_ids[neighbor], enode);
-                }
-            }
-        }
-
-        return graph;
-    }
-
-    public void Dump(string name)
-    {
-        using (var dumpStream = DumpScope.Current.OpenFile($"Costs/{name}.dot"))
-        {
-            using var writer = new StreamWriter(dumpStream);
-            writer.WriteLine("digraph G {");
-            writer.WriteLine("/*");
-            writer.WriteLine("[");
-            foreach (var (_, v) in _edges.OrderBy(kv => kv.Key))
-            {
-                writer.WriteLine($"[{string.Join(",", v.Keys)}],");
-            }
-
-            writer.WriteLine("]");
-            writer.WriteLine("*/");
-
-            foreach (var node in _nodes)
-            {
-                writer.WriteLine($"{node} [label = \"{_nodes_to_ids[node].Id}\"]");
-            }
-
-            foreach (var (u, v) in _edges)
-            {
-                foreach (var w in v.Keys)
-                {
-                    writer.WriteLine($"{u} -> {w};");
-                }
-            }
-
-            writer.WriteLine("}");
-        }
-    }
-}
diff --git a/src/Nncase.EGraph/packages.lock.json b/src/Nncase.EGraph/packages.lock.json
index 7a003a140..8fb2625a9 100644
--- a/src/Nncase.EGraph/packages.lock.json
+++ b/src/Nncase.EGraph/packages.lock.json
@@ -181,6 +181,14 @@
           "OrtKISharp": "[0.0.2, )"
         }
       },
+      "nncase.graph": {
+        "type": "Project",
+        "dependencies": {
+          "Nncase.Core": "[1.0.0, )",
+          "Nncase.Evaluator": "[1.0.0, )",
+          "QuikGraph": "[2.5.0, )"
+        }
+      },
       "CommunityToolkit.HighPerformance": {
         "type": "CentralTransitive",
         "requested": "[8.2.2, )",
@@ -240,6 +248,12 @@
           "libortki": "0.0.2"
         }
       },
+      "QuikGraph": {
+        "type": "CentralTransitive",
+        "requested": "[2.5.0, )",
+        "resolved": "2.5.0",
+        "contentHash": "sG+mrPpXwxlXknRK5VqWUGiOmDACa9X+3ftlkQIMgOZUqxVOQSe0+HIU9PTjwqazy0pqSf8MPDXYFGl0GYWcKw=="
+      },
       "System.CommandLine": {
         "type": "CentralTransitive",
         "requested": "[2.0.0-beta4.22272.1, )",
diff --git a/src/Nncase.Evaluator/Math/Binary.cs b/src/Nncase.Evaluator/Math/Binary.cs
index 4343cd168..3161d64d1 100755
--- a/src/Nncase.Evaluator/Math/Binary.cs
+++ b/src/Nncase.Evaluator/Math/Binary.cs
@@ -56,10 +56,10 @@ public static IRType CheckSBP(BinaryOp op, TensorType tensorType, DistributedTyp
                 case (SBPBroadCast, SBPBroadCast):
                     ndsbp[i] = SBP.B;
                     break;
-                case (SBPPartialSum, SBPPartialSum):
+                case (SBPPartial, SBPPartial):
                     if (op == BinaryOp.Add)
                     {
-                        ndsbp[i] = SBP.P;
+                        ndsbp[i] = SBP.P();
                     }
                     else
                     {
@@ -67,8 +67,8 @@ public static IRType CheckSBP(BinaryOp op, TensorType tensorType, DistributedTyp
                     }
 
                     break;
-                case (SBPPartialSum, _):
-                case (_, SBPPartialSum):
+                case (SBPPartial, _):
+                case (_, SBPPartial):
                     return new InvalidType("not support lhs or rhs partial.");
             }
         }
diff --git a/src/Nncase.Evaluator/Math/Compare.cs b/src/Nncase.Evaluator/Math/Compare.cs
index 69322ec18..a7d916339 100644
--- a/src/Nncase.Evaluator/Math/Compare.cs
+++ b/src/Nncase.Evaluator/Math/Compare.cs
@@ -54,9 +54,9 @@ public static IRType CheckSBP(TensorType tensorType, DistributedType a, Distribu
                 case (SBPBroadCast, SBPBroadCast):
                     ndsbp[i] = SBP.B;
                     break;
-                case (SBPPartialSum, SBPPartialSum):
-                case (SBPPartialSum, _):
-                case (_, SBPPartialSum):
+                case (SBPPartial, SBPPartial):
+                case (SBPPartial, _):
+                case (_, SBPPartial):
                     return new InvalidType("not support lhs or rhs partial.");
             }
         }
diff --git a/src/Nncase.Evaluator/Math/MatMul.cs b/src/Nncase.Evaluator/Math/MatMul.cs
index 601e5205a..946afd3bc 100644
--- a/src/Nncase.Evaluator/Math/MatMul.cs
+++ b/src/Nncase.Evaluator/Math/MatMul.cs
@@ -56,7 +56,7 @@ public static IRType VisitDistributedType(DistributedType a, DistributedType b,
                             return invalid;
                         }
 
-                        ndsbp[i] = SBP.P;
+                        ndsbp[i] = SBP.P(ReduceOp.Sum);
                     }
                     else if ((ax == lk && bx != rk) || (ax != lk && bx == rk) || (ax == lm && bx == rn))
                     {
@@ -130,7 +130,7 @@ public static IRType VisitDistributedType(DistributedType a, DistributedType b,
 
     public static IRType ConvertPartialToBroadcast(DistributedType a)
     {
-        var ndsbp = a.NdSBP.Select(x => x == SBP.P ? SBP.B : x).ToArray();
+        var ndsbp = a.NdSBP.Select(x => x is SBPPartial ? SBP.B : x).ToArray();
         return new DistributedType(a.TensorType, ndsbp, a.Placement);
     }
 
diff --git a/src/Nncase.Evaluator/Math/Reduce.cs b/src/Nncase.Evaluator/Math/Reduce.cs
index 1f0e84ac3..1efa2b174 100644
--- a/src/Nncase.Evaluator/Math/Reduce.cs
+++ b/src/Nncase.Evaluator/Math/Reduce.cs
@@ -26,7 +26,7 @@ public class ReduceEvaluator : IEvaluator<Reduce>, ITypeInferencer<Reduce>, ICos
     public IValue Visit(IEvaluateContext context, Reduce reduce)
     {
         var input = context.GetOrtArgumentValue(reduce, Reduce.Input);
-        var axis = context.GetArgumentValueAsArray<long>(reduce, Reduce.Axis);
+        var axis = context.GetArgumentValueAsArray<long>(reduce, Reduce.Axes);
         var keepDims = context.GetArgumentValueAsScalar<long>(reduce, Reduce.KeepDims);
 
         // when HasBindedMixQuantInfo is true, eval will do simulation of quant/dequant for some inputs, this is used for evaluate accumulated quant error for layers.
@@ -64,7 +64,7 @@ public IValue Visit(IEvaluateContext context, Reduce reduce)
             ReduceOp.Prod => OrtKI.ReduceProd(input, axis, keepDims),
             ReduceOp.Sum => OrtKI.ReduceSum(
                 input,
-                context.GetInt64OrtTensorArgumentValue(reduce, Reduce.Axis),
+                context.GetInt64OrtTensorArgumentValue(reduce, Reduce.Axes),
                 keepDims,
                 0),
             _ => throw new ArgumentOutOfRangeException(nameof(reduce)),
@@ -116,7 +116,7 @@ public Cost Visit(ICostEvaluateContext context, Reduce target)
     public Expr Visit(IShapeEvaluateContext context, Reduce target)
     {
         var keepDims = context.GetArgument(target, Reduce.KeepDims);
-        var axis = context.GetArgument(target, Reduce.Axis);
+        var axis = context.GetArgument(target, Reduce.Axes);
         if (keepDims is TensorConst keepDimsV &&
             axis is TensorConst axisValue)
         {
@@ -169,8 +169,8 @@ public Metric Visit(IMetricEvaluateContext context, Reduce target)
 
     private IRType Visit(ITypeInferenceContext context, Reduce target, TensorType input)
     {
-        context.CheckArgumentType<TensorType>(target, Reduce.Axis);
-        var args = context.GetArguments(target, Reduce.KeepDims, Reduce.Axis);
+        context.CheckArgumentType<TensorType>(target, Reduce.Axes);
+        var args = context.GetArguments(target, Reduce.KeepDims, Reduce.Axes);
         return TypeInference.ReduceType(input, args[0], args[1]);
     }
 
@@ -181,8 +181,7 @@ private IRType Visit(ITypeInferenceContext context, Reduce target, DistributedTy
             throw new InvalidOperationException();
         }
 
-        var axis = ((TensorConst)context.GetArgument(target, Reduce.Axis)).Value.ToArray<int>();
-        var invalid = new InvalidType($"{input}, not support");
+        var axes = ((TensorConst)context.GetArgument(target, Reduce.Axes)).Value.ToArray<int>();
 
         var ndsbp = new SBP[input.Placement.Rank];
 
@@ -190,8 +189,9 @@ private IRType Visit(ITypeInferenceContext context, Reduce target, DistributedTy
         {
             switch (input.NdSBP[i])
             {
-                case SBPSplit { Axis: int ix } when axis.Contains(ix):
-                    return invalid;
+                case SBPSplit { Axis: int ix } when axes.Contains(ix):
+                    ndsbp[i] = SBP.P(target.ReduceOp);
+                    break;
                 default:
                     ndsbp[i] = input.NdSBP[i];
                     break;
diff --git a/src/Nncase.Evaluator/Math/ReduceArg.cs b/src/Nncase.Evaluator/Math/ReduceArg.cs
index ad865c0e5..9096688f5 100644
--- a/src/Nncase.Evaluator/Math/ReduceArg.cs
+++ b/src/Nncase.Evaluator/Math/ReduceArg.cs
@@ -129,7 +129,7 @@ private IRType Visit(ITypeInferenceContext context, ReduceArg target, Distribute
 
                         ndsbp[i] = keepdim ? SBP.S(saxis) : SBP.S(saxis > axis ? saxis - 1 : saxis);
                         break;
-                    case SBPPartialSum:
+                    case SBPPartial:
                         return new InvalidType("not support partial sum.");
                     case SBPBroadCast:
                         ndsbp[i] = SBP.B;
diff --git a/src/Nncase.Evaluator/Math/Unary.cs b/src/Nncase.Evaluator/Math/Unary.cs
index 91caf59ca..85edc284b 100644
--- a/src/Nncase.Evaluator/Math/Unary.cs
+++ b/src/Nncase.Evaluator/Math/Unary.cs
@@ -144,7 +144,7 @@ private static IRType Visit(DistributedType inType, UnaryOp unaryOp)
         var ndsbp = new SBP[inType.Placement.Rank];
         for (int i = 0; i < inType.Placement.Rank; i++)
         {
-            if (inType.NdSBP[i] is SBPPartialSum && unaryOp != UnaryOp.Neg)
+            if (inType.NdSBP[i] is SBPPartial && unaryOp != UnaryOp.Neg)
             {
                 return invalid;
             }
diff --git a/src/Nncase.Evaluator/NN/Activations.cs b/src/Nncase.Evaluator/NN/Activations.cs
index 7b36c04f9..0b2b54ea3 100644
--- a/src/Nncase.Evaluator/NN/Activations.cs
+++ b/src/Nncase.Evaluator/NN/Activations.cs
@@ -514,7 +514,7 @@ public Metric Visit(IMetricEvaluateContext context, Erf target)
 
     private IRType Visit(IRType input)
     {
-        if (input is DistributedType d && d.NdSBP.Any(s => s is SBPPartialSum))
+        if (input is DistributedType d && d.NdSBP.Any(s => s is SBPPartial))
         {
             return new InvalidType("Erf with partial sum is not supported");
         }
@@ -569,7 +569,7 @@ public Metric Visit(IMetricEvaluateContext context, Swish target)
 
     private IRType Visit(IRType input)
     {
-        if (input is DistributedType d && d.NdSBP.Any(s => s is SBPPartialSum))
+        if (input is DistributedType d && d.NdSBP.Any(s => s is SBPPartial))
         {
             return new InvalidType("swish with partial sum is not supported");
         }
@@ -626,7 +626,7 @@ public Metric Visit(IMetricEvaluateContext context, Gelu target)
 
     private IRType Visit(IRType input)
     {
-        if (input is DistributedType d && d.NdSBP.Any(s => s is SBPPartialSum))
+        if (input is DistributedType d && d.NdSBP.Any(s => s is SBPPartial))
         {
             return new InvalidType("gelu with partial sum is not supported");
         }
diff --git a/src/Nncase.Evaluator/NN/Conv2D.cs b/src/Nncase.Evaluator/NN/Conv2D.cs
index 74e536dd2..a400abcfb 100644
--- a/src/Nncase.Evaluator/NN/Conv2D.cs
+++ b/src/Nncase.Evaluator/NN/Conv2D.cs
@@ -146,7 +146,7 @@ private IRType Visit(ITypeInferenceContext context, Conv2D target, DistributedTy
                     {
                         if (bias.NdSBP[i] is SBPBroadCast)
                         {
-                            ndsbp[i] = SBP.P;
+                            ndsbp[i] = SBP.P();
                         }
                         else
                         {
diff --git a/src/Nncase.Evaluator/Tensors/Cast.cs b/src/Nncase.Evaluator/Tensors/Cast.cs
index 6f32939c3..af0641bfc 100644
--- a/src/Nncase.Evaluator/Tensors/Cast.cs
+++ b/src/Nncase.Evaluator/Tensors/Cast.cs
@@ -73,7 +73,7 @@ private IRType Visit(Cast target, DistributedType inType)
         var ndsbp = new SBP[inType.Placement.Rank];
         for (int i = 0; i < inType.Placement.Rank; i++)
         {
-            if (inType.NdSBP[i] is SBPPartialSum)
+            if (inType.NdSBP[i] is SBPPartial)
             {
                 return invalid;
             }
diff --git a/src/Nncase.Evaluator/Tensors/Concat.cs b/src/Nncase.Evaluator/Tensors/Concat.cs
index 942c821bd..0e9e08127 100644
--- a/src/Nncase.Evaluator/Tensors/Concat.cs
+++ b/src/Nncase.Evaluator/Tensors/Concat.cs
@@ -197,7 +197,7 @@ private IRType Visit(TupleType inputs, int axis)
                 return new InvalidType("not support distribute on concat axis");
             }
 
-            if (sbps.Any(sbp => sbp is SBPPartialSum))
+            if (sbps.Any(sbp => sbp is SBPPartial))
             {
                 return new InvalidType("not support distribute with partialsum");
             }
diff --git a/src/Nncase.Evaluator/Tensors/Reshape.cs b/src/Nncase.Evaluator/Tensors/Reshape.cs
index ab211fdca..4d1e9423c 100644
--- a/src/Nncase.Evaluator/Tensors/Reshape.cs
+++ b/src/Nncase.Evaluator/Tensors/Reshape.cs
@@ -23,6 +23,69 @@ namespace Nncase.Evaluator.Tensors;
 /// </summary>
 public class ReshapeEvaluator : IEvaluator<Reshape>, ITypeInferencer<Reshape>, ICostEvaluator<Reshape>, IShapeEvaluator<Reshape>, IMetricEvaluator<Reshape>
 {
+    public static IRType VisitDistributedType(DistributedType inType, int[] newShape)
+    {
+        var inShape = inType.TensorType.Shape.ToValueArray();
+        IRType invalidType = new InvalidType($"not supported reshape {inType} to [{string.Join(",", newShape)}]");
+        if (!IRUtility.TryGetShapeMapMatrix(inShape, newShape, out var mat))
+        {
+            return invalidType;
+        }
+
+        var (forwardDict, backwardDict) = IRUtility.ShapeMapMatrixAsDict(mat);
+        var ndsbp = new SBP[inType.NdSBP.Count];
+        for (int meshAxis = 0; meshAxis < inType.NdSBP.Count; meshAxis++)
+        {
+            var inSBP = inType.NdSBP[meshAxis];
+            switch (inSBP)
+            {
+                case SBPSplit si:
+                    {
+                        var mapedOutAxes = forwardDict[si.Axis];
+
+                        // when input axis is splited-by-reshape, we can direct obtain the tensor which splited-by-sbp on the first maped axis.
+                        if (mapedOutAxes.Count > 1)
+                        {
+                            var firstValidAxis = mapedOutAxes.Where(axis => newShape[axis] > 1).First();
+                            var restAxes = mapedOutAxes.Skip(mapedOutAxes.IndexOf(firstValidAxis) + 1).ToArray();
+                            var restSize = restAxes.Aggregate(1, (x, i) => x * newShape[i]);
+                            if (restSize < (inShape[si.Axis] / inType.Placement.Hierarchy[meshAxis]))
+                            {
+                                ndsbp[meshAxis] = SBP.S(firstValidAxis);
+                            }
+                            else
+                            {
+                                return invalidType;
+                            }
+                        }
+                        else
+                        {
+                            // when input axis is merged-by-reshape, we can direct obtain the tensor which splited-by-sbp on the first maped axis.
+                            var outAxis = mapedOutAxes.First();
+
+                            // when the outAxis is merged dim, only support no transpose order and no pad.
+                            var inAxes = backwardDict[outAxis];
+                            if (inAxes.Where(inAxis => inShape[inAxis] != 1).First() == si.Axis)
+                            {
+                                ndsbp[meshAxis] = SBP.S(outAxis);
+                            }
+                            else
+                            {
+                                return invalidType;
+                            }
+                        }
+                    }
+
+                    break;
+                default:
+                    ndsbp[meshAxis] = inSBP;
+                    break;
+            }
+        }
+
+        return new DistributedType(new TensorType(inType.TensorType.DType, newShape), ndsbp, inType.Placement);
+    }
+
     /// <inheritdoc/>
     public IValue Visit(IEvaluateContext context, Reshape reshape)
     {
@@ -158,75 +221,6 @@ private IRType Visit(ITypeInferenceContext context, Reshape target, DistributedT
         }
 
         var newShape = outTensorType.Shape.ToValueArray();
-        var oldShape = inputType.TensorType.Shape.ToValueArray();
-
-        // check is unsequeeze/sequeeze
-        if (Enumerable.SequenceEqual(oldShape.Where(i => i != 1).ToArray(), newShape.Where(i => i != 1).ToArray()))
-        {
-            if (oldShape.Length < newShape.Length)
-            {
-                var axis = 0;
-                var axisMap = new Dictionary<int, int>();
-                for (var n = 0; n < newShape.Length; n++)
-                {
-                    if (newShape[n] == oldShape[axis])
-                    {
-                        axisMap.Add(axis++, n);
-                        if (axis >= oldShape.Length)
-                        {
-                            break;
-                        }
-                    }
-                }
-
-                var ndsbp = new SBP[inputType.Placement.Rank];
-                for (int i = 0; i < inputType.Placement.Rank; i++)
-                {
-                    ndsbp[i] = inputType.NdSBP[i] switch
-                    {
-                        SBPSplit { Axis: int sx } => SBPSplit.S(axisMap[sx]),
-                        SBP sbp => sbp,
-                    };
-                }
-
-                return inputType with { TensorType = outTensorType, NdSBP = new(ndsbp) };
-            }
-            else if (oldShape.Length > newShape.Length)
-            {
-                var axis = 0;
-                var axisMap = new Dictionary<int, int>();
-                for (var o = 0; o < oldShape.Length; o++)
-                {
-                    if (axis < newShape.Length && oldShape[o] == newShape[axis])
-                    {
-                        axisMap.Add(o, axis++);
-                        if (axis >= newShape.Length)
-                        {
-                            break;
-                        }
-                    }
-                }
-
-                var ndsbp = new SBP[inputType.Placement.Rank];
-                for (int i = 0; i < inputType.Placement.Rank; i++)
-                {
-                    ndsbp[i] = inputType.NdSBP[i] switch
-                    {
-                        SBPSplit { Axis: int sx } => SBPSplit.S(axisMap[sx]),
-                        SBP sbp => sbp,
-                    };
-                }
-
-                return inputType with { TensorType = outTensorType, NdSBP = new(ndsbp) };
-            }
-        }
-
-        // not the squeeze or unsqueeze
-        if (!inputType.NdSBP.Any(sbp => sbp is SBPSplit))
-        {
-            return inputType with { TensorType = outTensorType, NdSBP = inputType.NdSBP };
-        }
-
-        return invalid;
+        return VisitDistributedType(inputType, newShape);
     }
 }
diff --git a/src/Nncase.Graph/Graphs/HyperGraph.cs b/src/Nncase.Graph/Graphs/HyperGraph.cs
new file mode 100644
index 000000000..902487334
--- /dev/null
+++ b/src/Nncase.Graph/Graphs/HyperGraph.cs
@@ -0,0 +1,526 @@
+// Copyright (c) Canaan Inc. All rights reserved.
+// Licensed under the Apache license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Nncase.Diagnostics;
+
+namespace Nncase.Graphs;
+
+public sealed class HyperGraph<TEClass, TENode>
+    where TENode : class
+    where TEClass : class
+{
+    private readonly Dictionary<int, Dictionary<int, HashSet<TENode>>> _edges;
+    private readonly HashSet<int> _nodes;
+    private readonly Dictionary<TEClass, int> _ids_to_nodes;
+    private readonly Dictionary<int, TEClass> _nodes_to_ids;
+    private int _num_nodes;
+
+    public HyperGraph()
+    {
+        _edges = new();
+        _nodes = new();
+        _ids_to_nodes = new();
+        _nodes_to_ids = new();
+        _num_nodes = 0;
+    }
+
+    public bool Contains(TEClass id)
+    {
+        return _ids_to_nodes.ContainsKey(id);
+    }
+
+    public List<List<int>> AdjacencyEdges()
+    {
+        return _edges.Select(kv => kv.Value.Keys.ToList()).ToList();
+    }
+
+    public Dictionary<TEClass, HashSet<TENode>>? Edges(TEClass eclass)
+    {
+        if (Contains(eclass))
+        {
+            var result = new Dictionary<TEClass, HashSet<TENode>>();
+            foreach (var (to, enodes) in _edges[_ids_to_nodes[eclass]])
+            {
+                result.Add(_nodes_to_ids[to], enodes);
+            }
+
+            return result;
+        }
+
+        return null;
+    }
+
+    public int NumEdges()
+    {
+        return _edges.Select(kv => kv.Value.Keys.Count).Sum();
+    }
+
+    public HashSet<TEClass> Nodes()
+    {
+        return new(_nodes.Select(x => _nodes_to_ids[x]));
+    }
+
+    public void AddNode(TEClass k)
+    {
+        var node_id = _num_nodes;
+        _ids_to_nodes.Add(k, node_id);
+        _nodes_to_ids.Add(node_id, k);
+        _edges.Add(node_id, new());
+        _nodes.Add(node_id);
+        _num_nodes += 1;
+    }
+
+    public void Connect(TEClass cfrom, TEClass cto, TENode enode)
+    {
+        if (!Contains(cfrom))
+        {
+            AddNode(cfrom);
+        }
+
+        if (!Contains(cto))
+        {
+            AddNode(cto);
+        }
+
+        var from = _ids_to_nodes[cfrom];
+        var to = _ids_to_nodes[cto];
+        if (!_edges[from].ContainsKey(to))
+        {
+            _edges[from].Add(to, new HashSet<TENode>(new[] { enode }));
+        }
+        else
+        {
+            _edges[from][to].Add(enode);
+        }
+    }
+
+    public TEClass[] Neighbors(TEClass u)
+    {
+        if (Contains(u))
+        {
+            return _edges[_ids_to_nodes[u]].Keys.Select(x => _nodes_to_ids[x]).ToArray();
+        }
+
+        return Array.Empty<TEClass>();
+    }
+
+    public int GetNodeById(TEClass id)
+    {
+        return _ids_to_nodes[id];
+    }
+
+    public TEClass GetIdByNode(int node)
+    {
+        return _nodes_to_ids[node];
+    }
+
+    public void RemoveNodeRaw(int node)
+    {
+        if (_nodes.Contains(node))
+        {
+            _edges.Remove(node);
+            foreach (var (_, v) in _edges)
+            {
+                v.Remove(node);
+            }
+
+            _nodes.Remove(node);
+        }
+    }
+
+    public void RemoveNode(TEClass node)
+    {
+        var node_id = _ids_to_nodes[node];
+        if (Contains(node))
+        {
+            _edges.Remove(node_id);
+            foreach (var (_, v) in _edges)
+            {
+                v.Remove(node_id);
+            }
+
+            _nodes.Remove(node_id);
+        }
+    }
+
+    public int Size()
+    {
+        return _nodes.Count;
+    }
+
+    public HyperGraph<TEClass, TENode> SubGraph(IEnumerable<TEClass> nodes)
+    {
+        var graph = new HyperGraph<TEClass, TENode>();
+        var node_set = new HashSet<TEClass>(nodes);
+        foreach (var n in node_set)
+        {
+            var nedges = _edges[_ids_to_nodes[n]];
+            foreach (var (neighbor, enodes) in nedges)
+            {
+                if (!node_set.Contains(_nodes_to_ids[neighbor]))
+                {
+                    continue;
+                }
+
+                foreach (var enode in enodes)
+                {
+                    graph.Connect(n, _nodes_to_ids[neighbor], enode);
+                }
+            }
+        }
+
+        return graph;
+    }
+
+    public void Dump(string name)
+    {
+        using (var dumpStream = DumpScope.Current.OpenFile($"Costs/{name}.dot"))
+        {
+            using var writer = new StreamWriter(dumpStream);
+            writer.WriteLine("digraph G {");
+            writer.WriteLine("/*");
+            writer.WriteLine("[");
+            foreach (var (_, v) in _edges.OrderBy(kv => kv.Key))
+            {
+                writer.WriteLine($"[{string.Join(",", v.Keys)}],");
+            }
+
+            writer.WriteLine("]");
+            writer.WriteLine("*/");
+
+            foreach (var node in _nodes)
+            {
+                writer.WriteLine($"{node} [label = \"{_nodes_to_ids[node]}\"]");
+            }
+
+            foreach (var (u, v) in _edges)
+            {
+                foreach (var w in v.Keys)
+                {
+                    writer.WriteLine($"{u} -> {w};");
+                }
+            }
+
+            writer.WriteLine("}");
+        }
+    }
+
+    public List<List<TEClass>> FindCycles()
+    {
+        var edges = AdjacencyEdges();
+        var circuits = new List<List<TEClass>>();
+        foreach (var n in Nodes())
+        {
+            if (Neighbors(n).Contains(n))
+            {
+                circuits.Add(new() { n });
+            }
+        }
+
+        var stack = new List<int>();
+        bool[] blocked = new bool[NumEdges()];
+        var blockMap = new Dictionary<int, Dictionary<int, bool>>();
+        List<List<int>> adjList;
+        int s = 0;
+
+        void Unblock(int u)
+        {
+            blocked[u] = false;
+            if (blockMap.ContainsKey(u))
+            {
+                foreach (var w in blockMap[u].Keys)
+                {
+                    blockMap[u].Remove(w);
+                    if (blocked[w])
+                    {
+                        Unblock(w);
+                    }
+                }
+            }
+        }
+
+        bool Circuit(int v)
+        {
+            bool found = false;
+
+            stack.Add(v);
+            blocked[v] = true;
+
+            for (int i = 0; i < adjList[v].Count; i++)
+            {
+                int w = adjList[v][i];
+                if (w == s)
+                {
+                    Output(s, stack);
+                    found = true;
+                }
+                else if (!blocked[w])
+                {
+                    found = Circuit(w);
+                }
+            }
+
+            if (found)
+            {
+                Unblock(v);
+            }
+            else
+            {
+                for (int i = 0; i < adjList[v].Count; i++)
+                {
+                    int w = adjList[v][i];
+                    if (!blockMap.ContainsKey(w))
+                    {
+                        blockMap[w] = new Dictionary<int, bool>();
+                    }
+
+                    blockMap[w][w] = true;
+                }
+            }
+
+            stack.RemoveAt(stack.Count - 1);
+            return found;
+        }
+
+        void Output(int start, List<int> stack)
+        {
+            // var cycle = new List<int>(stack);
+            // {
+            //     start,
+            // };
+            circuits.Add(stack.Select(GetIdByNode).ToList());
+        }
+
+        void Subgraph(int minId)
+        {
+            for (int i = 0; i < edges!.Count; i++)
+            {
+                if (i < minId || edges[i] == null)
+                {
+                    edges[i] = new List<int>();
+                }
+
+                edges[i] = edges[i].Where(j => j >= minId).ToList();
+            }
+        }
+
+        (int LeastVertex, List<List<int>> AdjList) AdjacencyStructureSCC(int from)
+        {
+            Subgraph(from);
+            List<List<int>> g = edges;
+
+            var (components, _) = StronglyConnectedComponents(edges);
+
+            var ccs = components.Where(scc => scc.Count > 1).ToList();
+
+            int leastVertex = int.MaxValue;
+            int leastVertexComponent = -1;
+            for (int i = 0; i < ccs.Count; i++)
+            {
+                for (int j = 0; j < ccs[i].Count; j++)
+                {
+                    if (ccs[i][j] < leastVertex)
+                    {
+                        leastVertex = ccs[i][j];
+                        leastVertexComponent = i;
+                    }
+                }
+            }
+
+            var cc = leastVertexComponent >= 0 ? ccs[leastVertexComponent] : null;
+
+            if (cc == null)
+            {
+                return (-1, new());
+            }
+
+            var adjList = edges.Select((l, index) =>
+            {
+                if (cc.IndexOf(index) == -1)
+                {
+                    return new();
+                }
+
+                return l.Where(i => cc.IndexOf(i) != -1).ToList();
+            }).ToList();
+
+            return (leastVertex, adjList);
+        }
+
+        while (s < edges.Count)
+        {
+            var (leastVertex, leastAdjList) = AdjacencyStructureSCC(s);
+            s = leastVertex;
+            adjList = leastAdjList;
+
+            if (adjList.Any())
+            {
+                for (int i = 0; i < adjList.Count; i++)
+                {
+                    for (int j = 0; j < adjList[i].Count; j++)
+                    {
+                        int vertexId = adjList[i][j];
+                        blocked[vertexId] = false;
+                        if (!blockMap.ContainsKey(vertexId))
+                        {
+                            blockMap[vertexId] = new Dictionary<int, bool>();
+                        }
+                    }
+                }
+
+                Circuit(s);
+                s++;
+            }
+            else
+            {
+                s = edges.Count;
+            }
+        }
+
+        return circuits;
+    }
+
+    private static (List<List<int>> Components, List<List<int>> AdjacencyList) StronglyConnectedComponents(List<List<int>> adjList)
+    {
+        int numVertices = adjList.Count;
+        int[] index = new int[numVertices];
+        int[] lowValue = new int[numVertices];
+        bool[] active = new bool[numVertices];
+        int[] child = new int[numVertices];
+        int[] scc = new int[numVertices];
+        var sccLinks = new List<int>[numVertices];
+
+        // Initialize tables
+        for (int i = 0; i < numVertices; ++i)
+        {
+            index[i] = -1;
+            lowValue[i] = 0;
+            active[i] = false;
+            child[i] = 0;
+            scc[i] = -1;
+            sccLinks[i] = new List<int>();
+        }
+
+        int count = 0;
+        var components = new List<List<int>>();
+        var sccAdjList = new List<List<int>>();
+
+        void StrongConnect(int v)
+        {
+            var s = new Stack<int>();
+            var t = new Stack<int>();
+            s.Push(v);
+            t.Push(v);
+            index[v] = lowValue[v] = count;
+            active[v] = true;
+            count++;
+
+            while (t.Count > 0)
+            {
+                v = t.Peek();
+                var e = adjList[v];
+                if (child[v] < e.Count)
+                {
+                    int i;
+                    for (i = child[v]; i < e.Count; ++i)
+                    {
+                        int u = e[i];
+                        if (index[u] < 0)
+                        {
+                            index[u] = lowValue[u] = count;
+                            active[u] = true;
+                            count++;
+                            s.Push(u);
+                            t.Push(u);
+                            break;
+                        }
+                        else if (active[u])
+                        {
+                            lowValue[v] = Math.Min(lowValue[v], lowValue[u]);
+                        }
+
+                        if (scc[u] >= 0)
+                        {
+                            sccLinks[v].Add(scc[u]);
+                        }
+                    }
+
+                    child[v] = i;
+                }
+                else
+                {
+                    if (lowValue[v] == index[v])
+                    {
+                        var component = new List<int>();
+                        var links = Enumerable.Range(0, s.Count).Select(i => new List<int>()).ToArray();
+                        int linkCount = 0;
+                        for (int i = s.Count - 1; i >= 0; --i)
+                        {
+                            int w = s.Pop();
+                            active[w] = false;
+                            component.Add(w);
+                            links[i] = sccLinks[w];
+                            linkCount += sccLinks[w].Count;
+                            scc[w] = components.Count;
+                            if (w == v)
+                            {
+                                break;
+                            }
+                        }
+
+                        components.Add(component);
+                        var allLinks = new List<int>(linkCount);
+                        for (int i = 0; i < links.Length; i++)
+                        {
+                            for (int j = 0; j < links[i].Count; j++)
+                            {
+                                allLinks.Add(links[i][j]);
+                            }
+                        }
+
+                        sccAdjList.Add(allLinks);
+                    }
+
+                    t.Pop();
+                }
+            }
+        }
+
+        // Run strong connect starting from each vertex
+        for (int i = 0; i < numVertices; ++i)
+        {
+            if (index[i] < 0)
+            {
+                StrongConnect(i);
+            }
+        }
+
+        // Compact sccAdjList
+        for (int i = 0; i < sccAdjList.Count; i++)
+        {
+            var e = sccAdjList[i];
+            if (e.Count == 0)
+            {
+                continue;
+            }
+
+            e.Sort();
+            var newE = new List<int> { e[0] };
+            for (int j = 1; j < e.Count; j++)
+            {
+                if (e[j] != e[j - 1])
+                {
+                    newE.Add(e[j]);
+                }
+            }
+
+            sccAdjList[i] = newE;
+        }
+
+        return (components, sccAdjList);
+    }
+}
diff --git a/src/Nncase.Passes/Rules/Neutral/CombineTranspose.cs b/src/Nncase.Passes/Rules/Neutral/CombineTranspose.cs
index 3cfa6b4a8..ab04f00d1 100644
--- a/src/Nncase.Passes/Rules/Neutral/CombineTranspose.cs
+++ b/src/Nncase.Passes/Rules/Neutral/CombineTranspose.cs
@@ -48,6 +48,40 @@ public CombineBinaryTranspose()
     }
 }
 
+/// <summary>
+/// Combine Transpose with Binary left
+/// binary(transpose(a,p),b) => transpose(binary(a, transpose(b,invP)),p).
+/// </summary>
+[RuleGenerator]
+public sealed partial class CombineBinaryLeftTranspose : IRewriteRule
+{
+    /// <inheritdoc/>
+    public IPattern Pattern { get; } = IsBinary("binary", "binaryCall", x => true, IsTranspose(IsWildcard("x"), IsTensorConst("perm")), IsWildcard("y"));
+
+    private Expr? GetReplace(Binary binary, Call binaryCall, Expr x, Expr y, int[] perm)
+    {
+        var invPerm = perm.Select((p, i) => (p, i)).OrderBy(tp => tp.p).Select(p => p.i).ToArray();
+        return Transpose(Binary(binary.BinaryOp, x, Transpose(y, invPerm)).InheritMetaData(binaryCall), perm);
+    }
+}
+
+/// <summary>
+/// Combine Transpose with Binary right
+/// binary(a,transpose(b,p)) => transpose(binary(transpose(a,invP),b),p).
+/// </summary>
+[RuleGenerator]
+public sealed partial class CombineBinaryRightTranspose : IRewriteRule
+{
+    /// <inheritdoc/>
+    public IPattern Pattern { get; } = IsBinary("binary", "binaryCall", x => true, IsWildcard("x"), IsTranspose(IsWildcard("y"), IsTensorConst("perm")));
+
+    private Expr? GetReplace(Binary binary, Call binaryCall, Expr x, Expr y, int[] perm)
+    {
+        var invPerm = perm.Select((p, i) => (p, i)).OrderBy(tp => tp.p).Select(p => p.i).ToArray();
+        return Transpose(Binary(binary.BinaryOp, Transpose(x, invPerm), y).InheritMetaData(binaryCall), perm);
+    }
+}
+
 /// <summary>
 /// Combine Transpose with Const Binary, if Const has rank 1.
 /// binary(transpose(a,p),const(b)) => transpose(binary(a,const(b)),p) or binary(const(a),transpose(b,p)) => transpose(binary(const(a),b),p).
@@ -390,10 +424,10 @@ public sealed partial class CombineTransposeReshape : IRewriteRule
 
 /// <summary>
 /// Combine Transpose with Unary
-/// reduce(transpose(x,p), a) => transpose(reduce(x, invtranspose(a, p)), p).
+/// unary(transpose(x,p), a) => transpose(unary(x, p)).
 /// </summary>
 [RuleGenerator]
-public sealed partial class CombineTransposeUnary : IRewriteRule
+public sealed partial class CombineUnaryTranspose : IRewriteRule
 {
     /// <inheritdoc/>
     public IPattern Pattern { get; } = IsUnary("unary", "unaryCall", x => true, IsTranspose(IsWildcard("input"), IsWildcard("perm")));
diff --git a/src/Nncase.Passes/Rules/Neutral/CombineUnary.cs b/src/Nncase.Passes/Rules/Neutral/CombineUnary.cs
index 3b7a3d793..b91dc6a20 100644
--- a/src/Nncase.Passes/Rules/Neutral/CombineUnary.cs
+++ b/src/Nncase.Passes/Rules/Neutral/CombineUnary.cs
@@ -56,7 +56,7 @@ public sealed partial class CombinePadUnary : IRewriteRule
 /// Combine <see cref="IR.Math.Unary"/>(<see cref="IR.Tensors.Transpose"/>).
 /// </summary>
 [RuleGenerator]
-public sealed partial class CombineTranposeUnary : IRewriteRule
+public sealed partial class CombineTransposeUnary : IRewriteRule
 {
     /// <inheritdoc/>
     public IPattern Pattern { get; } = IsUnary(
diff --git a/src/Nncase.Passes/packages.lock.json b/src/Nncase.Passes/packages.lock.json
index d0853a48b..a11315041 100644
--- a/src/Nncase.Passes/packages.lock.json
+++ b/src/Nncase.Passes/packages.lock.json
@@ -159,7 +159,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/src/Nncase.Quantization/packages.lock.json b/src/Nncase.Quantization/packages.lock.json
index 3fa55befa..5d17b5a1c 100644
--- a/src/Nncase.Quantization/packages.lock.json
+++ b/src/Nncase.Quantization/packages.lock.json
@@ -179,7 +179,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/src/Nncase.Schedule/Schedule/TilingUtilities.cs b/src/Nncase.Schedule/Schedule/TilingUtilities.cs
index dc0361ff7..9081cfe6b 100644
--- a/src/Nncase.Schedule/Schedule/TilingUtilities.cs
+++ b/src/Nncase.Schedule/Schedule/TilingUtilities.cs
@@ -54,7 +54,7 @@ public static int[] InferDomainBounds(int[][] bufferShapes, AffineMap[] accessMa
 
         if (solutionCollector.SolutionCount() < 1)
         {
-            throw new InvalidOperationException();
+            throw new InvalidOperationException("Tiling bounds infer failed!");
         }
 
         var dims = dimVars.Select(x => (int)solutionCollector.Value(0, x)).ToArray();
diff --git a/src/Nncase.Schedule/packages.lock.json b/src/Nncase.Schedule/packages.lock.json
index 59faa2297..02e78a2f5 100644
--- a/src/Nncase.Schedule/packages.lock.json
+++ b/src/Nncase.Schedule/packages.lock.json
@@ -150,7 +150,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/src/Nncase.Studio/packages.lock.json b/src/Nncase.Studio/packages.lock.json
index 0e5108f8d..d37425bf0 100644
--- a/src/Nncase.Studio/packages.lock.json
+++ b/src/Nncase.Studio/packages.lock.json
@@ -962,7 +962,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/src/Nncase.Targets/packages.lock.json b/src/Nncase.Targets/packages.lock.json
index b844eb92a..c59a0f2c0 100644
--- a/src/Nncase.Targets/packages.lock.json
+++ b/src/Nncase.Targets/packages.lock.json
@@ -158,7 +158,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/src/Nncase.Tests.TestFixture/packages.lock.json b/src/Nncase.Tests.TestFixture/packages.lock.json
index daab8cf85..3f365de3b 100644
--- a/src/Nncase.Tests.TestFixture/packages.lock.json
+++ b/src/Nncase.Tests.TestFixture/packages.lock.json
@@ -749,7 +749,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {
diff --git a/src/Nncase.Tests/Rules/Packing/PackUtilityTest.cs b/src/Nncase.Tests/Core/UnitTestIRUtilities.cs
similarity index 93%
rename from src/Nncase.Tests/Rules/Packing/PackUtilityTest.cs
rename to src/Nncase.Tests/Core/UnitTestIRUtilities.cs
index 2552bb1a2..d26fa53eb 100644
--- a/src/Nncase.Tests/Rules/Packing/PackUtilityTest.cs
+++ b/src/Nncase.Tests/Core/UnitTestIRUtilities.cs
@@ -8,9 +8,9 @@
 using Nncase.Utilities;
 using Xunit;
 
-namespace Nncase.Tests.Rules.Packing;
+namespace Nncase.Tests.CoreTest;
 
-public sealed class PackUtilityTest
+public sealed class UnitTestIRUtilities
 {
     [Theory]
     [InlineData(new object[] { new int[] { 1, 3, 2, 3, 1, 1, 7 }, new int[] { 1, 1, 3, 6, 1, 7 }, true })]
@@ -18,7 +18,7 @@ public sealed class PackUtilityTest
     [InlineData(new object[] { new int[] { 4, 4096 }, new int[] { 1, 4, 64, 64 }, true })]
     public void TestComputeReshapeMapping(int[] inShape, int[] newShape, bool valid)
     {
-        Assert.Equal(valid, PackUtility.TryGetShapeMapMatrix(inShape, newShape, out var mat));
+        Assert.Equal(valid, IRUtility.TryGetShapeMapMatrix(inShape, newShape, out var mat));
         if (valid)
         {
 #if DEBUG
diff --git a/src/Nncase.Tests/Distributed/UnitTestDistributeScheme.cs b/src/Nncase.Tests/Distributed/UnitTestDistributeSchema.cs
similarity index 87%
rename from src/Nncase.Tests/Distributed/UnitTestDistributeScheme.cs
rename to src/Nncase.Tests/Distributed/UnitTestDistributeSchema.cs
index 4c331c27f..15418ce94 100644
--- a/src/Nncase.Tests/Distributed/UnitTestDistributeScheme.cs
+++ b/src/Nncase.Tests/Distributed/UnitTestDistributeSchema.cs
@@ -12,12 +12,12 @@
 namespace Nncase.Tests.DistributedTest;
 
 [AutoSetupTestMethod(InitSession = true)]
-public class UnitTestDistributeScheme : TestClassBase
+public class UnitTestDistributeSchema : TestClassBase
 {
     [Fact]
     public void TestExportScheme()
     {
-        var scheme = new DistributedScheme("1", "llama", new DistributedScheme.Node[] { new("hidden_in", new SBP[] { SBP.S(1), SBP.S(2), SBP.B, SBP.P }, new[] { 8, 4, 4 }, "cbt") });
+        var scheme = new DistributedSchema("1", "llama", new DistributedSchema.Node[] { new("hidden_in", new SBP[] { SBP.S(1), SBP.S(2), SBP.B, SBP.P() }, new[] { 8, 4, 4 }, "cbt") });
         var except = @"{
   ""Version"": ""1"",
   ""Model"": ""llama"",
@@ -37,7 +37,8 @@ public void TestExportScheme()
           ""$type"": ""B""
         },
         {
-          ""$type"": ""P""
+          ""$type"": ""P"",
+          ""Op"": ""Sum""
         }
       ],
       ""Hierarchy"": [
@@ -56,7 +57,7 @@ public void TestExportScheme()
 #endif
         Assert.Equal(except, export);
 
-        var obj = JsonSerializer.Deserialize<DistributedScheme>(export);
+        var obj = JsonSerializer.Deserialize<DistributedSchema>(export);
     }
 
     [Fact]
diff --git a/src/Nncase.Tests/Distributed/UnitTestDistributedTypeInfer.cs b/src/Nncase.Tests/Distributed/UnitTestDistributedTypeInfer.cs
new file mode 100644
index 000000000..30a0bd4fc
--- /dev/null
+++ b/src/Nncase.Tests/Distributed/UnitTestDistributedTypeInfer.cs
@@ -0,0 +1,116 @@
+// Copyright (c) Canaan Inc. All rights reserved.
+// Licensed under the Apache license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Nncase.IR;
+using Nncase.Passes.Distributed;
+using Nncase.Tests.TestFixture;
+using Nncase.Utilities;
+using Xunit;
+
+namespace Nncase.Tests.DistributedTest;
+
+public sealed class UnitTestDistributedTypeInfer
+{
+    public static TheoryData<DistributedType, int[], IRType> ReshapeTypeInferData { get; } = new()
+    {
+        {
+            // split on not related axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 1024 }), new SBP[] { SBP.S(1) }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 64, 16 },
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 64, 16 }), new SBP[] { SBP.S(1) }, new(new[] { 8 }, "t"))
+        },
+        {
+            // split on splited-by-reshape axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 1024 }), new SBP[] { SBP.S(2) }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 64, 16 },
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 64, 16 }), new SBP[] { SBP.S(2) }, new(new[] { 8 }, "t"))
+        },
+        {
+            // split on splited-by-reshape axis, but less than split factor.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 1024 }), new SBP[] { SBP.S(2) }, new(new[] { 128 }, "t")),
+            new[] { 1, 48, 64, 16 },
+            new InvalidType("not support")
+        },
+        {
+            // split on sequeezed axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 1, 48, 1024 }), new SBP[] { SBP.S(2) }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 64, 16 },
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 64, 16 }), new SBP[] { SBP.S(1) }, new(new[] { 8 }, "t"))
+        },
+        {
+            // split on right unsequeeze axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 1024 }), new SBP[] { SBP.S(1) }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 1, 64, 16 },
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 1, 64, 16 }), new SBP[] { SBP.S(1) }, new(new[] { 8 }, "t"))
+        },
+        {
+            // split on left unsequeeze axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 384, 128 }), new SBP[] { SBP.S(1) }, new(new[] { 8 }, "t")),
+            new[] { 1, 1, 384, 128 },
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 1, 384, 128 }), new SBP[] { SBP.S(2) }, new(new[] { 8 }, "t"))
+        },
+        {
+            // split on merged-by-reshape axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 64, 16 }), new SBP[] { SBP.S(2) }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 1024 },
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 1024 }), new SBP[] { SBP.S(2) }, new(new[] { 8 }, "t"))
+        },
+        {
+            // split on merged-by-reshape axis, but not support.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 64, 16 }), new SBP[] { SBP.S(3) }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 1024 },
+            new InvalidType("not support")
+        },
+        {
+            // mesh dim 0 split on first merged-by-reshape axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 64, 16 }), new SBP[] { SBP.S(1), SBP.S(2) }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 1024 },
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 1024 }), new SBP[] { SBP.S(1), SBP.S(2) }, new(new[] { 8 }, "t"))
+        },
+        {
+            // mesh dim 1 split on first merged-by-reshape axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 64, 16 }), new SBP[] { SBP.S(2), SBP.S(1),  }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 1024 },
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 1024 }), new SBP[] { SBP.S(2), SBP.S(1), }, new(new[] { 8 }, "t"))
+        },
+        {
+            // split on second merged-by-reshape axis.
+            new DistributedType(new(DataTypes.Float32, new[] { 1, 48, 64, 16 }), new SBP[] { SBP.S(1), SBP.S(3) }, new(new[] { 8 }, "t")),
+            new[] { 1, 48, 1024 },
+            new InvalidType("not support")
+        },
+        {
+            // unmapable reshape
+            new DistributedType(new(DataTypes.Float32, new[] { 2, 30 }), new SBP[] { SBP.S(0) }, new(new[] { 6 }, "t")),
+            new[] { 3, 20 },
+            new InvalidType("unmapable")
+        },
+    };
+
+    [Fact]
+    public void TestGenerateReduceGroups()
+    {
+        Assert.Single(LinqUtility.Combination(1));
+        Assert.Equal(3, LinqUtility.Combination(2).Count());
+    }
+
+    [Theory]
+    [MemberData(nameof(ReshapeTypeInferData))]
+    public void TestReshapeTypeInfer(DistributedType inType, int[] newShape, IRType except)
+    {
+        var reshape = IR.F.Tensors.Reshape(new Var(inType), newShape);
+        if (except is InvalidType)
+        {
+            Assert.IsType<InvalidType>(reshape.CheckedType);
+        }
+        else
+        {
+            Assert.Equal(except, reshape.CheckedType);
+        }
+    }
+}
diff --git a/src/Nncase.Tests/Distributed/UnitTestDistributedUtilities.cs b/src/Nncase.Tests/Distributed/UnitTestDistributedUtilities.cs
deleted file mode 100644
index d8c608b46..000000000
--- a/src/Nncase.Tests/Distributed/UnitTestDistributedUtilities.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Canaan Inc. All rights reserved.
-// Licensed under the Apache license. See LICENSE file in the project root for full license information.
-
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text.Json;
-using System.Threading.Tasks;
-using Nncase.IR;
-using Nncase.Passes.Distributed;
-using Nncase.Tests.TestFixture;
-using Xunit;
-
-namespace Nncase.Tests.DistributedTest;
-
-public sealed class UnitTestDistributedUtilities
-{
-    [Fact]
-    public void TestGenerateReduceGroups()
-    {
-        Assert.Single(Utilities.LinqUtility.Combination(1));
-        Assert.Equal(3, Utilities.LinqUtility.Combination(2).Count());
-    }
-}
diff --git a/src/Nncase.Tests/Rewrite/Fusion/UnitTestGraphPartition.cs b/src/Nncase.Tests/Rewrite/Fusion/UnitTestGraphPartition.cs
index bb230ac52..db71d4625 100644
--- a/src/Nncase.Tests/Rewrite/Fusion/UnitTestGraphPartition.cs
+++ b/src/Nncase.Tests/Rewrite/Fusion/UnitTestGraphPartition.cs
@@ -113,8 +113,9 @@ public async Task TestLineSmaeModuleC()
     [Fact]
     public async Task TestLineDiffModuleC2I()
     {
-        var input = new Var("input", new TensorType(DataTypes.Float32, new int[] { 1, 32, 32 }));
-        var main = new Function("main", IR.F.Math.Abs(IR.F.CPU.Boxing(input, input.CheckedType)), input);
+        var inType = new TensorType(DataTypes.Float32, new int[] { 1, 32, 32 });
+        var input = new Var("input", inType);
+        var main = new Function("main", IR.F.CPU.Boxing(IR.F.Math.Abs(IR.F.CPU.Boxing(input, new DistributedType(inType, new SBP[] { SBP.B }, new(new[] { 2 }, "t")))), inType), input);
 
         Assert.True(CompilerServices.InferenceType(main));
 
@@ -148,8 +149,9 @@ public async Task TestLineDiffModuleC2I()
     [Fact]
     public async Task TestLineDiffModuleI2C()
     {
-        var input = new Var("input", new TensorType(DataTypes.Float32, new int[] { 1, 32, 32 }));
-        var main = new Function("main", IR.F.CPU.Boxing(IR.F.Math.Abs(input), input.CheckedType), input);
+        var inType = new TensorType(DataTypes.Float32, new int[] { 1, 32, 32 });
+        var input = new Var("input", inType);
+        var main = new Function("main", IR.F.CPU.Boxing(IR.F.Math.Abs(IR.F.CPU.Boxing(input, new DistributedType(inType, new SBP[] { SBP.B }, new(new[] { 2 }, "t")))), inType), input);
 
         Assert.True(CompilerServices.InferenceType(main));
 
diff --git a/src/Nncase.Tests/Rewrite/RewriteBase.cs b/src/Nncase.Tests/Rewrite/RewriteBase.cs
index cb2b6c822..1f536718e 100644
--- a/src/Nncase.Tests/Rewrite/RewriteBase.cs
+++ b/src/Nncase.Tests/Rewrite/RewriteBase.cs
@@ -150,7 +150,7 @@ public Function PreExpr
         typeof(Passes.Rules.Neutral.FoldConstCall),
         typeof(Passes.Rules.Neutral.FoldNopTranspose),
         typeof(Passes.Rules.Neutral.FoldTwoTransposes),
-        typeof(Passes.Rules.Neutral.CombineTransposeUnary),
+        typeof(Passes.Rules.Neutral.CombineUnaryTranspose),
         typeof(Passes.Rules.Neutral.CombineTransposePad),
         typeof(Passes.Rules.Neutral.CombinePadTranspose),
         typeof(Passes.Rules.Neutral.CombineBinaryTranspose),
@@ -2984,3 +2984,41 @@ public MatMulTransposeCase()
 
     public Dictionary<Var, IValue> FeedDict { get; }
 }
+
+public sealed class PaperCase : IRewriteCase
+{
+    public PaperCase()
+    {
+        var atype = new TensorType(DataTypes.Float32, new[] { 30, 40, 20 });
+        var a = new Var(atype);
+        var btype = new TensorType(DataTypes.Float32, new[] { 30, 20, 40 });
+        var b = new Var(btype);
+        {
+            // A: [2, 0, 1],  invA: [1,2,0]
+            // B: [1, 0, 2],  invB: [1,0,2]
+            var transA = IR.F.Tensors.Transpose(a, new[] { 2, 0, 1 }); // 20,30,40;
+            var transB = IR.F.Tensors.Transpose(b, new[] { 1, 0, 2 }); // 20,30,40;
+            var exp = IR.F.Math.Cos(transA + transB); // 20,30,40;
+            var transC = IR.F.Tensors.Transpose(exp, new[] { 1, 2, 0 }); // 30,40,20
+            PreExpr = new IR.Function(transC, a, b);
+        }
+
+        FeedDict = new Dictionary<Var, IValue>()
+        {
+            { a, IR.F.Random.Normal(atype.DType, 0, 1, 2, atype.Shape.ToValueArray()).Evaluate() },
+            { b, IR.F.Random.Normal(btype.DType, 0, 1, 2, btype.Shape.ToValueArray()).Evaluate() },
+        };
+    }
+
+    public Function PreExpr { get; }
+
+    public IEnumerable<System.Type> Rules => new[] {
+        typeof(Passes.Rules.Neutral.CombineBinaryLeftTranspose),
+        typeof(Passes.Rules.Neutral.CombineBinaryRightTranspose),
+        typeof(Passes.Rules.Neutral.CombineUnaryTranspose),
+        typeof(Passes.Rules.Neutral.FoldTwoTransposes),
+        typeof(Passes.Rules.Neutral.FoldNopTranspose),
+    };
+
+    public Dictionary<Var, IValue> FeedDict { get; }
+}
diff --git a/src/Nncase.Tests/Rewrite/UnitTestDataFlowRewrite.cs b/src/Nncase.Tests/Rewrite/UnitTestDataFlowRewrite.cs
index 1f25ee26d..3fe3904c5 100644
--- a/src/Nncase.Tests/Rewrite/UnitTestDataFlowRewrite.cs
+++ b/src/Nncase.Tests/Rewrite/UnitTestDataFlowRewrite.cs
@@ -311,6 +311,72 @@ public void TestWithAnalysisInfoRewriteOnce()
                     param3[1] is TensorConst);
     }
 
+    [Theory]
+    [InlineData([true, 0])]
+    [InlineData([false, 1])]
+    public void TestPaperCase(bool left, int count)
+    {
+        var atype = new TensorType(DataTypes.Float32, new[] { 30, 40, 20 });
+        var a = new Var(atype);
+        var btype = new TensorType(DataTypes.Float32, new[] { 30, 20, 40 });
+        var b = new Var(btype);
+        Function pre;
+        {
+            // A: [2, 0, 1],  invA: [1,2,0]
+            // B: [1, 0, 2],  invB: [1,0,2]
+            var transA = IR.F.Tensors.Transpose(a, new[] { 2, 0, 1 }); // 20,30,40;
+            var transB = IR.F.Tensors.Transpose(b, new[] { 1, 0, 2 }); // 20,30,40;
+            var exp = IR.F.Math.Cos(transA + transB); // 20,30,40;
+            var transC = IR.F.Tensors.Transpose(exp, new[] { 1, 2, 0 }); // 30,40,20
+            pre = new IR.Function(transC, a, b);
+        }
+
+        using var scope = new Diagnostics.DumpScope(count.ToString(), Diagnostics.DumpFlags.Rewrite | Diagnostics.DumpFlags.EGraphCost);
+#if DEBUG
+        Diagnostics.DumpScope.Current.DumpIR(pre, $"pre");
+#endif
+        Expr post = pre;
+        if (left)
+        {
+            post = new DataFlowRewriter(new Passes.Rules.Neutral.CombineBinaryLeftTranspose(), new()).Rewrite(post);
+        }
+        else
+        {
+            post = new DataFlowRewriter(new Passes.Rules.Neutral.CombineBinaryRightTranspose(), new()).Rewrite(post);
+        }
+#if DEBUG
+        var name = left ? "Left" : "Rigth";
+        Diagnostics.DumpScope.Current.DumpIR(post, $"CombineBinary{name}Transpose");
+#endif
+
+        post = new DataFlowRewriter(new Passes.Rules.Neutral.CombineUnaryTranspose(), new()).Rewrite(post);
+#if DEBUG
+        Diagnostics.DumpScope.Current.DumpIR(post, $"CombineUnaryTranspose");
+#endif
+        post = new DataFlowRewriter(new Passes.Rules.Neutral.FoldTwoTransposes(), new()).Rewrite(post);
+#if DEBUG
+        Diagnostics.DumpScope.Current.DumpIR(post, $"FoldTwoTransposes");
+#endif
+        post = new DataFlowRewriter(new Passes.Rules.Neutral.FoldNopTranspose(), new()).Rewrite(post);
+#if DEBUG
+        Diagnostics.DumpScope.Current.DumpIR(post, $"FoldNopTranspose");
+#endif
+
+#if DEBUG
+        Diagnostics.DumpScope.Current.DumpIR(post, "post");
+#endif
+
+        var feedDict = new Dictionary<Var, IValue>()
+        {
+            { a, IR.F.Random.Normal(atype.DType, 0, 1, 2, atype.Shape.ToValueArray()).Evaluate() },
+            { b, IR.F.Random.Normal(btype.DType, 0, 1, 2, btype.Shape.ToValueArray()).Evaluate() },
+        };
+
+        var preValue = pre.Body.Evaluate(feedDict);
+        var postValue = ((Function)post).Body.Evaluate(feedDict);
+        Assert.True(Comparator.Compare(preValue, postValue));
+    }
+
     [Fact]
     public void TestBroadcastNopPadOutputNames()
     {
diff --git a/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs b/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs
index a5f881cd5..3186faaeb 100644
--- a/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs
+++ b/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs
@@ -28,11 +28,12 @@ public UnitTestEGraphRewriteFactory()
 
     public static TheoryData<IRewriteCase> DataOne => new()
     {
-        new MatMulTransposeCase(),
+        new PaperCase(),
     };
 
     public static TheoryData<IRewriteCase> DataAll => new()
     {
+        new MatMulTransposeCase(),
         new ReshapeTransposeReshapeCase(),
         new FoldConv2DBnCase(),
         new ActivationsTransposePRelu(),
diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineTranspose.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineTranspose.cs
index dcde48a8b..ec6d5569c 100644
--- a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineTranspose.cs
+++ b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineTranspose.cs
@@ -357,7 +357,7 @@ public void TestCombineTransposeUnaryPositive(UnaryOp opType, int[] inShape, int
         var normal = new Dictionary<Var, IValue>();
         normal.Add(a, Random.Normal(DataTypes.Float32, 0, 1, 0, inShape).Evaluate());
         var rootPre = IR.F.Math.Unary(opType, Tensors.Transpose(a, perm));
-        TestMatched<CombineTransposeUnary>(rootPre, normal);
+        TestMatched<CombineUnaryTranspose>(rootPre, normal);
     }
 
     [Theory]
diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineUnary.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineUnary.cs
index 5ffd3c7eb..11d4a9169 100644
--- a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineUnary.cs
+++ b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineUnary.cs
@@ -170,6 +170,6 @@ public void TestCombineTranposeUnaryPositive(UnaryOp opType, int[] inShape, int[
         var normal = new Dictionary<Var, IValue>();
         normal.Add(a, Random.Normal(DataTypes.Float32, 0, 1, 0, inShape).Evaluate());
         var rootPre = IR.F.Math.Unary(opType, Tensors.Transpose(a, perm));
-        TestMatched<CombineTranposeUnary>(rootPre, normal);
+        TestMatched<CombineTransposeUnary>(rootPre, normal);
     }
 }
diff --git a/src/Nncase.Tests/Targets/UnitTestCPUKernels.cs b/src/Nncase.Tests/Targets/UnitTestCPUKernels.cs
index c8fd4859d..efddd0194 100644
--- a/src/Nncase.Tests/Targets/UnitTestCPUKernels.cs
+++ b/src/Nncase.Tests/Targets/UnitTestCPUKernels.cs
@@ -67,31 +67,74 @@ public UnitTestCPUKernels()
     public static int Rank => 2;
 
     [Theory]
-    [InlineData(new object[] { new[] { 32, 64 }, new[] { 64, 48 }, new[] { 48, 16 }, new[] { 1 }, 0 })]
-    [InlineData(new object[] { new[] { 128, 256 }, new[] { 256, 384 }, new[] { 384, 512 }, new[] { 1 }, 1 })]
-    [InlineData(new object[] { new[] { 1024, 2048 }, new[] { 2048, 1024 }, new[] { 1024, 3072 }, new[] { 1 }, 2, true })]
-    [InlineData(new object[] { new[] { 128, 256 }, new[] { 256, 384 }, new[] { 384, 512 }, new[] { 8 }, 3, false })]
-    public async Task TestTileFlowCase(int[] ashape, int[] bshape, int[] eshape, int[] hierarchy, int count, bool packing = false)
+    [InlineData(new object[] { new[] { 32, 64 }, false, new[] { 64, 48 }, false, new[] { 48, 16 }, true, new[] { 1 }, 0 })]
+    [InlineData(new object[] { new[] { 128, 256 }, true, new[] { 256, 384 }, false, new[] { 384, 512 }, true, new[] { 2 }, 1 })]
+    [InlineData(new object[] { new[] { 1024, 2048 }, false, new[] { 2048, 1024 }, true, new[] { 1024, 3072 }, true, new[] { 4 }, 2, true })]
+    [InlineData(new object[] { new[] { 128, 256 }, true, new[] { 256, 384 }, false, new[] { 384, 512 }, true, new[] { 8 }, 3, false })]
+    public async Task TestTileFlowCase(int[] ashape, bool constA, int[] bshape, bool constB, int[] eshape, bool constE, int[] hierarchy, int count, bool packing = false)
     {
         var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
         targetOptions.Hierarchies[0] = hierarchy;
         targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
         targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
         targetOptions.Packing = packing;
-        var a = new Var("a", new TensorType(DataTypes.Float32, ashape));
-        var b = new Var("b", new TensorType(DataTypes.Float32, bshape));
+        Expr a = constA ? Const.FromValue(IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, ashape).Evaluate()) : new Var("a", new TensorType(DataTypes.Float32, ashape));
+        Expr b = constB ? Const.FromValue(IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, bshape).Evaluate()) : new Var("b", new TensorType(DataTypes.Float32, bshape));
         var c = IR.F.Tensors.MatMul(a, b);
         var d = IR.F.Math.Neg(c);
-        var e = new Var("e", new TensorType(DataTypes.Float32, eshape));
+        Expr e = constE ? Const.FromValue(IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, eshape).Evaluate()) : new Var("e", new TensorType(DataTypes.Float32, eshape));
         var f = IR.F.Tensors.MatMul(d, e);
 
+        var feedDict = new Dictionary<Var, IValue>();
+        if (a is Var va)
+        {
+            feedDict.Add(va, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, ashape).Evaluate());
+        }
+
+        if (b is Var vb)
+        {
+            feedDict.Add(vb, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, bshape).Evaluate());
+        }
+
+        if (e is Var ve)
+        {
+            feedDict.Add(ve, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, eshape).Evaluate());
+        }
+
+        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{count}"), feedDict, new[] { f });
+    }
+
+    [Theory]
+    [InlineData([new[] { 1, 77, 768 }, new[] { 2, 32, 4 }, new int[] { 2, -1, 2, 2, 2, -1 }, 0])]
+    public async Task TestReshard(int[] shape, int[] hierarchy, int[] sbps, int count)
+    {
+        var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
+        targetOptions.Hierarchies[0] = hierarchy;
+        targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
+        targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+
+        var inputType = new TensorType(DataTypes.Float32, shape);
+        var input = new Var(inputType);
         var feedDict = new Dictionary<Var, IValue>() {
-            { a, IR.F.Tensors.ConstantOfShape(ashape, 1.0f).Evaluate() /* IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, ashape).Evaluate() */ },
-            { b, IR.F.Tensors.ConstantOfShape(bshape, 1.0f).Evaluate() /* IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, bshape).Evaluate() */ },
-            { e, IR.F.Tensors.ConstantOfShape(eshape, 1.0f).Evaluate() /* IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, eshape).Evaluate() */ },
+            // { input, IR.F.Tensors.ConstantOfShape(shape, 1.0f).Evaluate() },
+            { input, IR.F.Random.Normal(DataTypes.Float32, 1.0f, 1.0f, 1, shape).Evaluate() },
         };
 
-        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{count}"), feedDict, new[] { f });
+        var placement = new Placement(hierarchy, targetOptions.HierarchyNames);
+        var ndsbps = sbps.Chunk(hierarchy.Length).Select<int[], SBP[]>(sbp => sbp.Select<int, SBP>(s => s > 0 ? SBP.S(s) : SBP.B).ToArray()).ToArray();
+        Expr boxed = input;
+        foreach (var ndsbp in ndsbps)
+        {
+            boxed = IR.F.CPU.Boxing(boxed, new DistributedType(inputType, ndsbp, placement));
+        }
+
+        var post = IR.F.CPU.Boxing(boxed, inputType);
+        post.Metadata = new Passes.Distributed.AutoDistributedMetaData() { Skip = true };
+        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{count}"), feedDict, new[] { post });
     }
 
     [Theory]
@@ -105,6 +148,9 @@ public async Task TestGatherReduceScatter(int[] shape, int[] hierarchy, int coun
         targetOptions.Hierarchies[0] = hierarchy;
         targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
         targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+
         var inputType = new TensorType(DataTypes.Float32, shape);
         var input = new Var(inputType);
         var feedDict = new Dictionary<Var, IValue>() {
@@ -121,10 +167,10 @@ public async Task TestGatherReduceScatter(int[] shape, int[] hierarchy, int coun
             var newsbp = ndsbp.ToArray();
             foreach (var axis in comb)
             {
-                newsbp[axis] = SBP.P;
+                newsbp[axis] = SBP.P();
             }
 
-            var partial = IR.F.CPU.Boxing(broadcast, new DistributedType(inputType, newsbp, placement));
+            var partial = IR.F.CPU.ForceBoxing(broadcast, new DistributedType(inputType, newsbp, placement));
             var sumed = IR.F.CPU.Boxing(partial, new DistributedType(inputType, ndsbp, placement));
             var post = IR.F.CPU.Boxing(sumed, inputType);
             post.Metadata = new Passes.Distributed.AutoDistributedMetaData() { Skip = true };
@@ -134,37 +180,6 @@ public async Task TestGatherReduceScatter(int[] shape, int[] hierarchy, int coun
         await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{count}"), feedDict, posts);
     }
 
-    [Fact]
-    public async Task TestPartialReshapeBoxing()
-    {
-        var hierarchy = new[] { 2, 4 };
-        var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
-        targetOptions.Hierarchies[0] = hierarchy;
-        targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
-        targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
-        var lhsType = new TensorType(DataTypes.Float32, new[] { 1, 4, 8 });
-        var rhsType = new TensorType(DataTypes.Float32, new[] { 8, 16 });
-        var lhs = new Var(lhsType);
-        var rhs = new Var(rhsType);
-
-        var feedDict = new Dictionary<Var, IValue>() {
-            { lhs, IR.F.Random.Normal(DataTypes.Float32, 1.0f, 1.0f, 1, lhsType.Shape.ToValueArray()).Evaluate() },
-            { rhs, IR.F.Random.Normal(DataTypes.Float32, 1.0f, 1.0f, 1, rhsType.Shape.ToValueArray()).Evaluate() },
-        };
-
-        var placement = new Placement(hierarchy, targetOptions.HierarchyNames);
-        var lhsBoxing = IR.F.CPU.Boxing(lhs, new DistributedType(lhsType, new SBP[] { SBP.S(2), SBP.B }, placement));
-        var rhsBoxing = IR.F.CPU.Boxing(rhs, new DistributedType(rhsType, new SBP[] { SBP.S(0), SBP.S(1) }, placement));
-        var matmul = IR.F.Tensors.MatMul(lhsBoxing, rhsBoxing);
-        var newShape = new[] { 1, 4, 8, 2 };
-        var reshape = IR.F.CPU.Boxing(matmul, new DistributedType(new TensorType(DataTypes.Float32, newShape), new SBP[] { SBP.B, SBP.S(2) }, placement), true);
-        var sumed = IR.F.CPU.Boxing(reshape, new DistributedType(new TensorType(DataTypes.Float32, newShape), new SBP[] { SBP.S(1), SBP.S(2) }, placement));
-        var post = IR.F.CPU.Boxing(sumed, new TensorType(DataTypes.Float32, newShape));
-        post.Metadata = new Passes.Distributed.AutoDistributedMetaData() { Skip = true };
-
-        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{0}"), feedDict, new[] { post });
-    }
-
     [Fact]
     public async Task TestMatmulBinaryBinary()
     {
@@ -181,14 +196,12 @@ public async Task TestMatmulBinaryBinary()
         var g = IR.F.Math.Binary(BinaryOp.Add, e, f);
 
         var feedDict = new Dictionary<Var, IValue>() {
-            { a, IR.F.Tensors.ConstantOfShape(ashape, 1.0f).Evaluate() /* IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, ashape).Evaluate() */ },
-            { b, IR.F.Tensors.ConstantOfShape(bshape, 1.0f).Evaluate() /* IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, bshape).Evaluate() */ },
-            { d, IR.F.Tensors.ConstantOfShape(dshape, 1.0f).Evaluate() /* IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, eshape).Evaluate() */ },
-            { f, IR.F.Tensors.ConstantOfShape(fshape, 1.0f).Evaluate() /* IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, eshape).Evaluate() */ },
+            { a, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, ashape).Evaluate() },
+            { b, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, bshape).Evaluate() },
+            { d, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, dshape).Evaluate() },
+            { f, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, fshape).Evaluate() },
         };
 
-        // var inputs = feedDict.Values.Select(v => v.AsTensor()).ToArray();
-        // var actuals = Testing.RunKModel("/Users/lisa/Documents/nncase/tests_output/UnitTestCPUKernels/TestMatmulBinaryBinary/Case0/test.kmodel", Diagnostics.DumpScope.Current.Directory, inputs).AsTensors();
         await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), string.Empty), feedDict, new[] { g });
     }
 
@@ -216,6 +229,8 @@ public async Task TestUnary(int[] shape, int[] hierarchy, int count)
     {
         var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
         targetOptions.Hierarchies[0] = hierarchy;
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
         var input = new Var(new TensorType(DataTypes.Float32, shape));
         var pre = IR.F.Math.Unary(UnaryOp.Neg, input);
         var feedDict = new Dictionary<Var, IValue>() {
@@ -229,11 +244,19 @@ public async Task TestUnary(int[] shape, int[] hierarchy, int count)
     }
 
     [Theory]
-    [InlineData(new object[] { BinaryOp.Add, new[] { 8, 2 }, new int[] { 8, 2 }, 0 })] // normal
-    [InlineData(new object[] { BinaryOp.Mul, new[] { 1, 8, 64, 2 * 8 }, new int[] { 1, 1, 64, 2 * 8 }, 1 })] // broadcast
-    [InlineData(new object[] { BinaryOp.Add, new[] { 8, 16 }, new int[] { 16 }, 2 })] // normal
-    public async Task TestPackBinary(BinaryOp op, int[] lhsShape, int[] rhsShape, int count)
+    [InlineData(new object[] { BinaryOp.Add, new[] { 8, 2 }, new int[] { 8, 2 }, new int[] { 1 }, new int[] { }, 0 })] // normal
+    [InlineData(new object[] { BinaryOp.Mul, new[] { 1, 8, 64, 2 * 8 }, new int[] { 1, 1, 64, 2 * 8 }, new int[] { 1 }, new int[] { }, 1 })] // broadcast
+    [InlineData(new object[] { BinaryOp.Add, new[] { 8, 16 }, new int[] { 16 }, new int[] { 1 }, new int[] { }, 2 })] // normal
+    [InlineData(new object[] { BinaryOp.Mul, new[] { 1, 8, 64, 2 * 8 }, new int[] { 1, 1, 64, 2 * 8 }, new[] { 4 }, new[] { 1 }, 3 })] // broadcast
+    public async Task TestPackBinary(BinaryOp op, int[] lhsShape, int[] rhsShape, int[] hierarchy, int[] sbps, int count)
     {
+        var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
+        targetOptions.Hierarchies[0] = hierarchy;
+        targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
+        targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+
         var lhs = new Var(new TensorType(DataTypes.Float32, lhsShape));
         var rhs = new Var(new TensorType(DataTypes.Float32, rhsShape));
         var pre = IR.F.Math.Binary(op, lhs, rhs);
@@ -246,6 +269,28 @@ public async Task TestPackBinary(BinaryOp op, int[] lhsShape, int[] rhsShape, in
         var rule = new Passes.Rules.CPU.PackBinary(Rank, Lane);
         CompilerServices.TryMatch(pre, rule.Pattern, out var result);
         var posts = new[] { pre }.Concat(rule.GetReplaceCandidates(result!, new Passes.RunPassContext()));
+
+        if (sbps.Any(i => i != -1))
+        {
+            foreach (var post in posts)
+            {
+                var call = ExprCollector.Collect(post).Where(e => e is Call { Target: IR.CPU.PackedBinary or IR.Math.Binary }).First();
+                call.Metadata = new() { OutputNames = new[] { "call" } };
+            }
+
+            var scheme = new Passes.Distributed.DistributedSchema("1", "llama", [new("call", sbps.Select<int, SBP>(s => s > 0 ? SBP.S(s) : SBP.B).ToArray(), hierarchy, targetOptions.HierarchyNames)]);
+            var export = System.Text.Json.JsonSerializer.Serialize(scheme, new System.Text.Json.JsonSerializerOptions() { WriteIndented = true });
+            var dumpper = Diagnostics.DumpScope.Current.CreateSubDummper($"Theory{count}");
+            targetOptions.DistributedScheme = Path.Join(dumpper.Directory, "schema.json");
+            using (var stream = dumpper.OpenFile("schema.json"))
+            {
+                using (var writer = new StreamWriter(stream))
+                {
+                    writer.Write(export);
+                }
+            }
+        }
+
         await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{count}"), feedDict, posts);
     }
 
@@ -304,6 +349,8 @@ public async Task TestPackMatMul(int[] lhsShape, int[] rhsShape, bool constA, bo
         var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
         targetOptions.Hierarchies[0] = hierarchy;
         targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".Skip(3 - hierarchy.Length));
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
         var lhsTensor = IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, lhsShape).Evaluate().AsTensor(); // IR.F.Tensors.ConstantOfShape(lhsShape, 1.0f).Evaluate().AsTensor();
         var rhsTensor = IR.F.Random.Normal(DataTypes.Float32, 0, 1, 3, rhsShape).Evaluate().AsTensor(); // IR.F.Tensors.ConstantOfShape(rhsShape, 1.0f).Evaluate().AsTensor();
 
@@ -348,25 +395,68 @@ public async Task TestGather(int[] shape, int axis, int[] indicesShape, int coun
     }
 
     [Theory]
-    [InlineData([ReduceOp.Sum, new[] { 1, 64, 384, 128 }, new[] { 3 }, 0, true, 0])]
-    [InlineData([ReduceOp.Mean, new[] { 1, 384, 128 }, new[] { 2 }, 0, true, 1])]
-    public async Task TestPackReduce(ReduceOp reduceOp, int[] shape, int[] axes, float init, bool keepDims, int count)
+    [InlineData([ReduceOp.Sum, new[] { 1, 64, 384, 128 }, new[] { 3 }, 0, true, new[] { 1 }, new int[] { }, 0])]
+    [InlineData([ReduceOp.Mean, new[] { 1, 384, 128 }, new[] { 2 }, 0, true, new[] { 1 }, new int[] { }, 1])]
+    [InlineData([ReduceOp.Mean, new[] { 1, 384, 1024 }, new[] { 2 }, 0, true, new[] { 4 }, new int[] { 2 }, 2])]
+    [InlineData([ReduceOp.Max, new[] { 1, 384, 1024 }, new[] { 2 }, 0, true, new[] { 4 }, new int[] { 2 }, 3])]
+    [InlineData([ReduceOp.Min, new[] { 1, 384, 1024 }, new[] { 2 }, 0, true, new[] { 4 }, new int[] { 2 }, 4])]
+    [InlineData([ReduceOp.Sum, new[] { 1, 384, 1024 }, new[] { 2 }, 0, true, new[] { 4 }, new int[] { 2 }, 5])]
+    [InlineData([ReduceOp.Mean, new[] { 1, 3, 1024 }, new[] { 2 }, 0, true, new[] { 4 }, new int[] { 2 }, 6])]
+    [InlineData([ReduceOp.Sum, new[] { 1, 64, 384, 384 }, new[] { 3 }, 0, true, new[] { 64 }, new int[] { }, 7])]
+    public async Task TestPackReduce(ReduceOp reduceOp, int[] shape, int[] axes, float init, bool keepDims, int[] hierarchy, int[] splitedAxes, int number)
     {
-        var input = new Var(new TensorType(DataTypes.Float32, shape));
+        var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
+        targetOptions.Hierarchies[0] = hierarchy;
+        targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
+        targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+
+        var tensorType = new TensorType(DataTypes.Float32, shape);
+        var input = new Var(tensorType);
         var pre = IR.F.Tensors.Reduce(reduceOp, input, axes, init, keepDims);
 
         var feedDict = new Dictionary<Var, IValue>() {
             { input, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, shape).Evaluate() },
         };
 
+        IEnumerable<Expr> posts;
         var rule = new Passes.Rules.CPU.PackReduce(Rank, Lane);
         if (!CompilerServices.TryMatch(pre, rule.Pattern, out var result))
         {
             return;
         }
 
-        var posts = new[] { pre }.Concat(rule.GetReplaceCandidates(result, new Passes.RunPassContext()));
-        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{count}"), feedDict, posts);
+        posts = new[] { pre }.Concat(rule.GetReplaceCandidates(result, new Passes.RunPassContext()));
+
+        if (splitedAxes.Any(i => i != -1))
+        {
+            foreach (var post in posts)
+            {
+                if (post is Call { Target: IR.CPU.Unpack } callUnPack && callUnPack.Arguments[0] is Call { Target: IR.CPU.PackedReduce } packedReduceCall)
+                {
+                    packedReduceCall.Arguments[0].Metadata = new() { OutputNames = new[] { "reduceIn" } };
+                }
+                else if (post is Call { Target: IR.Math.Reduce } reduceCall)
+                {
+                    reduceCall.Arguments[0].Metadata = new() { OutputNames = new[] { "reduceIn" } };
+                }
+            }
+
+            var scheme = new Passes.Distributed.DistributedSchema("1", "llama", [new("reduceIn", splitedAxes.Select<int, SBP>(s => s > 0 ? SBP.S(s) : SBP.B).ToArray(), hierarchy, targetOptions.HierarchyNames)]);
+            var export = System.Text.Json.JsonSerializer.Serialize(scheme, new System.Text.Json.JsonSerializerOptions() { WriteIndented = true });
+            var dumpper = Diagnostics.DumpScope.Current.CreateSubDummper($"Theory{number}");
+            targetOptions.DistributedScheme = Path.Join(dumpper.Directory, "schema.json");
+            using (var stream = dumpper.OpenFile("schema.json"))
+            {
+                using (var writer = new StreamWriter(stream))
+                {
+                    writer.Write(export);
+                }
+            }
+        }
+
+        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{number}"), feedDict, posts);
     }
 
     [Theory]
@@ -405,6 +495,8 @@ public async Task TestPackReshape(int[] inshape, int[] outshape, int packRank, i
         targetOptions.Hierarchies[0] = hierarchy;
         targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
         targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
         var input = new Var("input", new TensorType(DataTypes.Float32, inshape));
         Expr pre;
         {
@@ -442,10 +534,17 @@ public async Task TestTranspose(int[] shape, int[] perm, int rank, int number)
         await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{number}"), feedDict, posts);
     }
 
-    [Fact]
-    public async Task TestTransposeMatmul()
+    [Theory]
+    [InlineData([new[] { 2, 4 }, 0])]
+    public async Task TestTransposeMatmul(int[] hierarchy, int number)
     {
-        ((CpuTargetOptions)CompileOptions.TargetOptions).Packing = true;
+        var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
+        targetOptions.Packing = true;
+        targetOptions.Hierarchies[0] = hierarchy;
+        targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
+        targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
 
         var v13 = new Var("v13", new TensorType(DataTypes.Float32, new[] { 1, 1, 384, 128 }));
         var v15 = new Var("v15", new TensorType(DataTypes.Float32, new[] { 1, 64, 384, 128 }));
@@ -468,7 +567,7 @@ public async Task TestTransposeMatmul()
         };
 
         var posts = new[] { pre };
-        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{0}"), feedDict, posts);
+        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{number}"), feedDict, posts);
     }
 
     [Theory]
@@ -498,8 +597,37 @@ public async Task TestConv2DAndIm2col(int[] inputShape, int[] wShape, int[] padd
     }
 
     [Theory]
+    [InlineData(new object[] { new[] { 1, 48, 512 }, new[] { 1, 512, 1024 }, new[] { 1, 48, 64, 16 }, new[] { UnaryOp.Neg, UnaryOp.Cos }, new[] { 8 }, 0 })]
+    [InlineData(new object[] { new[] { 1, 48, 512 }, new[] { 1, 512, 1024 }, new[] { 1, 64, 768 }, new[] { UnaryOp.Neg, UnaryOp.Cos }, new[] { 8 }, 1 })]
+    public async Task TestMatMulReshapeUnary(int[] lhsShape, int[] rhsShape, int[] newShape, UnaryOp[] unaryOps, int[] hierarchy, int number)
+    {
+        var targetOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
+        targetOptions.Hierarchies[0] = hierarchy;
+        targetOptions.HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
+        targetOptions.HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
+        targetOptions.HierarchyLatencies = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        targetOptions.HierarchyBandWidths = Enumerable.Repeat(1, hierarchy.Length).ToArray();
+        var lhs = new Var(new TensorType(DataTypes.Float32, lhsShape));
+        var rhs = new Var(new TensorType(DataTypes.Float32, rhsShape));
+        var matmul = IR.F.Tensors.MatMul(lhs, rhs);
+        var reshaped = IR.F.Tensors.Reshape(matmul, newShape);
+        var unary = reshaped;
+        foreach (var item in unaryOps)
+        {
+            unary = IR.F.Math.Unary(item, unary);
+        }
 
-    // [InlineData(new object[] { false, 0 })]
+        var feedDict = new Dictionary<Var, IValue>()
+        {
+            { lhs, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 1, lhsShape).Evaluate() },
+            { rhs, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 2, rhsShape).Evaluate() },
+        };
+
+        await RunCases(Path.Join(CompileOptions.DumpDir.ToString(), $"Theory{number}"), feedDict, new[] { unary });
+    }
+
+    [Theory(Skip = "ToBig")]
+    [InlineData(new object[] { false, 0 })]
     [InlineData(new object[] { true, 1 })] // enable packing
     public async Task TestDecodeLayer(bool packing, int count)
     {
@@ -509,12 +637,11 @@ public async Task TestDecodeLayer(bool packing, int count)
             return;
         }
 
-        var cpuOptions = (CpuTargetOptions)CompileOptions.TargetOptions;
-        cpuOptions.Packing = packing;
-        cpuOptions.Hierarchies = new[] { new[] { 2, 64 } };
-        cpuOptions.HierarchyNames = "bt";
-        cpuOptions.MemoryCapacities = [524288, 2147483647];
-        cpuOptions.MemoryBandWidths = [128, 64];
+        ((CpuTargetOptions)CompileOptions.TargetOptions).Packing = packing;
+        var hierarchy = new[] { 2, 4 };
+        ((CpuTargetOptions)CompileOptions.TargetOptions).Hierarchies[0] = hierarchy;
+        ((CpuTargetOptions)CompileOptions.TargetOptions).HierarchyNames = string.Join(string.Empty, "cbt".TakeLast(hierarchy.Length));
+        ((CpuTargetOptions)CompileOptions.TargetOptions).HierarchySizes = Enumerable.Repeat((int)MathF.Pow(2, 30), hierarchy.Length).ToArray();
         var vhidden_in = new Var("vhidden_in", new TensorType(DataTypes.Float32, new[] { 1, 384, 8192 }));
         var vattn_mask = new Var("vattn_mask", new TensorType(DataTypes.Float32, new[] { 1, 1, 384, 384 }));
         var vposition_ids = new Var("vposition_ids", new TensorType(DataTypes.Int64, new[] { 1, 384 }));
diff --git a/src/Nncase.Tests/packages.lock.json b/src/Nncase.Tests/packages.lock.json
index 2e67ced83..8d434dedb 100644
--- a/src/Nncase.Tests/packages.lock.json
+++ b/src/Nncase.Tests/packages.lock.json
@@ -846,7 +846,8 @@
           "Google.OrTools": "[9.4.1874, )",
           "NetFabric.Hyperlinq": "[3.0.0-beta48, )",
           "Nncase.Core": "[1.0.0, )",
-          "Nncase.Evaluator": "[1.0.0, )"
+          "Nncase.Evaluator": "[1.0.0, )",
+          "Nncase.Graph": "[1.0.0, )"
         }
       },
       "nncase.evaluator": {