Skip to content

Commit

Permalink
[AIE2P] Support fifo loads with extra storage - IR lowering
Browse files Browse the repository at this point in the history
  • Loading branch information
@khallouh
khallouh committed Feb 4, 2025
1 parent ce3dc2c commit 3dd61ab
Show file tree
Hide file tree
Showing 5 changed files with 689 additions and 2 deletions.
3 changes: 3 additions & 0 deletions clang/include/clang/Basic/BuiltinsAIE2P.def
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,3 +349,6 @@ BUILTIN(__builtin_aie2p_fifo_ld_pop_2d_576_bfp16, "vi*&V32i&i&iii&iV64c&V8c&", "
BUILTIN(__builtin_aie2p_fifo_ld_pop_3d_512_unaligned, "V64ci*&V32i&i&iii&iii&i", "nc")
BUILTIN(__builtin_aie2p_fifo_ld_pop_3d_544_bfp16, "vi*&V32i&i&iii&iii&iV64c&V8c&", "nc")
BUILTIN(__builtin_aie2p_fifo_ld_pop_3d_576_bfp16, "vi*&V32i&i&iii&iii&iV64c&V8c&", "nc")
// Unaligned and bfp16 loads with extra storage
BUILTIN(__builtin_aie2p_fifo_ld_fillx, "vi*&V32i&i&V16i&i", "nc")
BUILTIN(__builtin_aie2p_fifo_ld_popx, "V64ci*&V32i&i&V16i&i", "nc")
20 changes: 20 additions & 0 deletions clang/lib/CodeGen/CGBuiltin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22081,6 +22081,10 @@ static llvm::Intrinsic::ID getAIE2PIntrinsicFunction(unsigned BuiltinID) {
return Intrinsic::aie2p_fifo_st_flush_3d_conv;
case AIE::BI__builtin_aie2p_fifo_ld_fill:
return Intrinsic::aie2p_fifo_ld_fill;
case AIE::BI__builtin_aie2p_fifo_ld_fillx:
return Intrinsic::aie2p_fifo_ld_fillx;
case AIE::BI__builtin_aie2p_fifo_ld_popx:
return Intrinsic::aie2p_fifo_ld_popx;
case AIE::BI__builtin_aie2p_fifo_ld_pop_512_unaligned:
return Intrinsic::aie2p_fifo_ld_pop_unaligned;
case AIE::BI__builtin_aie2p_fifo_ld_pop_576_bfp16:
Expand Down Expand Up @@ -22495,6 +22499,7 @@ Value *CodeGenFunction::EmitAIEBuiltinExpr(unsigned BuiltinID,
Builder.CreateDefaultAlignedStore(Pos, PosAddr);
return Builder.CreateDefaultAlignedStore(Ptr, PtrAddr);
}
case AIE::BI__builtin_aie2p_fifo_ld_fillx:
case AIE::BI__builtin_aie2p_fifo_ld_fill: {
SmallVector<Value *, 3> Ops;
for (unsigned I = 0; I < E->getNumArgs(); I++)
Expand All @@ -22512,11 +22517,18 @@ Value *CodeGenFunction::EmitAIEBuiltinExpr(unsigned BuiltinID,
Value *FifoAddr = EmitLValue(E->getArg(1)).getPointer(*this);
Value *PosAddr = EmitLValue(E->getArg(2)).getPointer(*this);

if (BuiltinID == AIE::BI__builtin_aie2p_fifo_ld_fillx) {
Value *Extra = Builder.CreateExtractValue(Val, 3);
Value *ExtraAddr = EmitLValue(E->getArg(3)).getPointer(*this);
Builder.CreateDefaultAlignedStore(Extra, ExtraAddr);
}

Builder.CreateDefaultAlignedStore(Fifo, FifoAddr);
Builder.CreateDefaultAlignedStore(Pos, PosAddr);
return Builder.CreateDefaultAlignedStore(Ptr, PtrAddr);
}
case AIE::BI__builtin_aie2p_fifo_ld_pop_512_unaligned:
case AIE::BI__builtin_aie2p_fifo_ld_popx:
case AIE::BI__builtin_aie2p_fifo_ld_pop_1d_512_unaligned:
case AIE::BI__builtin_aie2p_fifo_ld_pop_2d_512_unaligned:
case AIE::BI__builtin_aie2p_fifo_ld_pop_3d_512_unaligned: {
Expand Down Expand Up @@ -22577,6 +22589,12 @@ Value *CodeGenFunction::EmitAIEBuiltinExpr(unsigned BuiltinID,
Builder.CreateDefaultAlignedStore(Count2, Count2Addr);
}

if (BuiltinID == AIE::BI__builtin_aie2p_fifo_ld_popx) {
Value *Extra = Builder.CreateExtractValue(Val, 4);
Value *ExtraAddr = EmitLValue(E->getArg(3)).getPointer(*this);
Builder.CreateDefaultAlignedStore(Extra, ExtraAddr);
}

Builder.CreateDefaultAlignedStore(Fifo, FifoAddr);
Builder.CreateDefaultAlignedStore(Pos, PosAddr);
Builder.CreateDefaultAlignedStore(Ptr, PtrAddr);
Expand Down Expand Up @@ -22863,6 +22881,8 @@ Value *CodeGenFunction::EmitAIE2PBuiltinExpr(unsigned BuiltinID,
case AIE::BI__builtin_aie2p_fifo_ld_pop_576_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_544_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_fill:
case AIE::BI__builtin_aie2p_fifo_ld_fillx:
case AIE::BI__builtin_aie2p_fifo_ld_popx:
case AIE::BI__builtin_aie2p_fifo_ld_pop_1d_512_unaligned:
case AIE::BI__builtin_aie2p_fifo_ld_pop_1d_576_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_1d_544_bfp16:
Expand Down
44 changes: 43 additions & 1 deletion clang/lib/Headers/aie2p_ldst.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,34 @@ FIFO_ST(__aie_dm_resource_cd, restrict)
return r; \
}

#define FIFO_FILLX(T, DM_BANK, RESTRICT) \
INTRINSIC(void) \
fifo_ld_fillx(T DM_BANK *RESTRICT &p, fifo_state_t &s, int step, int mask) { \
int &pos = s.pos; \
sparse_fifo_t &fifo = s.fifo; \
v16int32 &extra = s.extra; \
int conf = (step << 6) | mask; \
__builtin_aie2p_fifo_ld_fillx((int *&)p, fifo, pos, extra, conf); \
} \
\
INTRINSIC(void) fifo_ld_fillx(T DM_BANK *RESTRICT &p, fifo_state_t &s) { \
return fifo_ld_fillx(p, s, 31, 31); \
}

#define FIFO_POPX(T, DM_BANK, RESTRICT) \
INTRINSIC(T) \
fifo_ld_popx(T DM_BANK *RESTRICT &p, fifo_state_t &s, int step, int mask) { \
int &pos = s.pos; \
sparse_fifo_t &fifo = s.fifo; \
v16int32 &extra = s.extra; \
int conf = (step << 6) | mask; \
T r = (T)__builtin_aie2p_fifo_ld_popx((int *&)p, fifo, pos, extra, conf); \
return r; \
} \
INTRINSIC(T) fifo_ld_popx(T DM_BANK *RESTRICT &p, fifo_state_t &s) { \
return fifo_ld_popx(p, s, 31, 31); \
}

#define FIFO_LD(DM_BANK, RESTRICT) \
FIFO_LD_NORMAL(v32bfloat16, DM_BANK, RESTRICT) \
FIFO_LD_NORMAL(v16float, DM_BANK, RESTRICT) \
Expand All @@ -388,7 +416,19 @@ FIFO_ST(__aie_dm_resource_cd, restrict)
FIFO_LD_NORMAL(v16int32, DM_BANK, RESTRICT) \
FIFO_LD_NORMAL(v16uint32, DM_BANK, RESTRICT) \
FIFO_LD_BFP16(v64bfp16ebs8, 576, DM_BANK, RESTRICT) \
FIFO_LD_BFP16(v64bfp16ebs16, 544, DM_BANK, RESTRICT)
FIFO_LD_BFP16(v64bfp16ebs16, 544, DM_BANK, RESTRICT) \
FIFO_POPX(v32bfloat16, DM_BANK, RESTRICT) \
FIFO_POPX(v16float, DM_BANK, RESTRICT) \
FIFO_POPX(v128int4, DM_BANK, RESTRICT) \
FIFO_POPX(v128uint4, DM_BANK, RESTRICT) \
FIFO_POPX(v64int8, DM_BANK, RESTRICT) \
FIFO_POPX(v64uint8, DM_BANK, RESTRICT) \
FIFO_POPX(v32int16, DM_BANK, RESTRICT) \
FIFO_POPX(v32uint16, DM_BANK, RESTRICT) \
FIFO_POPX(v16int32, DM_BANK, RESTRICT) \
FIFO_POPX(v16uint32, DM_BANK, RESTRICT) \
FIFO_FILLX(v64bfp16ebs8_unaligned, DM_BANK, RESTRICT) \
FIFO_FILLX(v64bfp16ebs16_unaligned, DM_BANK, RESTRICT)

FIFO_LD(, )
FIFO_LD(__aie_dm_resource_a, )
Expand Down Expand Up @@ -416,6 +456,8 @@ FIFO_LD(__aie_dm_resource_cd, restrict)

#undef FIFO_LD_NORMAL
#undef FIFO_LD_BFP16
#undef FIFO_POPX
#undef FIFO_FILLX
#undef FIFO_LD

#endif // AIE2P_LDST_H
Loading

0 comments on commit 3dd61ab

Please sign in to comment.