Merge branch 'aie-public' into sagarm.legal.g_and

.github/workflows/amd-aie-distro.yml

@@ -317,7 +317,7 @@ jobs:
 
     - name: Upload wheels
       if: ${{ needs.settings.outputs.UPLOAD_ARTIFACTS == 'true' }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # 4.6.0
       with:
         path: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}/wheelhouse/*.whl
         name: build_artifact
@@ -335,7 +335,7 @@ jobs:
       contents: write
 
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # 4.1.8
         with:
           # unpacks default artifact into dist/
           # if `name: artifact` is omitted, the action will create extra parent dir

.github/workflows/build-ci-container.yml

@@ -44,7 +44,7 @@ jobs:
         run: |
           podman save stage1-toolchain > stage1-toolchain.tar
       - name: Upload container image
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # 4.6.0
         with:
           name: stage1-toolchain
           path: stage1-toolchain.tar
@@ -81,7 +81,7 @@ jobs:
         # $GITHUB_WORKSPACE to avoid space limitations on the default drive
         # and use the permissions setup for /mnt/podman.
       - name: Download stage1-toolchain
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # 4.1.8
         with:
           name: stage1-toolchain
           path: /mnt/podman

.github/workflows/libclang-abi-tests.yml

@@ -131,7 +131,7 @@ jobs:
             sed -i 's/LLVM_[0-9]\+/LLVM_NOVERSION/' $lib-${{ matrix.ref }}.abi
           done
       - name: Upload ABI file
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # 4.6.0
         with:
           name: ${{ matrix.name }}
           path: '*${{ matrix.ref }}.abi'
@@ -144,12 +144,12 @@ jobs:
       - abi-dump
     steps:
       - name: Download baseline
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # 4.1.8
         with:
           name: build-baseline
           path: build-baseline
       - name: Download latest
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # 4.1.8
         with:
           name: build-latest
           path: build-latest
@@ -163,7 +163,7 @@ jobs:
           done
       - name: Upload ABI Comparison
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # 4.6.0
         with:
           name: compat-report-${{ github.sha }}
           path: compat_reports/

.github/workflows/llvm-tests.yml

@@ -137,14 +137,14 @@ jobs:
           # Remove symbol versioning from dumps, so we can compare across major versions.
           sed -i 's/LLVM_${{ matrix.llvm_version_major }}/LLVM_NOVERSION/' ${{ matrix.ref }}.abi
       - name: Upload ABI file
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # 4.6.0
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.ref }}.abi
 
       - name: Upload symbol list file
         if: matrix.name == 'build-baseline'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # 4.6.0
         with:
           name: symbol-list
           path: llvm.symbols
@@ -157,17 +157,17 @@ jobs:
       - abi-dump
     steps:
       - name: Download baseline
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # 4.1.8
         with:
           name: build-baseline
           path: build-baseline
       - name: Download latest
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # 4.1.8
         with:
           name: build-latest
           path: build-latest
       - name: Download symbol list
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # 4.1.8
         with:
           name: symbol-list
           path: symbol-list
@@ -186,7 +186,7 @@ jobs:
           abi-compliance-checker $EXTRA_ARGS -l libLLVM.so -old build-baseline/*.abi -new build-latest/*.abi || test "${{ needs.abi-dump-setup.outputs.ABI_HEADERS }}" = "llvm-c"
       - name: Upload ABI Comparison
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # 4.6.0
         with:
           name: compat-report-${{ github.sha }}
           path: compat_reports/

.github/workflows/release-documentation.yml

@@ -59,7 +59,7 @@ jobs:
           ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-doxygen
 
       - name: Create Release Notes Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # 4.6.0
         with:
           name: release-notes
           path: docs-build/html-export/

clang/include/clang/Basic/BuiltinsAIE2P.def

@@ -4,7 +4,7 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// (c) Copyright 2024 Advanced Micro Devices, Inc. or its affiliates
+// (c) Copyright 2024-2025 Advanced Micro Devices, Inc. or its affiliates
 //
 //===----------------------------------------------------------------------===//
 //
@@ -320,3 +320,18 @@ BUILTIN(__builtin_aie2p_tanh, "V16yV16g", "nc")
 
 //division/mod
 BUILTIN(__builtin_aie2p_divstep, "vUi&Ui&Ui", "nc")
+
+// SHUFFLE
+BUILTIN(__builtin_aie2p_vshuffle_576_bfp16, "vV64cV8cV64cV8ciV64c&V8c&", "nc")
+// Unaligned and bfp16 stores
+BUILTIN(__builtin_aie2p_fifo_st_push_576_bfp16, "vi*&V64cV8cV32i&i&", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_push_512_bfp16, "vi*&V16iV32i&i&", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_push_544_bfp16, "vi*&V64cV8cV32i&i&", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_flush, "vi*&V32i&i&", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_flush_1d_byte, "vi*&V32i&i&i", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_flush_2d_byte, "vi*&V32i&i&iii&i", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_flush_3d_byte, "vi*&V32i&i&iii&iii&i", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_flush_conv, "vi*&V32i&i&", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_flush_conv_1d_byte, "vi*&V32i&i&i", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_flush_conv_2d_byte, "vi*&V32i&i&iii&i", "nc")
+BUILTIN(__builtin_aie2p_fifo_st_flush_conv_3d_byte, "vi*&V32i&i&iii&iii&i", "nc")

clang/lib/CodeGen/CGBuiltin.cpp

@@ -4,7 +4,7 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// Modifications (c) Copyright 2023-2024 Advanced Micro Devices, Inc. or its
+// Modifications (c) Copyright 2023-2025 Advanced Micro Devices, Inc. or its
 // affiliates
 //
 //===----------------------------------------------------------------------===//
@@ -64,6 +64,7 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/MatrixBuilder.h"
 #include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/TargetParser/AArch64TargetParser.h"
@@ -22054,6 +22055,30 @@ static llvm::Intrinsic::ID getAIE2PIntrinsicFunction(unsigned BuiltinID) {
     return Intrinsic::aie2p_v64accfloat_to_v64bfp16ebs16;
   case AIE::BI__builtin_aie2p_v64bfp16ebs8_to_v64bfp16ebs16:
     return Intrinsic::aie2p_v64bfp16ebs8_to_v64bfp16ebs16;
+  case AIE::BI__builtin_aie2p_vshuffle_576_bfp16:
+    return Intrinsic::aie2p_vshuffle_576_bfp16;
+  case AIE::BI__builtin_aie2p_fifo_st_push_512_bfp16:
+    return Intrinsic::aie2p_fifo_st_push_512_bfp16;
+  case AIE::BI__builtin_aie2p_fifo_st_push_544_bfp16:
+    return Intrinsic::aie2p_fifo_st_push_544_bfp16;
+  case AIE::BI__builtin_aie2p_fifo_st_push_576_bfp16:
+    return Intrinsic::aie2p_fifo_st_push_576_bfp16;
+  case AIE::BI__builtin_aie2p_fifo_st_flush:
+    return Intrinsic::aie2p_fifo_st_flush;
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv:
+    return Intrinsic::aie2p_fifo_st_flush_conv;
+  case AIE::BI__builtin_aie2p_fifo_st_flush_1d_byte:
+    return Intrinsic::aie2p_fifo_st_flush_1d;
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_1d_byte:
+    return Intrinsic::aie2p_fifo_st_flush_1d_conv;
+  case AIE::BI__builtin_aie2p_fifo_st_flush_2d_byte:
+    return Intrinsic::aie2p_fifo_st_flush_2d_conv;
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_2d_byte:
+    return Intrinsic::aie2p_fifo_st_flush_2d;
+  case AIE::BI__builtin_aie2p_fifo_st_flush_3d_byte:
+    return Intrinsic::aie2p_fifo_st_flush_3d;
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_3d_byte:
+    return Intrinsic::aie2p_fifo_st_flush_3d_conv;
   default:
     break;
   }
@@ -22291,6 +22316,7 @@ Value *CodeGenFunction::EmitAIEBuiltinExpr(unsigned BuiltinID,
       Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 2)));
       Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
     }
+
     llvm::Intrinsic::ID IntrinsicID = getAIEIntrinsicFunction(BuiltinID, Arch);
     assert(IntrinsicID != Intrinsic::not_intrinsic);
     Function *F = CGM.getIntrinsic(IntrinsicID);
@@ -22305,6 +22331,144 @@ Value *CodeGenFunction::EmitAIEBuiltinExpr(unsigned BuiltinID,
 
     return Builder.CreateDefaultAlignedStore(Exp, ExpAddr);
   }
+  case AIE::BI__builtin_aie2p_vshuffle_576_bfp16: {
+    SmallVector<Value *, 3> Ops;
+    for (unsigned I = 0; I < E->getNumArgs() - 2; I++)
+      Ops.push_back(EmitScalarExpr(E->getArg(I)));
+
+    llvm::Intrinsic::ID IntrinsicID = getAIEIntrinsicFunction(BuiltinID, Arch);
+    assert(IntrinsicID != Intrinsic::not_intrinsic);
+    Function *F = CGM.getIntrinsic(IntrinsicID);
+    Value *Val = Builder.CreateCall(F, Ops);
+
+    // The first member of the returned struct is the mantissa part of bfp16,
+    // store it to the first input reference
+    Value *Mant = Builder.CreateExtractValue(Val, 0);
+    Value *MantAddr =
+        EmitLValue(E->getArg(E->getNumArgs() - 2)).getPointer(*this);
+    Builder.CreateDefaultAlignedStore(Mant, MantAddr);
+
+    // The second member of the returned struct is the exponent part of bfp16
+    // store it to the second input reference
+    Value *Exp = Builder.CreateExtractValue(Val, 1);
+    Value *ExpAddr =
+        EmitLValue(E->getArg(E->getNumArgs() - 1)).getPointer(*this);
+    return Builder.CreateDefaultAlignedStore(Exp, ExpAddr);
+  }
+  case AIE::BI__builtin_aie2p_fifo_st_push_512_bfp16:
+  case AIE::BI__builtin_aie2p_fifo_st_push_544_bfp16:
+  case AIE::BI__builtin_aie2p_fifo_st_push_576_bfp16:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv:
+  case AIE::BI__builtin_aie2p_fifo_st_flush: {
+    SmallVector<Value *, 3> Ops;
+    for (unsigned I = 0; I < E->getNumArgs(); I++)
+      Ops.push_back(EmitScalarExpr(E->getArg(I)));
+
+    llvm::Intrinsic::ID IntrinsicID = getAIEIntrinsicFunction(BuiltinID, Arch);
+    assert(IntrinsicID != Intrinsic::not_intrinsic);
+    Function *F = CGM.getIntrinsic(IntrinsicID);
+    Value *Val = Builder.CreateCall(F, Ops);
+
+    Value *Ptr = Builder.CreateExtractValue(Val, 0);
+    Value *Fifo = Builder.CreateExtractValue(Val, 1);
+    Value *Avail = Builder.CreateExtractValue(Val, 2);
+    Value *PtrAddr = EmitLValue(E->getArg(0)).getPointer(*this);
+
+    Value *FifoAddr = nullptr;
+    Value *AvailAddr = nullptr;
+    if (BuiltinID == AIE::BI__builtin_aie2p_fifo_st_flush ||
+        BuiltinID == AIE::BI__builtin_aie2p_fifo_st_flush_conv) {
+      FifoAddr = EmitLValue(E->getArg(1)).getPointer(*this);
+      AvailAddr = EmitLValue(E->getArg(2)).getPointer(*this);
+    } else if (BuiltinID == AIE::BI__builtin_aie2p_fifo_st_push_512_bfp16) {
+      FifoAddr = EmitLValue(E->getArg(2)).getPointer(*this);
+      AvailAddr = EmitLValue(E->getArg(3)).getPointer(*this);
+    } else {
+      assert(BuiltinID == AIE::BI__builtin_aie2p_fifo_st_push_576_bfp16 ||
+             BuiltinID == AIE::BI__builtin_aie2p_fifo_st_push_544_bfp16 &&
+                 "Unexpected BuiltinID");
+      FifoAddr = EmitLValue(E->getArg(3)).getPointer(*this);
+      AvailAddr = EmitLValue(E->getArg(4)).getPointer(*this);
+    }
+
+    Builder.CreateDefaultAlignedStore(Fifo, FifoAddr);
+    Builder.CreateDefaultAlignedStore(Avail, AvailAddr);
+    return Builder.CreateDefaultAlignedStore(Ptr, PtrAddr);
+  }
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_1d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_1d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_2d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_2d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_3d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_3d_byte: {
+    SmallVector<Value *, 3> Ops;
+    unsigned NumAddrIncs = 0;
+    switch (BuiltinID) {
+    case AIE::BI__builtin_aie2p_fifo_st_flush_1d_byte:
+    case AIE::BI__builtin_aie2p_fifo_st_flush_conv_1d_byte:
+      NumAddrIncs = 1;
+      break;
+    case AIE::BI__builtin_aie2p_fifo_st_flush_2d_byte:
+    case AIE::BI__builtin_aie2p_fifo_st_flush_conv_2d_byte:
+      NumAddrIncs = 4;
+      break;
+    case AIE::BI__builtin_aie2p_fifo_st_flush_3d_byte:
+    case AIE::BI__builtin_aie2p_fifo_st_flush_conv_3d_byte:
+      NumAddrIncs = 7;
+      break;
+    default:
+      llvm_unreachable("Unexpected BuiltinID");
+    }
+
+    for (unsigned I = 0; I < E->getNumArgs() - NumAddrIncs; I++)
+      Ops.push_back(EmitScalarExpr(E->getArg(I)));
+
+    for (unsigned i = E->getNumArgs() - NumAddrIncs, e = E->getNumArgs();
+         i != e; i++) {
+      Ops.push_back(
+          Builder.CreateTrunc(EmitScalarExpr(E->getArg(i)),
+                              llvm::Type::getInt20Ty(getLLVMContext())));
+    }
+    llvm::Intrinsic::ID IntrinsicID = getAIEIntrinsicFunction(BuiltinID, Arch);
+    assert(IntrinsicID != Intrinsic::not_intrinsic);
+    Function *F = CGM.getIntrinsic(IntrinsicID);
+    Value *Val = Builder.CreateCall(F, Ops);
+
+    Value *Ptr = Builder.CreateExtractValue(Val, 0);
+    Value *Fifo = Builder.CreateExtractValue(Val, 1);
+    Value *Avail = Builder.CreateExtractValue(Val, 2);
+    Value *PtrAddr = EmitLValue(E->getArg(0)).getPointer(*this);
+    Value *FifoAddr = EmitLValue(E->getArg(1)).getPointer(*this);
+    Value *AvailAddr = EmitLValue(E->getArg(2)).getPointer(*this);
+
+    if (BuiltinID == AIE::BI__builtin_aie2p_fifo_st_flush_2d_byte ||
+        BuiltinID == AIE::BI__builtin_aie2p_fifo_st_flush_conv_2d_byte) {
+      Value *Count1 =
+          Builder.CreateZExt(Builder.CreateExtractValue(Val, 3),
+                             llvm::Type::getInt32Ty(getLLVMContext()));
+      Value *Count1Addr = EmitLValue(E->getArg(5)).getPointer(*this);
+      Builder.CreateDefaultAlignedStore(Count1, Count1Addr);
+    } else if (BuiltinID == AIE::BI__builtin_aie2p_fifo_st_flush_3d_byte ||
+               BuiltinID == AIE::BI__builtin_aie2p_fifo_st_flush_conv_3d_byte) {
+      Value *Count1 =
+          Builder.CreateZExt(Builder.CreateExtractValue(Val, 3),
+                             llvm::Type::getInt32Ty(getLLVMContext()));
+      Value *Count1Addr =
+          EmitLValue(E->getArg(E->getNumArgs() - 5)).getPointer(*this);
+      Value *Count2 =
+          Builder.CreateZExt(Builder.CreateExtractValue(Val, 4),
+                             llvm::Type::getInt32Ty(getLLVMContext()));
+      EmitLValue(E->getArg(E->getNumArgs() - 2)).getPointer(*this);
+      Value *Count2Addr =
+          EmitLValue(E->getArg(E->getNumArgs() - 2)).getPointer(*this);
+      Builder.CreateDefaultAlignedStore(Count1, Count1Addr);
+      Builder.CreateDefaultAlignedStore(Count2, Count2Addr);
+    }
+
+    Builder.CreateDefaultAlignedStore(Fifo, FifoAddr);
+    Builder.CreateDefaultAlignedStore(Avail, AvailAddr);
+    return Builder.CreateDefaultAlignedStore(Ptr, PtrAddr);
+  }
   default:
     break;
   }
@@ -22486,7 +22650,19 @@ Value *CodeGenFunction::EmitAIE2PBuiltinExpr(unsigned BuiltinID,
   case AIE::BI__builtin_aie2p_divstep:
   case AIE::BI__builtin_aie2p_v64accfloat_to_v64bfp16ebs8:
   case AIE::BI__builtin_aie2p_v64accfloat_to_v64bfp16ebs16:
-  case AIE::BI__builtin_aie2p_v64bfp16ebs8_to_v64bfp16ebs16: {
+  case AIE::BI__builtin_aie2p_v64bfp16ebs8_to_v64bfp16ebs16:
+  case AIE::BI__builtin_aie2p_vshuffle_576_bfp16:
+  case AIE::BI__builtin_aie2p_fifo_st_push_576_bfp16:
+  case AIE::BI__builtin_aie2p_fifo_st_push_512_bfp16:
+  case AIE::BI__builtin_aie2p_fifo_st_push_544_bfp16:
+  case AIE::BI__builtin_aie2p_fifo_st_flush:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_1d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_1d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_2d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_2d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_3d_byte:
+  case AIE::BI__builtin_aie2p_fifo_st_flush_conv_3d_byte: {
     return this->EmitAIEBuiltinExpr(BuiltinID, E, Arch);
   }
   default: