feat: support custom solc for tests and benchmarks (#4)

.github/actions/build-llvm/action.yml

@@ -40,6 +40,10 @@ inputs:
     description: 'Github Actions cache key for CCache.'
     required: false
     default: ''
+  ccache-key-type:
+    description: 'Type of ccache key: `static` - rotate each month, or `dynamic` - rotate with each update of `LLVM.lock`.'
+    required: false
+    default: 'dynamic'
 runs:
   using: "composite"
   steps:
@@ -65,9 +69,13 @@ runs:
       shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }}
       id: ccache-key
       run: |
-        LLVM_BRANCH="$(git -C ./llvm rev-parse --abbrev-ref HEAD)"
-        LLVM_SHORT_SHA="$(git -C ./llvm rev-parse --short HEAD)"
-        echo "key=llvm-${LLVM_BRANCH}-${LLVM_SHORT_SHA}-${{ runner.os }}-${{ runner.arch }}" | tee -a "${GITHUB_OUTPUT}"
+        if [ ${{ inputs.ccache-key-type}} = static ]; then
+          echo "key=llvm-$(date +'%Y-%m')" | tee -a "${GITHUB_OUTPUT}"
+        else
+          LLVM_BRANCH="$(git -C ./llvm rev-parse --abbrev-ref HEAD)"
+          LLVM_SHORT_SHA="$(git -C ./llvm rev-parse --short HEAD)"
+          echo "key=llvm-${LLVM_BRANCH}-${LLVM_SHORT_SHA}-${{ runner.os }}-${{ runner.arch }}" | tee -a "${GITHUB_OUTPUT}"
+        fi
 
     # CCache action requires `apt update`
     # to be executed before it for Linux Docker jobs. See:

.github/workflows/benchmarks.yml

@@ -6,7 +6,7 @@ on:
       llvm_build_type:
         type: string
         description: "LLVM build type: debug | release"
-        required: true
+        required: false
         default: "release"
       compiler_tester_reference_branch:
         type: string
@@ -38,11 +38,26 @@ on:
         description: "Path filter for compiler-llvm benchmarks"
         required: false
         default: ""
-
-
-concurrency:
-  group: ${{ github.repository_id }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+      ccache-key-type:
+        type: string
+        required: false
+        default: 'dynamic'
+        description: 'Type of ccache key: `static` - rotate each month, or `dynamic` - rotate with each update of `LLVM.lock`.'
+      compiler-tester-repo:
+        type: string
+        required: false
+        default: 'matter-labs/era-compiler-tester'
+        description: 'Compiler tester repository to use. Required for forks testing.'
+      compiler-llvm-repo:
+        type: string
+        required: false
+        default: 'matter-labs/era-compiler-llvm'
+        description: 'Compiler LLVM repository to use. Required for forks testing.'
+      custom-solc-run-id:
+        type: string
+        required: false
+        default: ''
+        description: 'run id of custom zksync solc artifact to download.'
 
 jobs:
   benchmarks:
@@ -70,22 +85,27 @@ jobs:
       - name: Checkout compiler-tester
         uses: actions/checkout@v4
         with:
-          repository: matter-labs/era-compiler-tester
+          repository: ${{ inputs.compiler-tester-repo }}
           ref: ${{ steps.define-branches.outputs.compiler-tester-branch }}
           submodules: recursive
 
       - name: Checkout LLVM
         if: steps.define-branches.outputs.llvm-branch != ''
         uses: actions/checkout@v4
         with:
-          repository: matter-labs/era-compiler-llvm
+          repository: ${{ inputs.compiler-llvm-repo }}
           ref: ${{ steps.define-branches.outputs.llvm-branch }}
           path: llvm
 
+      # An issue prevents to correctly use the same version of composite actions from `workflow_call`
+      # https://github.com/actions/toolkit/issues/1264
+      # for now, it will always be taken from the latest main
       - name: Build LLVM
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@main
         with:
           clone-llvm: ${{ steps.define-branches.outputs.llvm-branch == '' && 'true' || 'false' }}
+          enable-assertions: true
+          ccache-key-type: ${{ inputs.ccache-key-type }}
 
       - name: Build compiler-tester
         run: cargo build --verbose --release --bin 'compiler-tester'
@@ -101,13 +121,28 @@ jobs:
             --manifest-path ${CARGO_CHECKOUT_DIR}/era-compiler-vyper-*/*/Cargo.toml \
             --target-dir './target-zkvyper/'
 
+      - name: Download custom solc
+        if: inputs.custom-solc-run-id != '' && matrix.type == 'candidate'
+        uses: actions/download-artifact@v4
+        with:
+          path: ./solc-bin/
+          run-id: ${{ inputs.custom-solc-run-id }}
+          merge-multiple: true
+
+      - name: Prepare custom solc
+        if: inputs.custom-solc-run-id != '' && matrix.type == 'candidate'
+        run: |
+          chmod a+x ./solc-bin/solc*
+          echo "Testing with custom solc from run ${{ inputs.custom-solc-run-id }}"
+          echo $(./solc-bin/solc* --version)
+
       - name: Run benchmarks
         run: |
           ./target/release/compiler-tester \
             --zksolc './target-zksolc/release/zksolc' \
             --zkvyper './target-zkvyper/release/zkvyper' \
             --path=${{ inputs.compiler_llvm_benchmark_path || '' }} \
-            --mode=${{ inputs.compiler_llvm_benchmark_mode || '^M^B3' }} \
+            --mode='${{ inputs.compiler_llvm_benchmark_mode || '^M^B3' }}' \
             --benchmark=${{ matrix.type }}.json
 
       - uses: actions/upload-artifact@v4
@@ -127,8 +162,8 @@ jobs:
       - name: Checking out the compiler-tester repository
         uses: actions/checkout@v4
         with:
-          repository: matter-labs/era-compiler-tester
-          ref: ${{ inputs.compiler_tester_candidate_branch || github.event.repository.default_branch }}
+          repository: ${{ inputs.compiler-tester-repo }}
+          ref: ${{ inputs.compiler_tester_candidate_branch || 'main' }}
           submodules: recursive
 
       - uses: actions/download-artifact@v4

.github/workflows/integration-tests.yaml

@@ -11,10 +11,36 @@ on:
         type: string
         required: true
         description: 'LLVM revision to use.'
-
-concurrency:
-  group: ${{ github.repository_id }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+      path:
+        type: string
+        required: false
+        default: ''
+        description: 'Path filter for integration tests execution. For example: `tests/solidity/`.'
+      extra-args:
+        type: string
+        required: false
+        default: ''
+        description: 'Extra arguments for era-compiler-tester.'
+      custom-solc-run-id:
+        type: string
+        required: false
+        default: ''
+        description: 'run id of custom zksync solc artifact to download.'
+      ccache-key-type:
+        type: string
+        required: false
+        default: 'dynamic'
+        description: 'Type of ccache key: `static` - rotate each month, or `dynamic` - rotate with each update of `LLVM.lock`.'
+      compiler-tester-repo:
+        type: string
+        required: false
+        default: 'matter-labs/era-compiler-tester'
+        description: 'Compiler tester repository to use. Required for forks testing.'
+      compiler-llvm-repo:
+        type: string
+        required: false
+        default: 'matter-labs/era-compiler-llvm'
+        description: 'Compiler LLVM repository to use. Required for forks testing.'
 
 jobs:
   run-integration-tests:
@@ -28,25 +54,29 @@ jobs:
       - name: Checkout compiler-tester
         uses: actions/checkout@v4
         with:
-          repository: matter-labs/era-compiler-tester
+          repository: ${{ inputs.compiler-tester-repo }}
           ref: ${{ inputs.compiler-tester-ref || '' }}
           submodules: recursive
 
       - name: Checkout LLVM
         if: inputs.llvm-ref != ''
         uses: actions/checkout@v4
         with:
-          repository: matter-labs/era-compiler-llvm
+          repository: ${{ inputs.compiler-llvm-repo }}
           ref: ${{ inputs.llvm-ref }}
           path: llvm
 
+      # An issue prevents to correctly use the same version of composite actions from `workflow_call`
+      # https://github.com/actions/toolkit/issues/1264
+      # for now, it will always be taken from the latest main
       - name: Build LLVM
         uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@main
         with:
           extra-args: "\\-DLLVM_ENABLE_WERROR=On \\-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
           enable-tests: true
           enable-assertions: true
           clone-llvm: ${{ inputs.llvm-ref == '' && 'true' || 'false' }}
+          ccache-key-type: ${{ inputs.ccache-key-type }}
 
       - name: Build compiler-tester
         run: cargo build --verbose --release --bin 'compiler-tester'
@@ -62,11 +92,28 @@ jobs:
             --manifest-path ${CARGO_CHECKOUT_DIR}/era-compiler-vyper-*/*/Cargo.toml \
             --target-dir './target-zkvyper/'
 
+      - name: Download custom solc
+        if: inputs.custom-solc-run-id != ''
+        uses: actions/download-artifact@v4
+        with:
+          path: ./solc-bin/
+          run-id: ${{ inputs.custom-solc-run-id }}
+          merge-multiple: true
+
+      - name: Prepare custom solc
+        if: inputs.custom-solc-run-id != ''
+        run: |
+          chmod a+x ./solc-bin/solc*
+          echo "Testing with custom solc from run ${{ inputs.custom-solc-run-id }}"
+          echo $(./solc-bin/solc* --version)
+
       - name: Run integration tests
         run: |
+          set -x
           ./target/release/compiler-tester \
             --zksolc './target-zksolc/release/zksolc' \
-            --zkvyper './target-zkvyper/release/zkvyper'
+            --zkvyper './target-zkvyper/release/zkvyper' \
+            --path '${{ inputs.path }}' ${{ inputs.extra-args }}
 
       - name: Send Slack notification
         uses: 8398a7/action-slack@v3