From ba26a9df1d7aeedcafe521abb30f9c4e492244c5 Mon Sep 17 00:00:00 2001 From: Diego <31937514+Ferret-san@users.noreply.github.com> Date: Tue, 19 Nov 2024 09:30:34 -0500 Subject: [PATCH] Add Github CI (#472) --- .circleci/config.yml | 1674 ----------------------------- .github/workflows/cannon.yml | 253 +++++ .github/workflows/ci.yml | 106 ++ .github/workflows/tag-service.yml | 69 -- .github/workflows/validate.yml | 28 + 5 files changed, 387 insertions(+), 1743 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/cannon.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/tag-service.yml create mode 100644 .github/workflows/validate.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 648217dbcdf8..000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1674 +0,0 @@ -version: 2.1 - -parameters: - ci_builder_image: - type: string - default: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:v0.54.0 - ci_builder_rust_image: - type: string - default: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder-rust:latest - base_image: - type: string - default: default - # The dispatch parameters are used to manually dispatch pipelines that normally only run post-merge on develop - # from the CircleCI UI. Example configuration: - # when: - # or: - # - equal: [ "develop", <> ] - # - equal: [ true, <> ] - # Add a new `*_dispatch` parameter for any pipeline you want manual dispatch for. - main_dispatch: - type: boolean - default: true # default to running main in case the manual run cancelled an automatic run - fault_proofs_dispatch: - type: boolean - default: false - reproducibility_dispatch: - type: boolean - default: false - kontrol_dispatch: - type: boolean - default: false - cannon_full_test_dispatch: - type: boolean - default: false - sdk_dispatch: - type: boolean - default: false - docker_publish_dispatch: - type: boolean - default: false - publish_contract_artifacts_dispatch: - type: boolean - default: false - -orbs: - go: circleci/go@1.8.0 - gcp-cli: circleci/gcp-cli@3.0.1 - slack: circleci/slack@4.10.1 - shellcheck: circleci/shellcheck@3.2.0 -commands: - gcp-oidc-authenticate: - description: "Authenticate with GCP using a CircleCI OIDC token." - parameters: - project_id: - type: env_var_name - default: GCP_PROJECT_ID - workload_identity_pool_id: - type: env_var_name - default: GCP_WIP_ID - workload_identity_pool_provider_id: - type: env_var_name - default: GCP_WIP_PROVIDER_ID - service_account_email: - type: env_var_name - default: GCP_SERVICE_ACCOUNT_EMAIL - gcp_cred_config_file_path: - type: string - default: /home/circleci/gcp_cred_config.json - oidc_token_file_path: - type: string - default: /home/circleci/oidc_token.json - steps: - - run: - name: "Create OIDC credential configuration" - command: | - # Store OIDC token in temp file - echo $CIRCLE_OIDC_TOKEN > << parameters.oidc_token_file_path >> - # Create a credential configuration for the generated OIDC ID Token - gcloud iam workload-identity-pools create-cred-config \ - "projects/${<< parameters.project_id >>}/locations/global/workloadIdentityPools/${<< parameters.workload_identity_pool_id >>}/providers/${<< parameters.workload_identity_pool_provider_id >>}"\ - --output-file="<< parameters.gcp_cred_config_file_path >>" \ - --service-account="${<< parameters.service_account_email >>}" \ - --credential-source-file=<< parameters.oidc_token_file_path >> - - run: - name: "Authenticate with GCP using OIDC" - command: | - # Configure gcloud to leverage the generated credential configuration - gcloud auth login --brief --cred-file "<< parameters.gcp_cred_config_file_path >>" - # Configure ADC - echo "export GOOGLE_APPLICATION_CREDENTIALS='<< parameters.gcp_cred_config_file_path >>'" | tee -a "$BASH_ENV" - - check-changed: - description: "Conditionally halts a step if certain modules change" - parameters: - patterns: - type: string - description: "Comma-separated list of dependencies" - no_go_deps: - type: string - default: "" - description: "If set, does not trigger on `go.mod` / `go.sum` changes." - steps: - - run: - name: "Check for changes" - environment: - CHECK_CHANGED_NO_GO_DEPS: "<>" - command: | - cd ops/check-changed - pip3 install -r requirements.txt - python3 main.py "<>" - - install-contracts-dependencies: - description: "Install the dependencies for the smart contracts" - steps: - - run: - command: just install - working_directory: packages/contracts-bedrock - - notify-failures-on-develop: - description: "Notify Slack" - parameters: - channel: - type: string - default: C03N11M0BBN - mentions: - type: string - default: "" - steps: - - slack/notify: - channel: << parameters.channel >> - event: fail - template: basic_fail_1 - branch_pattern: develop - mentions: "<< parameters.mentions >>" - - run-contracts-check: - parameters: - command: - description: Just command that runs the check - type: string - steps: - - run: - name: <> - command: | - git reset --hard - just <> - git diff --quiet --exit-code - working_directory: packages/contracts-bedrock - when: always - environment: - FOUNDRY_PROFILE: ci - -jobs: - cannon-go-lint-and-test: - machine: true - resource_class: ethereum-optimism/latitude-1 - parameters: - skip_slow_tests: - type: boolean - default: false - notify: - description: Whether to notify on failure - type: boolean - default: false - mips64: - type: boolean - default: false - steps: - - checkout - - check-changed: - patterns: cannon,packages/contracts-bedrock/src/cannon,op-preimage,go.mod - - attach_workspace: - at: "." - - run: - name: prep Cannon results dir - command: | - mkdir -p ./tmp/test-results - mkdir -p ./tmp/testlogs - - run: - name: build Cannon example binaries - command: make elf # only compile ELF binaries with Go, we do not have MIPS GCC for creating the debug-dumps. - working_directory: cannon/testdata/example - - run: - name: Cannon Go lint - command: | - make lint - working_directory: cannon - - when: - condition: - not: <> - steps: - - run: - name: Cannon Go 32-bit tests - command: | - export SKIP_SLOW_TESTS=<> - gotestsum --format=testname --junitfile=../tmp/test-results/cannon.xml --jsonfile=../tmp/testlogs/log.json \ - -- -parallel=$(nproc) -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./... - working_directory: cannon - - when: - condition: <> - steps: - - run: - name: Cannon Go 64-bit tests - command: | - export SKIP_SLOW_TESTS=<> - gotestsum --format=testname --junitfile=../tmp/test-results/cannon.xml --jsonfile=../tmp/testlogs/log.json \ - -- --tags=cannon64 -parallel=$(nproc) -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./... - working_directory: cannon - - run: - name: upload Cannon coverage - command: codecov --verbose --clean --flags cannon-go-tests - - store_test_results: - path: ./tmp/test-results - - store_artifacts: - path: ./tmp/testlogs - when: always - - when: - condition: <> - steps: - - notify-failures-on-develop: - mentions: "@proofs-team" - - cannon-build-test-vectors: - docker: - - image: <> - resource_class: medium - steps: - - checkout - - check-changed: - patterns: cannon/mipsevm/tests/open_mips_tests/test - - run: - name: Build MIPS test vectors - command: python3 maketests.py && git diff --exit-code - working_directory: cannon/mipsevm/tests/open_mips_tests - - contracts-bedrock-build: - machine: true - resource_class: ethereum-optimism/latitude-1 - parameters: - skip_pattern: - description: Glob pattern of tests to skip - type: string - default: "" - steps: - - checkout - - install-contracts-dependencies - - run: - name: Print forge version - command: forge --version - - run: - name: Pull artifacts - command: bash scripts/ops/pull-artifacts.sh - working_directory: packages/contracts-bedrock - - run: - name: Build contracts - command: forge build --deny-warnings --skip <> - environment: - FOUNDRY_PROFILE: ci - working_directory: packages/contracts-bedrock - - run: - name: Generate allocs - command: | - make devnet-allocs - - persist_to_workspace: - root: "." - paths: - - "packages/contracts-bedrock/cache" - - "packages/contracts-bedrock/artifacts" - - "packages/contracts-bedrock/forge-artifacts" - - "packages/contracts-bedrock/deploy-config/devnetL1.json" - - "packages/contracts-bedrock/deployments/devnetL1" - - ".devnet" - - ".devnet-standard" - - ".devnet-l2oo" - - ".devnet-alt-da" - - ".devnet-alt-da-generic" - - ".devnet-mt-cannon" - - notify-failures-on-develop - - check-kontrol-build: - docker: - - image: <> - resource_class: xlarge - steps: - - checkout - - attach_workspace: { at: "." } - - install-contracts-dependencies - - check-changed: - patterns: contracts-bedrock - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Run Kontrol build - command: just kontrol-summary-full - working_directory: packages/contracts-bedrock - - run: - name: Build Kontrol summary files - command: forge build ./test/kontrol/proofs - working_directory: packages/contracts-bedrock - - notify-failures-on-develop - - docker-build: - environment: - DOCKER_BUILDKIT: 1 - parameters: - docker_tags: - description: Docker image tags, comma-separated - type: string - docker_name: - description: "Docker buildx bake target" - type: string - default: "" - registry: - description: Docker registry - type: string - default: "us-docker.pkg.dev" - repo: - description: Docker repo - type: string - default: "oplabs-tools-artifacts/images" - save_image_tag: - description: Save docker image with given tag - type: string - default: "" - platforms: - description: Platforms to build for, comma-separated - type: string - default: "linux/amd64" - publish: - description: Publish the docker image (multi-platform, all tags) - type: boolean - default: false - release: - description: Run the release script - type: boolean - default: false - resource_class: - description: Docker resoruce class - type: string - default: medium - machine: - image: <> - resource_class: "<>" - docker_layer_caching: true # we rely on this for faster builds, and actively warm it up for builds with common stages - steps: - - checkout - - attach_workspace: - at: /tmp/docker_images - - run: - command: mkdir -p /tmp/docker_images - - when: - condition: - or: - - "<>" - - "<>" - steps: - - gcp-cli/install - - when: - condition: - or: - - "<>" - - "<>" - steps: - - gcp-oidc-authenticate - - run: - name: Build - command: | - # Check to see if DOCKER_HUB_READ_ONLY_TOKEN is set (i.e. we are in repo) before attempting to use secrets. - # Building should work without this read only login, but may get rate limited. - if [[ -v DOCKER_HUB_READ_ONLY_TOKEN ]]; then - echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin - fi - - export REGISTRY="<>" - export REPOSITORY="<>" - export IMAGE_TAGS="$(echo -ne "<>" | sed "s/[^a-zA-Z0-9\n,]/-/g")" - export GIT_COMMIT="$(git rev-parse HEAD)" - export GIT_DATE="$(git show -s --format='%ct')" - export PLATFORMS="<>" - - echo "Checking git tags pointing at $GIT_COMMIT:" - tags_at_commit=$(git tag --points-at $GIT_COMMIT) - echo "Tags at commit:\n$tags_at_commit" - - filtered_tags=$(echo "$tags_at_commit" | grep "^<>/" || true) - echo "Filtered tags: $filtered_tags" - - if [ -z "$filtered_tags" ]; then - export GIT_VERSION="untagged" - else - sorted_tags=$(echo "$filtered_tags" | sed "s/<>\///" | sort -V) - echo "Sorted tags: $sorted_tags" - - # prefer full release tag over "-rc" release candidate tag if both exist - full_release_tag=$(echo "$sorted_tags" | grep -v -- "-rc" || true) - if [ -z "$full_release_tag" ]; then - export GIT_VERSION=$(echo "$sorted_tags" | tail -n 1) - else - export GIT_VERSION=$(echo "$full_release_tag" | tail -n 1) - fi - fi - - echo "Setting GIT_VERSION=$GIT_VERSION" - - # Create, start (bootstrap) and use a *named* docker builder - # This allows us to cross-build multi-platform, - # and naming allows us to use the DLC (docker-layer-cache) - docker buildx create --driver=docker-container --name=buildx-build --bootstrap --use - - DOCKER_OUTPUT_DESTINATION="" - if [ "<>" == "true" ]; then - gcloud auth configure-docker <> - echo "Building for platforms $PLATFORMS and then publishing to registry" - DOCKER_OUTPUT_DESTINATION="--push" - if [ "<>" != "" ]; then - echo "ERROR: cannot save image to docker when publishing to registry" - exit 1 - fi - else - if [ "<>" == "" ]; then - echo "Running $PLATFORMS build without destination (cache warm-up)" - DOCKER_OUTPUT_DESTINATION="" - elif [[ $PLATFORMS == *,* ]]; then - echo "ERROR: cannot perform multi-arch (platforms: $PLATFORMS) build while also loading the result into regular docker" - exit 1 - else - echo "Running single-platform $PLATFORMS build and loading into docker" - DOCKER_OUTPUT_DESTINATION="--load" - fi - fi - - # Let them cook! - docker buildx bake \ - --progress plain \ - --builder=buildx-build \ - -f docker-bake.hcl \ - $DOCKER_OUTPUT_DESTINATION \ - <> - - no_output_timeout: 45m - - when: - condition: "<>" - steps: - - notify-failures-on-develop - - when: - condition: "<>" - steps: - - run: - name: Save - command: | - IMAGE_NAME="<>/<>/<>:<>" - docker save -o /tmp/docker_images/<>.tar $IMAGE_NAME - - persist_to_workspace: - root: /tmp/docker_images - paths: # only write the one file, to avoid concurrent workspace-file additions - - "<>.tar" - - when: - condition: "<>" - steps: - - run: - name: Tag - command: | - ./ops/scripts/ci-docker-tag-op-stack-release.sh <>/<> $CIRCLE_TAG $CIRCLE_SHA1 - - when: - condition: - or: - - and: - - "<>" - - "<>" - - and: - - "<>" - - equal: [develop, << pipeline.git.branch >>] - steps: - - gcp-oidc-authenticate: - service_account_email: GCP_SERVICE_ATTESTOR_ACCOUNT_EMAIL - - run: - name: Sign - command: | - VER=$(jq -r .binary_signer < versions.json) - wget -O - "https://github.com/ethereum-optimism/binary_signer/archive/refs/tags/v${VER}.tar.gz" | tar xz - cd "binary_signer-${VER}/signer" - - IMAGE_PATH="<>/<>/<>:<>" - echo $IMAGE_PATH - pip3 install -r requirements.txt - - python3 ./sign_image.py --command="sign"\ - --attestor-project-name="$ATTESTOR_PROJECT_NAME"\ - --attestor-name="$ATTESTOR_NAME"\ - --image-path="$IMAGE_PATH"\ - --signer-logging-level="INFO"\ - --attestor-key-id="//cloudkms.googleapis.com/v1/projects/$ATTESTOR_PROJECT_NAME/locations/global/keyRings/$ATTESTOR_NAME-key-ring/cryptoKeys/$ATTESTOR_NAME-key/cryptoKeyVersions/1" - - # Verify newly published images (built on AMD machine) will run on ARM - check-cross-platform: - docker: - - image: cimg/base:current - resource_class: arm.medium - parameters: - registry: - description: Docker registry - type: string - default: "us-docker.pkg.dev" - repo: - description: Docker repo - type: string - default: "oplabs-tools-artifacts/images" - op_component: - description: "Name of op-stack component (e.g. op-node)" - type: string - default: "" - docker_tag: - description: "Tag of docker image" - type: string - default: "<>" - steps: - - setup_remote_docker - - run: - name: "Verify Image Platform" - command: | - image_name="<>/<>/<>:<>" - echo "Retrieving Docker image manifest: $image_name" - MANIFEST=$(docker manifest inspect $image_name) - - echo "Verifying 'linux/arm64' is supported..." - SUPPORTED_PLATFORM=$(echo "$MANIFEST" | jq -r '.manifests[] | select(.platform.architecture == "arm64" and .platform.os == "linux")') - echo $SUPPORT_PLATFORM - if [ -z "$SUPPORTED_PLATFORM" ]; then - echo "Platform 'linux/arm64' not supported by this image" - exit 1 - fi - - run: - name: "Pull and run docker image" - command: | - image_name="<>/<>/<>:<>" - docker pull $image_name || exit 1 - docker run $image_name <> --version || exit 1 - - contracts-bedrock-coverage: - docker: - - image: <> - resource_class: xlarge - steps: - - checkout - - install-contracts-dependencies - - check-changed: - patterns: contracts-bedrock,op-node - - run: - name: print forge version - command: forge --version - working_directory: packages/contracts-bedrock - # We do not use the pre-built contracts becuase forge coverage uses different optimizer settings - - run: - name: test and generate coverage - command: just coverage-lcov - no_output_timeout: 18m - environment: - FOUNDRY_PROFILE: ci - working_directory: packages/contracts-bedrock - - run: - name: upload coverage - command: codecov --verbose --clean --flags contracts-bedrock-tests - environment: - FOUNDRY_PROFILE: ci - - contracts-bedrock-tests: - docker: - - image: <> - resource_class: xlarge - parameters: - test_parallelism: - description: Number of test jobs to run in parallel - type: integer - default: 4 - test_list: - description: List of test files to run - type: string - test_fuzz_runs: - description: Number of fuzz runs to apply - type: integer - default: 512 - test_invariant_runs: - description: Number of invariant runs to apply - type: integer - default: 32 - test_invariant_depth: - description: Depth of invariant runs - type: integer - default: 64 - test_timeout: - description: Timeout for running tests - type: string - default: 15m - test_profile: - description: Profile to use for testing - type: string - default: ci - parallelism: <> - steps: - - checkout - - attach_workspace: { at: "." } - - install-contracts-dependencies - - run: - name: Check if test list is empty - command: | - TEST_FILES=$(<>) - if [ -z "$TEST_FILES" ]; then - echo "No test files to run. Exiting early." - circleci-agent step halt - fi - working_directory: packages/contracts-bedrock - - check-changed: - patterns: contracts-bedrock,op-node - - restore_cache: - name: Restore Go modules cache - key: gomod-{{ checksum "go.sum" }} - - restore_cache: - name: Restore Go build cache - keys: - - golang-build-cache-contracts-bedrock-tests-{{ checksum "go.sum" }} - - golang-build-cache-contracts-bedrock-tests- - - run: - name: Print dependencies - command: just dep-status - working_directory: packages/contracts-bedrock - - run: - name: Print forge version - command: forge --version - working_directory: packages/contracts-bedrock - - run: - name: Pull artifacts - command: bash scripts/ops/pull-artifacts.sh - working_directory: packages/contracts-bedrock - - run: - name: Build go-ffi - command: just build-go-ffi - working_directory: packages/contracts-bedrock - - run: - name: Run tests - command: | - TEST_FILES=$(<>) - TEST_FILES=$(echo "$TEST_FILES" | circleci tests split --split-by=timings) - TEST_FILES=$(echo "$TEST_FILES" | sed 's|^test/||') - MATCH_PATH="./test/{$(echo "$TEST_FILES" | paste -sd "," -)}" - forge test --match-path "$MATCH_PATH" - environment: - FOUNDRY_PROFILE: <> - working_directory: packages/contracts-bedrock - no_output_timeout: <> - - run: - name: Print failed test traces - command: just test-rerun - environment: - FOUNDRY_PROFILE: ci - working_directory: packages/contracts-bedrock - when: on_fail - - save_cache: - name: Save Go build cache - key: golang-build-cache-contracts-bedrock-tests-{{ checksum "go.sum" }} - paths: - - "/root/.cache/go-build" - - notify-failures-on-develop - - contracts-bedrock-checks: - docker: - - image: <> - resource_class: xlarge - steps: - - checkout - - attach_workspace: { at: "." } - - install-contracts-dependencies - - check-changed: - patterns: contracts-bedrock,op-node - - setup_remote_docker: - docker_layer_caching: true - - run: - name: print forge version - command: forge --version - - run-contracts-check: - command: semgrep-test-validity-check - - run-contracts-check: - command: semgrep - - run-contracts-check: - command: semver-lock - - run-contracts-check: - command: semver-diff-check-no-build - - run-contracts-check: - command: semver-natspec-check-no-build - - run-contracts-check: - command: validate-deploy-configs - - run-contracts-check: - command: lint - - run-contracts-check: - command: gas-snapshot-check - - run-contracts-check: - command: snapshots-check-no-build - - run-contracts-check: - command: interfaces-check-no-build - - run-contracts-check: - command: size-check - - run-contracts-check: - command: unused-imports-check-no-build - - contracts-bedrock-validate-spacers: - docker: - - image: <> - resource_class: medium - steps: - - checkout - - attach_workspace: { at: "." } - - install-contracts-dependencies - - check-changed: - patterns: contracts-bedrock - - run: - name: validate spacers - command: just validate-spacers-no-build - working_directory: packages/contracts-bedrock - - todo-issues: - parameters: - check_closed: - type: boolean - default: true - machine: - image: <> - steps: - - checkout - - run: - name: Install ripgrep - command: sudo apt-get install -y ripgrep - - run: - name: Check TODO issues - command: ./ops/scripts/todo-checker.sh --verbose <<#parameters.check_closed>> --check-closed <> - - notify-failures-on-develop - - fuzz-golang: - parameters: - package_name: - description: Go package name - type: string - on_changes: - description: changed pattern to fire fuzzer on - type: string - uses_artifacts: - description: should load in foundry artifacts - type: boolean - default: false - docker: - - image: <> - steps: - - checkout - - check-changed: - patterns: "<>" - - attach_workspace: - at: "." - if: ${{ uses_artifacts }} - - restore_cache: - name: Restore Go modules cache - key: gomod-{{ checksum "go.sum" }} - - restore_cache: - name: Restore Go build cache - keys: - - golang-build-cache-fuzz-golang-{{ checksum "go.sum" }} - - golang-build-cache-fuzz-golang- - - run: - name: Fuzz - command: make fuzz - working_directory: "<>" - - save_cache: - name: Save Go build cache - key: golang-build-cache-fuzz-golang-{{ checksum "go.sum" }} - paths: - - "/root/.cache/go-build" - - go-lint: - machine: true - resource_class: ethereum-optimism/latitude-1 - steps: - - checkout - - run: - name: run Go linter - command: | - make lint-go - working_directory: . - - go-test-kurtosis: - parameters: - module: - description: Go Module Name - type: string - uses_artifacts: - description: Uses contract artifacts - type: boolean - default: false - test_directory: - description: Test directory - type: string - default: "./..." - machine: - image: <> - resource_class: xlarge - steps: - - run: - name: Install components - command: | - go version - go install gotest.tools/gotestsum@v1.11.0 - - run: - name: Install Kurtosis - command: | - echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list - sudo apt update - sudo apt install kurtosis-cli=1.3.0 - kurtosis engine start - - checkout - - when: - condition: <> - steps: - - attach_workspace: { at: "." } - - run: - name: prep results dir - command: | - # Make sure the workspace is properly owned - mkdir -p ./tmp/test-results - mkdir -p ./tmp/testlogs - - run: - name: run tests - command: | - ENABLE_KURTOSIS=true gotestsum \ - --format=testname \ - --junitfile=../tmp/test-results/<>.xml \ - --jsonfile=../tmp/testlogs/log.json \ - -- -parallel=$(nproc) \ - -coverpkg=github.com/ethereum-optimism/optimism/... \ - -coverprofile=coverage.out <> - working_directory: <> - - store_test_results: - path: tmp/test-results - - store_artifacts: - path: tmp/testlogs - when: always - - go-test: - machine: true - resource_class: ethereum-optimism/latitude-1 - steps: - - checkout - - attach_workspace: - at: "." - - run: - name: run tests - command: | - mkdir -p ./tmp/test-results && mkdir -p ./tmp/testlogs - - packages=( - op-batcher - op-chain-ops - op-node - op-proposer - op-challenger - op-dispute-mon - op-conductor - op-program - op-service - op-supervisor - op-deployer - packages/contracts-bedrock/scripts/checks/semver-natspec - ) - formatted_packages="" - for package in "${packages[@]}"; do - formatted_packages="$formatted_packages ./$package/..." - done - - export ENABLE_ANVIL=true - export SEPOLIA_RPC_URL="https://ci-sepolia-l1-archive.optimism.io" - - gotestsum --format=testname \ - --junitfile=./tmp/test-results/results.xml \ - --jsonfile=./tmp/testlogs/log.json \ - -- -coverpkg=github.com/ethereum-optimism/optimism/... \ - -coverprofile=coverage.out $formatted_packages - - store_test_results: - path: ./tmp/test-results - - store_artifacts: - path: ./tmp/testlogs - when: always - - go-e2e-test: - parameters: - module: - description: Go Module Name - type: string - target: - description: The make target to execute - type: string - notify: - description: Whether to notify on failure - type: boolean - default: false - mentions: - description: Slack user or group to mention when notifying of failures - type: string - default: "" - resource_class: - description: Machine resource class - type: string - default: ethereum-optimism/latitude-1 - machine: true - resource_class: <> - steps: - - checkout - - attach_workspace: - at: ./tmp/workspace - - run: - name: Load devnet-allocs and artifacts - command: | - mkdir -p .devnet - cp -r ./tmp/workspace/.devnet* . - cp -r ./tmp/workspace/packages/contracts-bedrock/forge-artifacts packages/contracts-bedrock/forge-artifacts - cp ./tmp/workspace/packages/contracts-bedrock/deploy-config/devnetL1.json packages/contracts-bedrock/deploy-config/devnetL1.json - cp -r ./tmp/workspace/packages/contracts-bedrock/deployments/devnetL1 packages/contracts-bedrock/deployments/devnetL1 - - run: - name: print go's available MIPS targets - command: go tool dist list | grep mips - - run: - name: run tests - no_output_timeout: 20m - command: | - mkdir -p ./tmp/testlogs - mkdir -p ./tmp/test-results - - # The below env var gets overridden when running make test-cannon, but we - # need to explicitly set it here to prevent Cannon from running when we don't - # want it to. - export OP_E2E_CANNON_ENABLED="false" - # Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional - # constraint that gotestsum does not currently (nor likely will) accept files from different packages when building. - JUNIT_FILE=../tmp/test-results/<>_<>.xml JSON_LOG_FILE=../tmp/testlogs/test.log make <> - working_directory: <> - - store_artifacts: - path: ./tmp/testlogs - when: always - - store_artifacts: - path: ./tmp/test-results - when: always - - store_test_results: - path: ./tmp/test-results - - when: - condition: "<>" - steps: - - notify-failures-on-develop: - mentions: "<>" - - cannon-prestate: - machine: true - resource_class: ethereum-optimism/latitude-1 - steps: - - checkout - - run: - name: Build cannon - command: make cannon - - run: - name: Build op-program - command: make op-program - - restore_cache: - name: Restore cannon prestate cache - key: cannon-prestate-{{ checksum "./cannon/bin/cannon" }}-{{ checksum "op-program/bin/op-program-client.elf" }} - - run: - name: Sanitize op-program guest - command: make -f cannon/Makefile sanitize-program GUEST_PROGRAM=op-program/bin/op-program-client.elf - - run: - name: generate cannon prestate - command: make cannon-prestate - - save_cache: - key: cannon-prestate-{{ checksum "./cannon/bin/cannon" }}-{{ checksum "op-program/bin/op-program-client.elf" }} - name: Save Cannon prestate to cache - paths: - - "op-program/bin/prestate.bin.gz" - - "op-program/bin/meta.json" - - "op-program/bin/prestate-proof.json" - - run: - name: generate cannon-mt prestate - command: make cannon-prestate-mt - - save_cache: - key: cannon-prestate-mt-{{ checksum "./cannon/bin/cannon" }}-{{ checksum "op-program/bin/op-program-client.elf" }} - name: Save MT-Cannon prestate to cache - paths: - - "op-program/bin/prestate-mt.json" - - "op-program/bin/meta-mt.json" - - "op-program/bin/prestate-proof-mt.json" - - persist_to_workspace: - root: . - paths: - - "op-program/bin/prestate.bin.gz" - - "op-program/bin/meta.json" - - "op-program/bin/prestate-proof.json" - - preimage-reproducibility: - docker: - - image: <> - steps: - - checkout - - setup_remote_docker - - run: - make -C op-program verify-reproducibility - - notify-failures-on-develop: - mentions: "@proofs-team" - - cannon-stf-verify: - docker: - - image: <> - steps: - - checkout - - setup_remote_docker - - restore_cache: - name: Restore Go modules cache - key: gomod-{{ checksum "go.sum" }} - - restore_cache: - name: Restore Go build cache - keys: - - golang-build-cache-cannon-stf-verify-{{ checksum "go.sum" }} - - golang-build-cache-cannon-stf-verify- - - run: - name: Build cannon - command: make cannon - - run: - name: Verify the Cannon STF - command: make -C ./cannon cannon-stf-verify - - save_cache: - name: Save Go build cache - key: golang-build-cache-cannon-stf-verify-{{ checksum "go.sum" }} - paths: - - "/root/.cache/go-build" - - notify-failures-on-develop: - mentions: "@proofs-squad" - - semgrep-scan: - parameters: - diff_branch: - type: string - default: develop - scan_command: - type: string - default: semgrep ci --timeout=100 - environment: - TEMPORARY_BASELINE_REF: << parameters.diff_branch >> - SEMGREP_REPO_URL: << pipeline.project.git_url >> - SEMGREP_BRANCH: << pipeline.git.branch >> - SEMGREP_COMMIT: << pipeline.git.revision >> - docker: - - image: returntocorp/semgrep - resource_class: xlarge - steps: - - checkout - - unless: - condition: - equal: [ "develop", << pipeline.git.branch >> ] - steps: - - run: - # Scan changed files in PRs, block on new issues only (existing issues ignored) - # Do a full scan when scanning develop, otherwise do an incremental scan. - name: "Conditionally set BASELINE env var" - command: | - echo 'export SEMGREP_BASELINE_REF=${TEMPORARY_BASELINE_REF}' >> $BASH_ENV - - run: - name: "Set environment variables" # for PR comments and in-app hyperlinks to findings - command: | - echo 'export SEMGREP_PR_ID=${CIRCLE_PULL_REQUEST##*/}' >> $BASH_ENV - echo 'export SEMGREP_JOB_URL=$CIRCLE_BUILD_URL' >> $BASH_ENV - echo 'export SEMGREP_REPO_NAME=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME' >> $BASH_ENV - - run: - name: "Semgrep scan" - # --timeout (in seconds) limits the time per rule and file. - # SEMGREP_TIMEOUT is the same, but docs have conflicting defaults (5s in CLI flag, 1800 in some places) - # https://semgrep.dev/docs/troubleshooting/semgrep-app#if-the-job-is-aborted-due-to-taking-too-long - command: << parameters.scan_command >> - # If semgrep hangs, stop the scan after 20m, to prevent a useless 5h job - no_output_timeout: 20m - - notify-failures-on-develop - - go-mod-download: - docker: - - image: <> - parameters: - file: - default: go.sum - description: The file name of checksum for restore_cache and save_cache. - type: string - key: - default: gomod - description: The key of restore_cache and save_cache. - type: string - steps: - - checkout - - restore_cache: - key: << parameters.key >>-{{ checksum "<< parameters.file >>" }} - name: Restore Go modules cache - - run: - name: Sanity check go mod cache path - command: test "$(go env GOMODCACHE)" == "/go/pkg/mod" # yes, it's an odd path - - run: - command: go mod download - name: Download Go module dependencies - - run: - name: "Go mod tidy" - command: make mod-tidy && git diff --exit-code - - save_cache: - key: << parameters.key >>-{{ checksum "<< parameters.file >>" }} - name: Save Go modules cache - paths: - - "/go/pkg/mod" - - bedrock-go-tests: # just a helper, that depends on all the actual test jobs - docker: - # Use a smaller base image to avoid pulling the huge ci-builder - # image which is not needed for this job and sometimes misses - # the cache. - - image: cimg/base:2024.01 - resource_class: medium - steps: - - run: echo Done - - fpp-verify: - docker: - - image: cimg/go:1.21 - steps: - - checkout - - run: - name: verify-sepolia - command: | - make verify-sepolia - working_directory: op-program - - notify-failures-on-develop: - mentions: "@proofs-team" - - op-program-compat: - docker: - - image: <> - steps: - - checkout - - restore_cache: - name: Restore Go modules cache - key: gomod-{{ checksum "go.sum" }} - - restore_cache: - key: golang-build-cache-op-program-compat-{{ checksum "go.sum" }} - - run: - name: compat-sepolia - command: | - make verify-compat - working_directory: op-program - - save_cache: - name: Save Go build cache - key: golang-build-cache-op-program-compat-{{ checksum "go.sum" }} - paths: - - "/root/.cache/go-build" - - check-generated-mocks-op-node: - docker: - - image: <> - steps: - - checkout - - check-changed: - patterns: op-node - - run: - name: check-generated-mocks - command: make generate-mocks-op-node && git diff --exit-code - - check-generated-mocks-op-service: - docker: - - image: <> - steps: - - checkout - - check-changed: - patterns: op-service - - run: - name: check-generated-mocks - command: make generate-mocks-op-service && git diff --exit-code - - kontrol-tests: - docker: - - image: << pipeline.parameters.ci_builder_image >> - resource_class: xlarge - steps: - - checkout - - install-contracts-dependencies - - check-changed: - no_go_deps: "true" - patterns: contracts-bedrock/test/kontrol,contracts-bedrock/src/L1/OptimismPortal\.sol,contracts-bedrock/src/L1/OptimismPortal2\.sol,contracts-bedrock/src/L1/L1CrossDomainMessenger\.sol,contracts-bedrock/src/L1/L1ERC721Bridge\.sol,contracts-bedrock/src/L1/L1StandardBridge\.sol,contracts-bedrock/src/L1/ResourceMetering\.sol,contracts-bedrock/src/universal/StandardBridge\.sol,contracts-bedrock/src/universal/ERC721Bridge\.sol,contracts-bedrock/src/universal/CrossDomainMessenger\.sol - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Run Kontrol Tests - command: | - curl -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $RV_COMPUTE_TOKEN" \ - https://api.github.com/repos/runtimeverification/optimism-ci/actions/workflows/optimism-ci.yaml/dispatches \ - -d '{ - "ref": "master", - "inputs": { - "branch_name": "<>", - "extra_args": "script", - "statuses_sha": "<< pipeline.git.revision >>", - "org": "ethereum-optimism", - "repository": "optimism" - } - }' - working_directory: ./packages/contracts-bedrock - - notify-failures-on-develop - - publish-contract-artifacts: - docker: - - image: <> - resource_class: xlarge - steps: - - gcp-cli/install - - gcp-oidc-authenticate: - gcp_cred_config_file_path: /root/gcp_cred_config.json - oidc_token_file_path: /root/oidc_token.json - project_id: GCP_TOOLS_ARTIFACTS_PROJECT_ID - service_account_email: GCP_CONTRACTS_PUBLISHER_SERVICE_ACCOUNT_EMAIL - - checkout - - install-contracts-dependencies - - run: - name: Pull artifacts - command: bash scripts/ops/pull-artifacts.sh - working_directory: packages/contracts-bedrock - - run: - name: Build contracts - environment: - FOUNDRY_PROFILE: ci - command: just forge-build - working_directory: packages/contracts-bedrock - - run: - name: Publish artifacts - command: bash scripts/ops/publish-artifacts.sh - working_directory: packages/contracts-bedrock - - go-release: - parameters: - module: - description: Go Module Name - type: string - filename: - description: Goreleaser config file - default: .goreleaser.yaml - type: string - machine: true - resource_class: ethereum-optimism/latitude-1 - steps: - - checkout - - run: - name: Run goreleaser - command: | - goreleaser release --clean -f ./<>/<> - -workflows: - main: - when: - and: - - or: - # Trigger on new commits - - equal: [ webhook, << pipeline.trigger_source >> ] - # Trigger on manual triggers if explicitly requested - - equal: [ true, << pipeline.parameters.main_dispatch >> ] - - not: - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] - jobs: - - go-mod-download - - contracts-bedrock-build: - # Build with just core + script contracts. - skip_pattern: test - - check-kontrol-build: - requires: - - contracts-bedrock-build - - contracts-bedrock-tests: - # Test everything except PreimageOracle.t.sol since it's slow. - name: contracts-bedrock-tests - test_parallelism: 4 - test_list: find test -name "*.t.sol" -not -name "PreimageOracle.t.sol" - - contracts-bedrock-tests: - # PreimageOracle test is slow, run it separately to unblock CI. - name: contracts-bedrock-tests-preimage-oracle - test_parallelism: 1 - test_list: find test -name "PreimageOracle.t.sol" - - contracts-bedrock-tests: - # Heavily fuzz any fuzz tests within added or modified test files. - name: contracts-bedrock-tests-heavy-fuzz-modified - test_parallelism: 1 - test_list: git diff origin/develop...HEAD --name-only -- './test/**/*.t.sol' | sed 's|packages/contracts-bedrock/||' - test_timeout: 1h - test_profile: ciheavy - - contracts-bedrock-coverage - - contracts-bedrock-checks: - requires: - - contracts-bedrock-build - - contracts-bedrock-validate-spacers: - requires: - - contracts-bedrock-build - - semgrep-scan - - semgrep-scan: - name: semgrep-scan-local - scan_command: semgrep scan --timeout=100 --config=./semgrep --error . - - semgrep-scan: - name: semgrep-test - scan_command: semgrep scan --test semgrep/ - - go-lint - - fuzz-golang: - name: fuzz-golang-<> - requires: - - go-mod-download - on_changes: <> - matrix: - parameters: - package_name: - - op-challenger - - op-node - - op-service - - op-chain-ops - - fuzz-golang: - name: cannon-fuzz - package_name: cannon - on_changes: cannon,packages/contracts-bedrock/src/cannon - uses_artifacts: true - requires: ["go-mod-download", "contracts-bedrock-build"] - - fuzz-golang: - name: op-e2e-fuzz - package_name: op-e2e - on_changes: op-e2e,packages/contracts-bedrock/src - uses_artifacts: true - requires: ["go-mod-download", "contracts-bedrock-build"] - - go-test: - name: go-test-all - requires: - - contracts-bedrock-build - - go-test-kurtosis: - name: op-deployer-integration - module: op-deployer - test_directory: ./pkg/deployer/integration_test - uses_artifacts: true - requires: ["contracts-bedrock-build"] - - go-e2e-test: - name: op-e2e-HTTP-tests - module: op-e2e - target: test-http - requires: - - contracts-bedrock-build - - go-e2e-test: - name: op-e2e-action-tests - module: op-e2e - target: test-actions - requires: - - contracts-bedrock-build - - go-e2e-test: - name: op-e2e-fault-proof-tests - module: op-e2e - target: test-fault-proofs - requires: - - contracts-bedrock-build - - cannon-prestate - - op-program-compat - - bedrock-go-tests: - requires: - - go-mod-download - - go-lint - - cannon-build-test-vectors - - cannon-go-lint-and-test - - check-generated-mocks-op-node - - check-generated-mocks-op-service - - go-mod-download - - op-deployer-integration - - op-program-compat - - op-e2e-HTTP-tests - - op-e2e-fault-proof-tests - - op-e2e-action-tests - # Not needed for the devnet but we want to make sure they build successfully - - cannon-docker-build - - op-dispute-mon-docker-build - - op-program-docker-build - - op-supervisor-docker-build - - proofs-tools-docker-build - - go-test-all - - docker-build: - name: <>-docker-build - docker_tags: <>,<> - save_image_tag: <> - matrix: - parameters: - docker_name: - - op-node - - op-batcher - - op-program - - op-proposer - - op-challenger - - proofs-tools - - op-dispute-mon - - op-conductor - - da-server - - op-supervisor - - cannon - - cannon-prestate - - check-generated-mocks-op-node - - check-generated-mocks-op-service - - cannon-go-lint-and-test: - requires: - - contracts-bedrock-build - skip_slow_tests: true - notify: true - - cannon-build-test-vectors - - todo-issues: - name: todo-issues-check - check_closed: false - - shellcheck/check: - name: shell-check - # We don't need the `exclude` key as the orb detects the `.shellcheckrc` - dir: . - ignore-dirs: - ./packages/contracts-bedrock/lib - - go-release-deployer: - jobs: - - go-release: - filters: - tags: - only: /^op-deployer.*/ - branches: - ignore: /.*/ - module: op-deployer - context: - - oplabs-gcr-release - - release: - when: - not: - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] - jobs: - # Wait for approval on the release - - hold: - type: approval - filters: - tags: - only: /^(da-server|ci-builder(-rust)?|proofs-tools|cannon|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/ - branches: - ignore: /.*/ - # Standard (medium) cross-platform docker images go here - - docker-build: - matrix: - parameters: - docker_name: - - op-node - - op-batcher - - op-proposer - - op-challenger - - op-dispute-mon - - op-conductor - - da-server - - op-ufm - - op-supervisor - - op-deployer - - cannon - name: <>-docker-release - docker_tags: <> - platforms: "linux/amd64,linux/arm64" - publish: true - release: true - filters: - tags: - only: /^<>\/v.*/ - branches: - ignore: /.*/ - context: - - oplabs-gcr-release - requires: - - hold - # Checks for cross-platform images go here - - check-cross-platform: - matrix: - parameters: - op_component: - - op-node - - op-batcher - - op-proposer - - op-challenger - - op-dispute-mon - - op-conductor - - da-server - - op-ufm - - op-supervisor - - op-deployer - - cannon - name: <>-cross-platform - requires: - - op-node-docker-release - - op-batcher-docker-release - - op-proposer-docker-release - - op-challenger-docker-release - - op-dispute-mon-docker-release - - op-conductor-docker-release - - da-server-docker-release - - op-ufm-docker-release - - op-supervisor-docker-release - - cannon-docker-release - # Standard (xlarge) AMD-only docker images go here - - docker-build: - matrix: - parameters: - docker_name: - - ci-builder - - ci-builder-rust - - proofs-tools - name: <>-docker-release - resource_class: xlarge - docker_tags: <> - publish: true - release: true - filters: - tags: - only: /^<>\/v.*/ - branches: - ignore: /.*/ - context: - - oplabs-gcr-release - requires: - - hold - - scheduled-todo-issues: - when: - equal: [ build_four_hours, <> ] - jobs: - - todo-issues: - name: todo-issue-checks - context: - - slack - - scheduled-fpp: - when: - equal: [ build_hourly, <> ] - jobs: - - fpp-verify: - context: - - slack - - oplabs-fpp-nodes - - develop-publish-contract-artifacts: - when: - or: - - equal: [ "develop", <> ] - - equal: [ true, <> ] - jobs: - - publish-contract-artifacts - - develop-fault-proofs: - when: - and: - - or: - - equal: [ "develop", <> ] - - equal: [ true, <> ] - - not: - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] - jobs: - - go-mod-download - - cannon-prestate - - cannon-stf-verify: - requires: - - go-mod-download - context: - - slack - - contracts-bedrock-build: - skip_pattern: test - context: - - slack - - go-e2e-test: - name: op-e2e-cannon-tests - module: op-e2e - target: test-cannon - notify: true - mentions: "@proofs-team" - requires: - - contracts-bedrock-build - - cannon-prestate - context: - - slack - - develop-kontrol-tests: - when: - and: - - or: - - equal: [ "develop", <> ] - - equal: [ true, <> ] - - not: - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] - jobs: - - kontrol-tests: - context: - - slack - - runtimeverification - - scheduled-cannon-full-tests: - when: - or: - - equal: [ build_four_hours, <> ] - - equal: [ true, << pipeline.parameters.cannon_full_test_dispatch >> ] - jobs: - - contracts-bedrock-build: - skip_pattern: test - - cannon-go-lint-and-test: - requires: - - contracts-bedrock-build - context: - - slack - - scheduled-docker-publish: - when: - or: - - equal: [ build_hourly, <> ] - # Trigger on manual triggers if explicitly requested - - equal: [ true, << pipeline.parameters.docker_publish_dispatch >> ] - jobs: - - docker-build: - matrix: - parameters: - docker_name: - - op-node - - op-batcher - - op-program - - op-proposer - - op-challenger - - op-dispute-mon - - op-conductor - - op-supervisor - - cannon - name: <>-docker-publish - docker_tags: <>,<> - platforms: "linux/amd64,linux/arm64" - publish: true - context: - - oplabs-gcr - - slack - - check-cross-platform: - matrix: - parameters: - op_component: - - op-node - - op-batcher - - op-program - - op-proposer - - op-challenger - - op-dispute-mon - - op-conductor - - op-supervisor - - cannon - name: <>-cross-platform - requires: - - <>-docker-publish - - docker-build: - name: contracts-bedrock-docker-publish - docker_name: contracts-bedrock - docker_tags: <>,<> - resource_class: xlarge - publish: true - context: - - oplabs-gcr - - slack - - scheduled-preimage-reproducibility: - when: - or: - - equal: [build_daily, <> ] - # Trigger on manual triggers if explicitly requested - - equal: [ true, << pipeline.parameters.reproducibility_dispatch >> ] - jobs: - - preimage-reproducibility: - context: - slack diff --git a/.github/workflows/cannon.yml b/.github/workflows/cannon.yml new file mode 100644 index 000000000000..ce0a8d768810 --- /dev/null +++ b/.github/workflows/cannon.yml @@ -0,0 +1,253 @@ +name: Cannon Tests + +env: + FOUNDRY_PROFILE: ci + OP_E2E_CANNON_ENABLED: "false" + +on: + push: + branches: [celestia-develop] + pull_request: + branches: [celestia-develop] + workflow_dispatch: + +jobs: + contracts-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Debug step to check directory structure + - name: Debug directory structure + run: | + echo "Current directory: $(pwd)" + echo "Directory contents:" + ls -la + echo "Checking for packages directory:" + ls -la packages || echo "No packages directory" + + # Install just command + - name: Install just + run: | + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to "$HOME/.local/bin" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + # Install Node.js without npm cache initially + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "18" + + - name: Install npm dependencies + run: | + if [ ! -d "packages/contracts-bedrock" ]; then + echo "contracts-bedrock directory not found. Creating it..." + mkdir -p packages/contracts-bedrock + fi + cd packages/contracts-bedrock + + # Initialize package.json if it doesn't exist + if [ ! -f "package.json" ]; then + echo "Creating package.json..." + echo '{ + "name": "contracts-bedrock", + "version": "1.0.0", + "private": true, + "description": "Optimism Bedrock smart contracts", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + } + }' > package.json + fi + + # List contents after setup + echo "Directory contents after setup:" + ls -la + + npm install + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + + - name: Install contract dependencies + working-directory: packages/contracts-bedrock + run: just install + + - name: Pull artifacts + working-directory: packages/contracts-bedrock + run: bash scripts/ops/pull-artifacts.sh + + - name: Build contracts + working-directory: packages/contracts-bedrock + run: forge build --deny-warnings + + - name: Upload contract artifacts + uses: actions/upload-artifact@v4 + with: + name: contract-artifacts + path: | + packages/contracts-bedrock/artifacts + packages/contracts-bedrock/forge-artifacts + packages/contracts-bedrock/cache + + cannon-prestate: + needs: [contracts-build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create contract artifact directories + run: | + mkdir -p packages/contracts-bedrock/{artifacts,forge-artifacts,cache} + + - name: Download contract artifacts + uses: actions/download-artifact@v4 + with: + name: contract-artifacts + path: packages/contracts-bedrock + + - name: Build cannon + run: make cannon + + - name: Build op-program + run: make op-program + + - name: Create prestate directories + run: mkdir -p op-program/bin + + - name: Cache cannon prestate + id: cache-prestate + uses: actions/cache@v4 + with: + path: | + op-program/bin/prestate.bin.gz + op-program/bin/meta.json + op-program/bin/prestate-proof.json + key: cannon-prestate-${{ hashFiles('cannon/bin/cannon') }}-${{ hashFiles('op-program/bin/op-program-client.elf') }} + + - name: Sanitize op-program guest + run: make -f cannon/Makefile sanitize-program GUEST_PROGRAM=op-program/bin/op-program-client.elf + + - name: Generate cannon prestate + if: steps.cache-prestate.outputs.cache-hit != 'true' + run: make cannon-prestate + + - name: Generate cannon-mt prestate + run: make cannon-prestate-mt + + - name: Upload prestate artifacts + uses: actions/upload-artifact@v4 + with: + name: prestate-artifacts + path: | + op-program/bin/prestate.bin.gz + op-program/bin/meta.json + op-program/bin/prestate-proof.json + + cannon-go-test: + needs: [contracts-build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create contract artifact directories + run: | + mkdir -p packages/contracts-bedrock/{artifacts,forge-artifacts,cache} + + - name: Download contract artifacts + uses: actions/download-artifact@v4 + with: + name: contract-artifacts + path: packages/contracts-bedrock + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.21" + + - name: Build example binaries + working-directory: cannon/testdata/example + run: make elf + + - name: Run Go lint + working-directory: cannon + run: make lint + + - name: Create test results directory + run: | + mkdir -p tmp/test-results + mkdir -p tmp/testlogs + + - name: Run tests + working-directory: cannon + run: | + go install gotest.tools/gotestsum@latest + gotestsum --format=testname --junitfile=../tmp/test-results/cannon.xml --jsonfile=../tmp/testlogs/log.json \ + -- -parallel="$(nproc)" -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./... + + - name: Upload coverage + uses: codecov/codecov-action@v4 + with: + files: ./cannon/coverage.out + flags: cannon-go-tests + + - name: Upload test results + uses: actions/upload-artifact@v4 + with: + name: test-results + path: | + tmp/test-results + tmp/testlogs + + # op-e2e-cannon-tests: + # needs: [contracts-build, cannon-prestate] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + + # - name: Install Foundry + # uses: foundry-rs/foundry-toolchain@v1 + + # - name: Create directories + # run: | + # mkdir -p packages/contracts-bedrock/{artifacts,forge-artifacts,cache} + # mkdir -p op-program/bin + + # - name: Download contract artifacts + # uses: actions/download-artifact@v4 + # with: + # name: contract-artifacts + # path: packages/contracts-bedrock + + # - name: Download prestate artifacts + # uses: actions/download-artifact@v4 + # with: + # name: prestate-artifacts + # path: op-program/bin + + # # Add Go installation since op-e2e tests are in Go + # - name: Set up Go + # uses: actions/setup-go@v5 + # with: + # go-version: "1.21" + + # # Install any required Go dependencies + # - name: Install Go dependencies + # run: | + # go mod download + + # # Debug step to verify forge installation and environment + # - name: Debug environment + # run: | + # echo "Foundry version:" + # forge --version + # echo "Go version:" + # go version + # echo "Directory structure:" + # ls -R packages/contracts-bedrock/ + # ls -R op-program/bin/ + + # - name: Run cannon tests + # working-directory: op-e2e + # run: | + # make test-cannon diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..c57c08f8a38a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,106 @@ +name: CI + +env: + FOUNDRY_PROFILE: ci + DOCKER_BUILDKIT: 1 + REGISTRY: ghcr.io + REPO: ${{ github.repository_owner }} + +on: + push: + branches: + - celestia-develop + - test/github-actions + pull_request: + branches: + - celestia-develop + types: [opened, synchronize, reopened] + workflow_dispatch: + +jobs: + docker-build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + docker_name: + - op-node + - op-batcher + - op-program + - op-proposer + - op-challenger + - proofs-tools + - op-dispute-mon + - op-conductor + - da-server + - op-supervisor + - cannon + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate clean branch name + id: clean_branch + run: | + # Replace invalid characters with dash + CLEAN_BRANCH="$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9.]/-/g')" + echo "name=${CLEAN_BRANCH}" >> "$GITHUB_OUTPUT" + + - name: Set build environment + run: | + { + echo "REGISTRY=${{ env.REGISTRY }}" + echo "REPOSITORY=${{ env.REPO }}" + echo "GIT_COMMIT=$(git rev-parse HEAD)" + echo "GIT_DATE=$(git show -s --format='%ct')" + echo "GIT_VERSION=untagged" + echo "IMAGE_TAGS=${{ github.sha }},${{ steps.clean_branch.outputs.name }}" + echo "PLATFORMS=linux/amd64" + } >> "$GITHUB_ENV" + + - name: Build Docker image + run: | + # Create buildx builder + docker buildx create --driver=docker-container --name=buildx-build --bootstrap --use + + # For PRs, use --load to make the image available locally + # For pushes, use --push to publish to registry + OUTPUT_ARG="" + if [ "${{ github.event_name }}" = "pull_request" ]; then + OUTPUT_ARG="--load" + else + OUTPUT_ARG="--push" + fi + + # Build using docker-bake.hcl + docker buildx bake \ + --progress plain \ + --builder=buildx-build \ + -f docker-bake.hcl \ + "${OUTPUT_ARG}" \ + "${{ matrix.docker_name }}" + + - name: Save Docker image + if: github.event_name == 'pull_request' + run: | + CLEAN_TAG="${{ steps.clean_branch.outputs.name }}" + IMAGE_NAME="${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.docker_name }}:${CLEAN_TAG}" + docker save "${IMAGE_NAME}" > "/tmp/${{ matrix.docker_name }}.tar" + + - name: Upload Docker image + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: docker-${{ matrix.docker_name }} + path: /tmp/${{ matrix.docker_name }}.tar diff --git a/.github/workflows/tag-service.yml b/.github/workflows/tag-service.yml deleted file mode 100644 index 439b48f13d4e..000000000000 --- a/.github/workflows/tag-service.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Tag Service - -on: - workflow_dispatch: - inputs: - bump: - description: 'How much to bump the version by' - required: true - type: choice - options: - - major - - minor - - patch - - prerelease - - finalize-prerelease - service: - description: 'Which service to release' - required: true - type: choice - options: - - ci-builder - - ci-builder-rust - - op-node - - op-batcher - - op-proposer - - op-challenger - - op-program - - op-dispute-mon - - op-ufm - - da-server - - op-contracts - - op-conductor - prerelease: - description: Increment major/minor/patch as prerelease? - required: false - type: boolean - default: false - -permissions: - contents: write - -jobs: - release: - runs-on: ubuntu-latest - environment: op-stack-production - - steps: - - uses: actions/checkout@v4 - - name: Fetch tags - run: git fetch --tags origin --force - - name: Setup Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install deps - run: pip install -r requirements.txt - working-directory: ops/tag-service - - run: ops/tag-service/tag-service.py --bump="$BUMP" --service="$SERVICE" - env: - INPUT_GITHUB_TOKEN: ${{ github.token }} - BUMP: ${{ github.event.inputs.bump }} - SERVICE: ${{ github.event.inputs.service }} - if: ${{ github.event.inputs.prerelease == 'false' }} - - run: ops/tag-service/tag-service.py --bump="$BUMP" --service="$SERVICE" --pre-release - env: - INPUT_GITHUB_TOKEN: ${{ github.token }} - BUMP: ${{ github.event.inputs.bump }} - SERVICE: ${{ github.event.inputs.service }} - if: ${{ github.event.inputs.prerelease == 'true' }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 000000000000..af25a9533814 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,28 @@ +name: Validate Workflows + +on: + pull_request: + branches: + - celestia-develop + paths: + - ".github/workflows/**" + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create actionlint config + run: | + echo "self-hosted-runner: + # Add your custom runner labels here if needed + labels: + - ubuntu-latest + - ubuntu-latest-16-cores" > .actionlint.yaml + + - name: Validate workflows + uses: docker://rhysd/actionlint:latest + with: + shellcheck: true + fail-on-error: true