diff --git a/.github/scripts/gramine.sh b/.github/scripts/gramine.sh new file mode 100644 index 000000000..8b3e12f62 --- /dev/null +++ b/.github/scripts/gramine.sh @@ -0,0 +1,43 @@ +#/bin/sh +# this is to be ran in a docker container via an github action that has gramine set-up already e.g., +# notaryserverbuilds.azurecr.io/builder/gramine +# with sgx hardware: +# ./gramine.sh sgx +# +# without: +# ./gramine.sh +## + +if [ -z "$1" ] + then + run='gramine-direct notary-server &' + + else + run='gramine-sgx notary-server &' +fi + + + +curl https://sh.rustup.rs -sSf | sh -s -- -y +. "$HOME/.cargo/env" +apt install libssl-dev + +gramine-sgx-gen-private-key +SGX=1 make +gramine-sgx-sign -m notary-server.manifest -o notary-server.sgx +mr_enclave=$(gramine-sgx-sigstruct-view --verbose --output-format=json notary-server.sig |jq .mr_enclave) +echo "mrenclave=$mr_enclave" >> "$GITHUB_OUTPUT" +echo "#### sgx mrenclave" | tee >> $GITHUB_STEP_SUMMARY +echo "\`\`\`${mr_enclave}\`\`\`" | tee >> $GITHUB_STEP_SUMMARY +eval "$run" +sleep 5 + +if [ "$1" ]; then + curl 127.0.0.1:7047/info +else + quote=$(curl 127.0.0.1:7047/info | jq .quote.rawQuote) + echo $quote + echo "quote=$quote" >> $GITHUB_OUTPUT + echo "#### 🔒 signed quote ${quote}" | tee >> $GITHUB_STEP_SUMMARY + echo "${quote}" | tee >> $GITHUB_STEP_SUMMARY +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba7717b74..5339e57bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,4 +136,11 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: lcov.info - fail_ci_if_error: true \ No newline at end of file + fail_ci_if_error: true + # trigger-deployment: + # doing this here due to feedback @ https://github.com/tlsnotary/tlsn/pull/631#issuecomment-2415806267 + # needs: tests-integration + # uses: ./.github/workflows/tee-cd.yml + # with: + # # what this is supposed to do -> $ref is the tag: e.g., v0.1.0-alpha.7; pass the $ref string to the cd script and update reverse proxy / deploy + # ref: ${{ github.ref_name }} diff --git a/.github/workflows/tee-cd.yml b/.github/workflows/tee-cd.yml new file mode 100644 index 000000000..76734611a --- /dev/null +++ b/.github/workflows/tee-cd.yml @@ -0,0 +1,156 @@ +name: azure-tee-release + +permissions: + contents: read + id-token: write + attestations: write + +on: + workflow_dispatch: + inputs: + ref: + description: 'git branch' + required: false + default: 'dev' + type: string + +#on: +# release: +# types: [published] +# branches: +# - 'releases/**' + +env: + GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }} + GIT_COMMIT_TIMESTAMP: ${{ github.event.repository.updated_at}} + REGISTRY: notaryserverbuilds.azurecr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + update-reverse-proxy: + permissions: + contents: write + environment: tee + runs-on: [self-hosted, linux] + outputs: + teeport: ${{ steps.portbump.outputs.newport}} + deploy: ${{ steps.portbump.outputs.deploy}} + steps: + - name: checkout repository + uses: actions/checkout@v4 + - name: update caddyfile + id: portbump + env: + RELEASE_TAG: ${{ github.event.release.tag_name || inputs.ref }} + run: | + echo "tag: $RELEASE_TAG" + NEXT_PORT=$(bash cd-scripts/tee/azure/updateproxy.sh 'cd-scripts/tee/azure/Caddyfile' $RELEASE_TAG) + echo "newport=$NEXT_PORT" >> $GITHUB_OUTPUT + echo "new deploy port: $NEXT_PORT 🚀" >> $GITHUB_STEP_SUMMARY + chmod +r -R cd-scripts/tee/azure/ + - name: Deploy updated Caddyfile to server + if: ${{ steps.portbump.outputs.deploy == 'new' }} + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.AZURE_TEE_PROD_HOST }} + username: ${{ secrets.AZURE_PROD_TEE_USERNAME }} + key: ${{ secrets.AZURE_TEE_PROD_KEY }} + source: "cd-scripts/tee/azure/Caddyfile" + target: "~/" + - name: Reload Caddy on server + if: ${{ steps.portbump.outputs.deploy == 'new' }} + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.AZURE_TEE_PROD_HOST }} + username: ${{ secrets.AZURE_PROD_TEE_USERNAME }} + key: ${{ secrets.AZURE_TEE_PROD_KEY }} + script: | + sudo cp ~/cd-scripts/tee/azure/Caddyfile /etc/caddy/Caddyfile + sudo systemctl reload caddy + build-measure: + environment: tee + runs-on: [self-hosted, linux] + needs: [ update-reverse-proxy ] + container: + image: notaryserverbuilds.azurecr.io/prod/gramine + credentials: + username: notaryserverbuilds + password: ${{ secrets.AZURE_CR_BUILDS_PW }} + env: + GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }} + volumes: + - /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket + options: "--device /dev/sgx_enclave" + steps: + - name: get code + uses: actions/checkout@v4 + - name: sccache + if: github.event_name != 'release' + # && github.event_name != 'workflow_dispatch' + uses: mozilla-actions/sccache-action@v0.0.6 + - name: set rust env for scc + if: github.event_name != 'release' + # && github.event_name != 'workflow_dispatch' + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + - name: reverse proxy port + run: echo "${{needs.update-reverse-proxy.outputs.teeport}}" | tee >> $GITHUB_STEP_SUMMARY + - name: get hardware measurement + working-directory: ${{ github.workspace }}/crates/notary/server/tee + run: | + chmod +x ../../../../.github/scripts/gramine.sh && ../../../../.github/scripts/gramine.sh sgx + artifact-deploy: + environment: tee + runs-on: [self-hosted, linux] + needs: [ build-measure, update-reverse-proxy ] + steps: + - name: auth to registry + uses: docker/login-action@v3 + with: + registry: notaryserverbuilds.azurecr.io + username: notaryserverbuilds + password: ${{ secrets.AZURE_CR_BUILDS_PW }} + - name: get code + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Get Git commit timestamps + run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV + - name: Build and push + id: deploypush + uses: docker/build-push-action@v6 + with: + provenance: mode=max + no-cache: true + context: ${{ github.workspace }}/crates/notary/server/tee + push: true + tags: notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }} + labels: ${{needs.update-reverse-proxy.outputs.teeport}} + env: + # reproducible builds: https://github.com/moby/buildkit/blob/master/docs/build-repro.md#source_date_epoch + SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} + - name: Generate SBOM + uses: anchore/sbom-action@v0 + with: + image: notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }} + format: 'cyclonedx-json' + output-file: 'sbom.cyclonedx.json' + # attestation section :: + # https://docs.docker.com/build/ci/github-actions/attestations/ + - name: Attest + uses: actions/attest-build-provenance@v1 + with: + subject-name: notaryserverbuilds.azurecr.io/prod/notary-sgx + subject-digest: ${{ steps.deploypush.outputs.digest }} + push-to-registry: true + - + name: run + run: | + if [[ ${{ needs.update-reverse-proxy.outputs.deploy }} == 'new' ]]; then + docker run --device /dev/sgx_enclave --device /dev/sgx_provision --volume=/var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket -p ${{needs.update-reverse-proxy.outputs.teeport}}:7047 notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }} & + else + old=$(docker ps --filter "name=${{needs.update-reverse-proxy.outputs.teeport}}") + docker rm -f $old + docker run --name ${{needs.update-reverse-proxy.outputs.teeport}} --device /dev/sgx_enclave --device /dev/sgx_provision --volume=/var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket -p ${{needs.update-reverse-proxy.outputs.teeport}}:7047 notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }} & + fi diff --git a/.github/workflows/tee-ci.yml b/.github/workflows/tee-ci.yml new file mode 100644 index 000000000..badacfaf6 --- /dev/null +++ b/.github/workflows/tee-ci.yml @@ -0,0 +1,42 @@ +name: tee-build + +on: + push: + branches: [ "dev" ] + pull_request: + branches: [ "dev" ] + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-measure-emulated: + environment: tee + runs-on: [self-hosted, linux] + container: + image: notaryserverbuilds.azurecr.io/prod/gramine + credentials: + username: notaryserverbuilds + password: ${{ secrets.AZURE_CR_BUILDS_PW }} + env: + GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }} + steps: + - name: get code + uses: actions/checkout@v4 + - name: sccache + if: github.event_name != 'release' + # && github.event_name != 'workflow_dispatch' + uses: mozilla-actions/sccache-action@v0.0.6 + - name: set rust env for scc + if: github.event_name != 'release' + # && github.event_name != 'workflow_dispatch' + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + + - name: get emulated measurement (call gramine.sh without the sgx arg) + working-directory: ${{ github.workspace }}/crates/notary/server/tee + run: | + # this fails current ci because gramine.sh is part of this pr so the file doesnt exist + # bash .github/scripts/gramine.sh diff --git a/cd-scripts/tee/azure/Caddyfile b/cd-scripts/tee/azure/Caddyfile new file mode 100644 index 000000000..03298948e --- /dev/null +++ b/cd-scripts/tee/azure/Caddyfile @@ -0,0 +1,90 @@ +# +# global block => +# email is for acme +# # # # + +{ + key_type p256 + email mac@pse.dev # for acme + servers { + metrics + } + log { + output stdout + format console { + time_format common_log + time_local + } + level DEBUG + } +} + +# +# server block, acme turned on (default when using dns) +# reverse proxy with fail_duration + lb will try upstreams sequentially (fallback) +# e.g. => `reverse_proxy :4000 :5000 10.10.10.10:1000 tlsnotary.org:443` +# will always deliver to :4000 if its up, but if :4000 is down for more than 4s it trys the next one +# # # # + +notary.codes { + handle_path /v0.1.0-alpha.8* { + reverse_proxy :4003 :3333 { + lb_try_duration 4s + fail_duration 10s + lb_policy header X-Upstream { + fallback first + } + } + } + handle_path /v0.1.0-alpha.7* { + reverse_proxy :4002 :3333 { + lb_try_duration 4s + fail_duration 10s + lb_policy header X-Upstream { + fallback first + } + } + } + handle_path /v0.1.0-alpha.6* { + reverse_proxy :4001 :3333 { + lb_try_duration 4s + fail_duration 10s + lb_policy header X-Upstream { + fallback first + } + } + } + + handle_path /nightly* { + reverse_proxy :3333 { + lb_try_duration 4s + fail_duration 10s + lb_policy header X-Upstream { + fallback first + } + } + } + + handle_path /proxy* { + reverse_proxy :55688 proxy.notary.codes:443 { + lb_try_duration 4s + fail_duration 10s + lb_policy header X-Upstream { + fallback first + } + } + } + + handle { + root * /srv + file_server + } + + handle_errors { + @404 { + expression {http.error.status_code} == 404 + } + rewrite @404 /index.html + file_server + } +} diff --git a/cd-scripts/tee/azure/prometheus.yaml b/cd-scripts/tee/azure/prometheus.yaml new file mode 100644 index 000000000..f152c2905 --- /dev/null +++ b/cd-scripts/tee/azure/prometheus.yaml @@ -0,0 +1,7 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: caddy + static_configs: + - targets: ['localhost:2019'] diff --git a/cd-scripts/tee/azure/updateproxy.sh b/cd-scripts/tee/azure/updateproxy.sh new file mode 100644 index 000000000..45e873798 --- /dev/null +++ b/cd-scripts/tee/azure/updateproxy.sh @@ -0,0 +1,84 @@ +#!/bin/sh + +# Variables (Update these as needed)x +CADDYFILE=${1:-/etc/caddy/Caddyfile} # Path to your Caddyfile +GIT_COMMIT_HASH=${2:-dev} +BASE_PORT=6061 # The starting port for your reverse_proxy directives + +# Function to check if handle_path for the given commit hash exists +handle_path_exists() { + local commit_hash=$1 + #echo "handle_path_exists $1 -- CADDYFILE: $CADDYFILE" + grep -q "handle_path /${commit_hash}\*" "$CADDYFILE" +} + +# Function to extract the port for a given commit hash +extract_port_for_commit() { + local commit_hash=$1 + #echo "extract_port_for_commit $1 -- 2: $2" + grep -Pzo "handle_path /${commit_hash}\* \{\n\s*reverse_proxy :(.*) " "$CADDYFILE" | grep -Poa "reverse_proxy :(.*) " | awk '{print $2}' +} + +# Function to get the last port in the Caddyfile +get_last_port() { + grep -Po "reverse_proxy :([0-9]+)" "$CADDYFILE" | awk -F: '{print $2}' | sort -n | tail -1 +} + +# Function to add a new handle_path block with incremented port inside notary.codes block +add_new_handle_path() { + local new_port=$1 + local commit_hash=$2 + + # Use a temporary file for inserting the handle_path block + tmp_file=$(mktemp) + + # Add the new handle_path in the notary.codes block + awk -v port="$new_port" -v hash="$commit_hash" ' + /notary\.codes \{/ { + print; + print " handle_path /" hash "* {"; + print " reverse_proxy :" port " :3333 {"; + print " lb_try_duration 4s"; + print " fail_duration 10s"; + print " lb_policy header X-Upstream {"; + print " fallback first"; + print " }"; + print " }"; + print " }"; + next; + } + { print } + ' "$CADDYFILE" > "$tmp_file" + + # Overwrite the original Caddyfile with the updated content + mv "$tmp_file" "$CADDYFILE" + +} +#git action perms +r +chmod 664 cd-scripts/tee/azure/Caddyfile + +# Check if the commit hash already exists in a handle_path +if handle_path_exists "$GIT_COMMIT_HASH"; then + existing_port=$(extract_port_for_commit "$GIT_COMMIT_HASH") + echo "${existing_port:1}" + exit 0 +else + # Get the last port used and increment it + last_port=$(get_last_port) + if [[ -z "$last_port" ]]; then + last_port=$BASE_PORT + fi + new_port=$((last_port + 1)) + + # Add the new handle_path block inside notary.codes block + add_new_handle_path "$new_port" "$GIT_COMMIT_HASH" + echo $new_port + # commit the changes + git config user.name github-actions + git config user.email github-actions@github.com + git add -A + git commit --quiet --allow-empty -m "azure tee reverse proxy => port:$NEXT_PORT/${RELEASE_TAG}" + git push --quiet + echo "deploy=new" >> $GITHUB_OUTPUT + exit 0 +fi