azure-pipelines.yml

# Trigger on pushes to `master`
trigger:
- master

# Trigger on PRs against `master`
pr:
- master

# By default, use the `Native` pool of agents
pool: Native

variables:
    JULIA: unbuffer julia --project=$(Build.SourcesDirectory)/.ci --color=yes
    # We limit our parallelism somewhat in order to avoid strange OOM errors while building LLVM
    BINARYBUILDER_NPROC: 16
    JULIA_PKG_SERVER: us-east.pkg.julialang.org
    # Use eager registry to not have to wait for updates of the conservative registry
    JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager

jobs:
- job: generator
  steps:
  - checkout: self
    fetchDepth: 99999
    clean: true
  - bash: |
      # Be fragile, like a beautiful porcelain doll
      set -e

      # Normally we look at the last pushed commit
      COMPARE_AGAINST="HEAD~1"
      # Keyword to be used in the commit message to skip a rebuild
      SKIP_BUILD_COOKIE="[skip build]"
      # This variable will tell us whether we want to skip the build
      export SKIP_BUILD="false"

      if [[ $(Build.Reason) == "PullRequest" ]]; then
          # If we're on a PR though, we look at the entire branch at once
          TARGET_BRANCH="remotes/origin/$(System.PullRequest.TargetBranch)"
          COMPARE_AGAINST=$(git merge-base --fork-point ${TARGET_BRANCH} HEAD)
          git fetch origin "refs/pull/$(System.PullRequest.PullRequestNumber)/head:refs/remotes/origin/pr/$(System.PullRequest.PullRequestNumber)"
          if [[ "$(git show -s --format=%B origin/pr/$(System.PullRequest.PullRequestNumber))" == *"${SKIP_BUILD_COOKIE}"* ]]; then
              SKIP_BUILD="true"
          fi
      else
          if [[ "$(git show -s --format=%B)" == *"${SKIP_BUILD_COOKIE}"* ]]; then
              SKIP_BUILD="true"
          fi
      fi

      $(JULIA) --compile=min -O0 -e 'using InteractiveUtils, Pkg, Downloads, Dates
          versioninfo()
          if !isnothing(Pkg.pkg_server())
              resp = try
                  headers = Pkg.PlatformEngines.get_metadata_headers(Pkg.pkg_server())
                  Downloads.request("$(Pkg.pkg_server())/registries"; headers)
              catch e
                  # Let us know the download of the registry went wrong, but do not hard fail
                  @error "Could not download the registry" exception=(e, catch_backtrace())
                  exit(0)
              end
              last_mod_idx = findfirst(h -> first(h) == "last-modified", resp.headers)
              msg = "PkgServer: " * resp.url
              delay = if !isnothing(last_mod_idx)
                  last_mod = last(resp.headers[last_mod_idx])
                  msg *= " -- last updated: " * last_mod
                  # Manually strip the "GMT" timezone and hope it never changes.
                  # Do not error out if parsing fails.
                  dt = tryparse(DateTime, replace(last_mod, " GMT"=>""), dateformat"e, d u y H:M:S")
                  # If parsing did fail, set the delay to 0.
                  isnothing(dt) ? Second(0) : now(UTC) - dt
              else
                  Second(0)
              end
              delay > Second(0) && (msg *= " (" * string(Dates.canonicalize(round(delay, Second))) * " ago)")
              @info(msg)
              tolerance = Hour(1)
              delay > tolerance && @warn "The PkgServer registry is older than $(tolerance)"
          end'

      # Get the directories holding changed files
      # 1. All changed files
      # 2. Only files in directories
      # 3. dirname
      # 4. Unique the directories
      PROJECTS=$(git diff-tree --no-commit-id --name-only -r HEAD "${COMPARE_AGAINST}" | grep -E ".+/.+"| sed 's#/[^/]*$##' | sort -u)

      # If there are scary projects we need to exclude, we list them here. (Used to contain `LLVM`)
      EXCLUDED_NAMES=" "

      # This is the dynamic mapping we're going to build up, if it's empty we don't do anything
      PROJECTS_ACCEPTED=()
      for PROJECT in ${PROJECTS}; do
          NAME=$(basename "${PROJECT}")
          echo "Considering ${PROJECT}"
          # Only accept things that contain a `build_tarballs.jl`
          while [[ ! -f "${PROJECT}/build_tarballs.jl" ]] && [[ "${PROJECT}" == */* ]]; do
              echo " --> ${PROJECT} does not contain a build_tarballs.jl, moving up a directory"
              PROJECT="$(dirname "${PROJECT}")"
          done
          if [[ "${PROJECT}" != */* ]]; then
              echo " --> Skipping as we could not find a build_tarballs.jl"
              continue
          fi

          # Ignore RootFS stuff, we'll do that manually
          if [[ "${PROJECT}" == "0_RootFS/"* ]]; then
              echo " --> Skipping as it's within 0_RootFS/"
              continue
          fi

          # Ignore stuff in our excluded projects
          if [[ "${EXCLUDED_NAMES}" == *" ${NAME} "* ]]; then
              echo " --> Skipping as it's excluded"
              continue
          fi

          # Otherwise, emit a build with `PROJECT` set to `${PROJECT}`
          if [[ " ${PROJECTS_ACCEPTED[@]} " =~ " ${PROJECT} " ]]; then
              echo " --> Already in accepted projects, skipping"
          else
              echo " --> Accepted!"
              PROJECTS_ACCEPTED+=("${PROJECT}")
          fi
      done
      if [[ -n "${PROJECTS_ACCEPTED[@]}" ]]; then
          if [[ ${#PROJECTS_ACCEPTED[@]} -gt 20 ]]; then
              echo "Too many projects requested"
              exit 1
          fi

          # Next, we're going to ensure that our BB is up to date and precompiled
          $(JULIA) -e "import Pkg; Pkg.instantiate(); Pkg.precompile()"

          # We're going to snarf out the BB and BBB tree hashes and combine them to be used later in our build cache
          BB_HASH=$($(JULIA) -e "using Pkg, SHA; \
                                 gethash(uuid) = collect(Pkg.Types.Context().env.manifest[Pkg.Types.UUID(uuid)].tree_hash.bytes); \
                                 print(bytes2hex(sha256(vcat( \
                                     gethash(\"7f725544-6523-48cd-82d1-3fa08ff4056e\"), \
                                     gethash(\"12aac903-9f7c-5d81-afc2-d9565ea332ae\"), \
                                 ))));")

          # Next, for each project, download its sources. We do this by generating meta.json
          # files, then parsing them with `download_sources.jl`
          for PROJECT in "${PROJECTS_ACCEPTED[@]}"; do
              NAME=$(basename ${PROJECT})

              # We always invoke a `build_tarballs.jl` file from its own directory
              pushd ${PROJECT} >/dev/null
              echo "Generating meta.json..."
              JSON_PATH="$(Agent.TempDirectory)/${NAME}.meta.json"
              $(JULIA) --compile=min ./build_tarballs.jl --meta-json="${JSON_PATH}"

              echo "Downloading sources..."
              $(JULIA) $(Build.SourcesDirectory)/.ci/download_sources.jl "${JSON_PATH}" $(Agent.TempDirectory)/${NAME}.platforms.list

              # Pop back up to the overworld
              popd >/dev/null
          done

          # Emit project variable declarations
          for PROJECT in "${PROJECTS_ACCEPTED[@]}"; do
              NAME=$(basename ${PROJECT})
          done

          # Emit project/platform joint variable declarations
          VAR_PROJECTS="##vso[task.setVariable variable=projects;isOutput=true]{"
          VAR_PROJPLATFORMS="##vso[task.setVariable variable=projplatforms;isOutput=true]{"
          echo "Determining builds to queue..."
          for PROJECT in "${PROJECTS_ACCEPTED[@]}"; do
              NAME=$(basename ${PROJECT})

              # "project source hash" is a combination of meta.json (to absorb
              # changes from include()'ing a `common.jl`) as well as the entire
              # tree the project lives in (to absorb changes from patches)
              # In order to support symlinked directories as dependencies we calculate
              # the tree hash on a TMP_PROJECT using `cp -RL` to resolve the symlink and
              # hash the actual content.
              TMP_PROJECT=$(mktemp -d)
              cp -RL ${PROJECT} ${TMP_PROJECT}
              TREE_HASH=$($(JULIA) -e "using Pkg; print(bytes2hex(Pkg.GitTools.tree_hash(\"${TMP_PROJECT}\")))")
              META_HASH=$(shasum -a 256 "$(Agent.TempDirectory)/${NAME}.meta.json" | cut -d' ' -f1)
              PROJ_HASH=$(echo -n ${TREE_HASH}${META_HASH} | shasum -a 256 | cut -d' ' -f1)

              # Load in the platforms
              PLATFORMS=$(cat $(Agent.TempDirectory)/${NAME}.platforms.list)
              if [[ -z "${PLATFORMS}" ]]; then
                  echo "##vso[task.logissue type=error]Unable to determine the proper platforms for ${NAME}"
                  continue
              fi

              # That's everything we need to know for `$(PROJECTS)` later on down
              VAR_PROJECTS="${VAR_PROJECTS} '${NAME}':{ \
                                'NAME': '${NAME}', \
                                'PROJECT':'${PROJECT}', \
                                'PLATFORMS':'${PLATFORMS}', \
                                'BB_HASH':'${BB_HASH}', \
                                'PROJ_HASH':'${PROJ_HASH}', \
                                'SKIP_BUILD':'${SKIP_BUILD}' \
                            }, "

              # Some debugging info
              echo " ---> ${NAME}: ${BB_HASH}/${PROJ_HASH} (${TREE_HASH} + ${META_HASH})"

              # For $(PROJPLATFORMS)`, we need to know more...
              for PLATFORM in ${PLATFORMS}; do
                  if [[ "${SKIP_BUILD}" == "true" ]]; then
                      echo "The commit messages contains ${SKIP_BUILD_COOKIE}, skipping build"
                      break
                  fi

                  # Here, we hit the build cache to see if we can skip this particular combo
                  CACHE_URL="https://julia-bb-buildcache.s3.amazonaws.com/${BB_HASH}/${PROJ_HASH}/${PLATFORM}.tar.gz"
                  CURL_HTTP_CODE=$(curl --output /tmp/curl_${PROJ_HASH}_${PLATFORM}.log --silent --include --HEAD "${CACHE_URL}" --write-out '%{http_code}')
                  if [[ "${CURL_HTTP_CODE}" == "200" ]]; then
                      echo "    ${PLATFORM}: skipping, existant"
                      continue;
                  fi
                  echo "    ${PLATFORM}: building"

                  # # Debugging: let's see why `curl` failed:
                  # echo "CACHE_URL: ${CACHE_URL}"
                  # cat /tmp/curl_${PROJ_HASH}_${PLATFORM}.log || true

                  # Otherwise, emit the build
                  VAR_PROJPLATFORMS="${VAR_PROJPLATFORMS} '${NAME}-${PLATFORM}':{ \
                      'NAME': '${NAME}', \
                      'PROJECT':'${PROJECT}', \
                      'PLATFORM':'${PLATFORM}', \
                      'PROJ_HASH':'${PROJ_HASH}', \
                      'BB_HASH':'${BB_HASH}' \
                  }, "
              done
              rm -f /tmp/curl_${PROJ_HASH}*.log
          done
          # Add closing parens
          VAR_PROJECTS="${VAR_PROJECTS} }"
          VAR_PROJPLATFORMS="${VAR_PROJPLATFORMS} }"
          
          # Actually output the variables
          echo "${VAR_PROJECTS}"
          echo "${VAR_PROJPLATFORMS}"
      fi
    env:
      GITHUB_TOKEN: $(GITHUB_TOKEN)
    name: mtrx

- job: jll_init
  dependsOn: generator
  timeoutInMinutes: 10
  cancelTimeoutInMinutes: 2
  strategy:
    matrix: $[ dependencies.generator.outputs['mtrx.projects'] ]
  variables:
    projects: $[ dependencies.generator.outputs['mtrx.projects'] ]
  steps:
  - script: |
      # Fail on error
      set -e

      cd $(PROJECT)
      echo "Generating meta.json..."
      $(JULIA) --compile=min ./build_tarballs.jl --meta-json=$(Agent.TempDirectory)/$(NAME).meta.json
      echo "Initializing JLL package..."
      $(JULIA) $(Build.SourcesDirectory)/.ci/jll_init.jl "$(Agent.TempDirectory)/${NAME}.meta.json"
    env:
      GITHUB_TOKEN: $(GITHUB_TOKEN)
    displayName: "initialize JLL package"
    condition: and(and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master')), ne(variables['projects'], ''))

- job: build
  dependsOn:
   - generator
   - jll_init
  timeoutInMinutes: 240
  cancelTimeoutInMinutes: 2
  strategy:
    matrix: $[ dependencies.generator.outputs['mtrx.projplatforms'] ]
  variables:
    projplatforms: $[ dependencies.generator.outputs['mtrx.projplatforms'] ]
  steps:
  - script: |
      # Fail on error
      set -e

      # Cleanup temporary things that might have been left-over
      ./clean_builds.sh
      ./clean_products.sh

      cd $(PROJECT)
      $(JULIA) ./build_tarballs.jl --verbose $(PLATFORM)

      # After building, we take the single tarball produced with the proper NAME, and upload it:
      TARBALLS=( ./products/${NAME%@*}*${PLATFORM}*.tar.gz )
      if [[ "${#TARBALLS[@]}" != 1 ]]; then
          echo "Multiple tarballs?  This isn't right!" >&2
          exit 1
      fi

      # Upload with curl
      ACL="x-amz-acl:public-read"
      CONTENT_TYPE="application/x-gtar"
      BUCKET="julia-bb-buildcache"
      BUCKET_PATH="${BB_HASH}/${PROJ_HASH}/${PLATFORM}.tar.gz"
      DATE="$(date -R)"
      S3SIGNATURE=$(echo -en "PUT\n\n${CONTENT_TYPE}\n${DATE}\n${ACL}\n/${BUCKET}/${BUCKET_PATH}" | openssl sha1 -hmac "${S3SECRET}" -binary | base64)
      HOST="${BUCKET}.s3.amazonaws.com"
      echo "Uploading artifact to https://${HOST}/${BUCKET_PATH}"
      curl -X PUT -T "${TARBALLS[0]}" \
          -H "Host: ${HOST}" \
          -H "Date: ${DATE}" \
          -H "Content-Type: ${CONTENT_TYPE}" \
          -H "${ACL}" \
          -H "Authorization: AWS ${S3KEY}:${S3SIGNATURE}" \
          "https://${HOST}/${BUCKET_PATH}"

      if [[ "$?" != 0 ]]; then
          echo "Failed to upload artifact!" >&2
          exit 1
      fi
    env:
      GITHUB_TOKEN: $(GITHUB_TOKEN)
      S3KEY: $(S3KEY)
      S3SECRET: $(S3SECRET)
      BINARYBUILDER_NPROC: $(BINARYBUILDER_NPROC)
    displayName: "run build_tarballs.jl"
    condition: and(ne(variables['projplatforms'], ''), ne(variables['projplatforms'], '{ }'))

- job: register
  dependsOn:
  - generator
  - build
  strategy:
    matrix: $[ dependencies.generator.outputs['mtrx.projects'] ]
    maxParallel: 1
  variables:
    projects: $[ dependencies.generator.outputs['mtrx.projects'] ]
  steps:
  - script: |
      # Fail on error
      set -e

      cd $(PROJECT)
      echo "Generating meta.json..."
      $(JULIA) --compile=min ./build_tarballs.jl --meta-json=$(Agent.TempDirectory)/$(NAME).meta.json
      echo "Registering $(NAME)..."
      export BB_HASH PROJ_HASH
      $(JULIA) $(Build.SourcesDirectory)/.ci/register_package.jl $(Agent.TempDirectory)/$(NAME).meta.json --verbose
    env:
      GITHUB_TOKEN: $(GITHUB_TOKEN)
    displayName: "register JLL package"
    # We only register if this is on `master`; same as setting `${DEPLOY}` above.
    condition: and(and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master')), ne(variables['projects'], ''))