From 7a7cb7c3df4405f0d14c97634f8aab0259cb7900 Mon Sep 17 00:00:00 2001 From: Mark Do Date: Wed, 7 Feb 2024 20:27:20 +0000 Subject: [PATCH] Added module logic to docker_context.sh --- .github/templates/docker_context/action.yml | 6 ++ .../docker_context/docker_context.sh | 17 +++- .github/workflows/build_and_unitest.yml | 91 +++++++++++++++++++ .github/workflows/temp_ci_test.yml | 46 ++++++++-- src/action/test.txt | 2 +- src/interfacing/test2.txt | 2 +- src/world_modeling/test3.txt | 2 +- 7 files changed, 150 insertions(+), 16 deletions(-) diff --git a/.github/templates/docker_context/action.yml b/.github/templates/docker_context/action.yml index 0395e2bc..da48f300 100644 --- a/.github/templates/docker_context/action.yml +++ b/.github/templates/docker_context/action.yml @@ -1,5 +1,11 @@ name: Generate Docker Environment +inputs: + modified-modules: + description: 'Space deliminated list of modified modules' + required: true + default: '' + outputs: docker_matrix: description: "list of docker compose services" diff --git a/.github/templates/docker_context/docker_context.sh b/.github/templates/docker_context/docker_context.sh index 53fdefc7..aef164e0 100755 --- a/.github/templates/docker_context/docker_context.sh +++ b/.github/templates/docker_context/docker_context.sh @@ -7,18 +7,25 @@ set -e # Find docker compose files in 'modules' directory modules=$(find modules -maxdepth 1 -name "docker-compose*") -echo $modules -echo ================================ # Initialize an empty array for JSON objects json_objects=() # Loop through each module while read -r module; do + # Retrieve docker compose service names services=$(docker-compose -f "$module" config --services) module_out=$(echo "$module" | sed -n 's/modules\/docker-compose\.\(.*\)\.yaml/\1/p') + # Only work with modules that are modified + if [[ $1 = *$module_out* ]]; then + echo "$module_out modified" + else + echo "$module_out not changed" + continue + fi + # Loop through each service while read -r service_out; do # Construct JSON object for each service with module and service name @@ -31,7 +38,7 @@ done <<< "$modules" # Convert the array of JSON objects to a single JSON array json_services=$(jq -nc '[( $ARGS.positional[] | fromjson )]' --args -- ${json_objects[*]}) -echo "docker_matrix=$(echo $json_services | jq -c '{include: .}')" #>> #$GITHUB_OUTPUT +echo "docker_matrix=$(echo $json_services | jq -c '{include: .}')" >> #$GITHUB_OUTPUT ################# Setup Docker Registry and Repository Name ################# # Docker Registry to pull/push images @@ -40,5 +47,5 @@ REGISTRY_URL="ghcr.io/watonomous/wato_monorepo" REGISTRY=$(echo "$REGISTRY_URL" | sed 's|^\(.*\)/.*$|\1|') REPOSITORY=$(echo "$REGISTRY_URL" | sed 's|^.*/\(.*\)$|\1|') -echo "registry=$REGISTRY" #>> $GITHUB_OUTPUT -echo "repository=$REPOSITORY" #>> $GITHUB_OUTPUT +echo "registry=$REGISTRY" >> $GITHUB_OUTPUT +echo "repository=$REPOSITORY" >> $GITHUB_OUTPUT diff --git a/.github/workflows/build_and_unitest.yml b/.github/workflows/build_and_unitest.yml index 8e741f65..0c3d72c1 100644 --- a/.github/workflows/build_and_unitest.yml +++ b/.github/workflows/build_and_unitest.yml @@ -9,9 +9,98 @@ on: - main jobs: + # ------------------------------------------------------------------------ + # Event `pull_request`: Compare the last commit of the main branch or last + # remote commit of the PR branch -> to the current commit of a PR branch. + # ------------------------------------------------------------------------ + get_modified_modules: + runs-on: ubuntu-latest + name: Get changed files + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Find changed files inside action folder + id: changed-files-action + uses: tj-actions/changed-files@v42 + with: + files: src/action/** + + - name: Get changed files inside interfacing folder + id: changed-files-interfacing + uses: tj-actions/changed-files@v42 + with: + files: src/interfacing/** + + - name: Get changed files inside perception folder + id: changed-files-perception + uses: tj-actions/changed-files@v42 + with: + files: src/perception/** + + - name: Get changed files inside samples folder + id: changed-files-samples + uses: tj-actions/changed-files@v42 + with: + files: src/samples/** + + - name: Get changed files inside simulation folder + id: changed-files-simulation + uses: tj-actions/changed-files@v42 + with: + files: src/simulation/** + + - name: Get changed files inside world_modeling folder + id: changed-files-world_modeling + uses: tj-actions/changed-files@v42 + with: + files: src/world_modeling/** + + - name: Update CHANGED_MODULES for action + if: steps.changed-files-action.outputs.any_changed == 'true' + run: | + echo "Detected action changes" + echo "CHANGED_MODULES=$CHANGED_MODULES action" >> $GITHUB_ENV + + - name: Update CHANGED_MODULES for interfacing + if: steps.changed-files-interfacing.outputs.any_changed == 'true' + run: | + echo "Detected interfacing changes" + echo "CHANGED_MODULES=$CHANGED_MODULES interfacing" >> $GITHUB_ENV + + - name: Update CHANGED_MODULES for perception + if: steps.changed-files-perception.outputs.any_changed == 'true' + run: | + echo "Detected perception changes" + echo "CHANGED_MODULES=$CHANGED_MODULES perception" >> $GITHUB_ENV + + - name: Update CHANGED_MODULES for samples + if: steps.changed-files-samples.outputs.any_changed == 'true' + run: | + echo "Detected samples changes" + echo "CHANGED_MODULES=$CHANGED_MODULES samples" >> $GITHUB_ENV + + - name: Update CHANGED_MODULES for simulations + if: steps.changed-files-simulations.outputs.any_changed == 'true' + run: | + echo "Detected simulations changes" + echo "CHANGED_MODULES=$CHANGED_MODULES simulations" >> $GITHUB_ENV + + - name: Update CHANGED_MODULES for world_modeling + if: steps.changed-files-world_modeling.outputs.any_changed == 'true' + run: | + echo "Detected world_modeling changes" + echo "CHANGED_MODULES=$CHANGED_MODULES world_modeling" >> $GITHUB_ENV + + - name: List all changed files + run: | + echo "CHANGED_MODULES: $CHANGED_MODULES" + setup-environment: name: Setup environment runs-on: ubuntu-latest + needs: get_modified_modules outputs: docker_matrix: ${{ steps.docker-environment.outputs.docker_matrix }} @@ -31,6 +120,8 @@ jobs: - name: Generate Docker Environment id: docker-environment uses: "./.github/templates/docker_context" + with: + modified-modules: {{env.CHANGED_MODULES}} - name: Generate GitHub Environment id: github-environment diff --git a/.github/workflows/temp_ci_test.yml b/.github/workflows/temp_ci_test.yml index 28103c7e..4defbc9d 100644 --- a/.github/workflows/temp_ci_test.yml +++ b/.github/workflows/temp_ci_test.yml @@ -7,13 +7,13 @@ on: - 'ci-specific-modules' jobs: - changed_files: - # ------------------------------------------------------------------------ - # Event `pull_request`: Compare the last commit of the main branch or last - # remote commit of the PR branch -> to the current commit of a PR branch. - # ------------------------------------------------------------------------ - runs-on: ubuntu-latest # windows-latest || macos-latest - name: Test changed-files + # ------------------------------------------------------------------------ + # Event `pull_request`: Compare the last commit of the main branch or last + # remote commit of the PR branch -> to the current commit of a PR branch. + # ------------------------------------------------------------------------ + get_modified_modules: + runs-on: ubuntu-latest + name: Get changed files steps: - uses: actions/checkout@v4 with: @@ -93,4 +93,34 @@ jobs: - name: List all changed files run: | - echo "CHANGED_MODULES: $CHANGED_MODULES" \ No newline at end of file + echo "CHANGED_MODULES: $CHANGED_MODULES" + +setup-environment: + name: Setup environment + runs-on: ubuntu-latest + needs: get_modified_modules + + outputs: + docker_matrix: ${{ steps.docker-environment.outputs.docker_matrix }} + registry: ${{ steps.docker-environment.outputs.registry }} + repository: ${{ steps.docker-environment.outputs.repository }} + source_branch: ${{ steps.github-environment.outputs.source_branch }} + target_branch: ${{ steps.github-environment.outputs.target_branch }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Setup Watod Environment + run: ./watod_scripts/watod-setup-env.sh + shell: bash + + - name: Generate Docker Environment + id: docker-environment + uses: "./.github/templates/docker_context" + with: + modified-modules: {{env.CHANGED_MODULES}} + + - name: Generate GitHub Environment + id: github-environment + uses: "./.github/templates/github_context" \ No newline at end of file diff --git a/src/action/test.txt b/src/action/test.txt index e440e5c8..bf0d87ab 100644 --- a/src/action/test.txt +++ b/src/action/test.txt @@ -1 +1 @@ -3 \ No newline at end of file +4 \ No newline at end of file diff --git a/src/interfacing/test2.txt b/src/interfacing/test2.txt index e440e5c8..d8263ee9 100644 --- a/src/interfacing/test2.txt +++ b/src/interfacing/test2.txt @@ -1 +1 @@ -3 \ No newline at end of file +2 \ No newline at end of file diff --git a/src/world_modeling/test3.txt b/src/world_modeling/test3.txt index bf0d87ab..56a6051c 100644 --- a/src/world_modeling/test3.txt +++ b/src/world_modeling/test3.txt @@ -1 +1 @@ -4 \ No newline at end of file +1 \ No newline at end of file