Cache workspaces [ros2]

.ci.clang_tidy

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# usage: echo -e $(colorize RED Some ${fancy} text.)
+function colorize() {
+   local color reset
+   while true ; do
+      case "${1:-}" in
+         RED|GREEN|YELLOW|BLUE)
+            color="ANSI_$1"; eval "color=\$$color"; reset="${ANSI_RESET}" ;;
+         THIN)
+            color="${color:-}${ANSI_THIN}" ;;
+         BOLD)
+            color="${color:-}${ANSI_BOLD}"; reset="${reset:-${ANSI_THIN}}" ;;
+         *) break ;;
+      esac
+      shift
+   done
+   echo -e "${color:-}$@${reset:-}"
+}
+
+# collect files that are modified since commit
+function collect_modified_files() {
+  local -n __modified_files=$1     # -n to modify argument by reference
+  local filter=$2
+  local src_dir=${3:-$PWD}
+  local base=${4:-$GITHUB_BASE_REF}
+
+  # Find top-level git folder of src_dir
+  local strip_prefix=$(cd "$src_dir"; git rev-parse --show-toplevel)
+  # Strip git folder from src_dir to keep relative path from git root to source files as stip_prefix
+  strip_prefix="${src_dir#$strip_prefix/}"
+  while IFS='' read -r line ; do
+    # Add modified or added files to array - only using the relative path from src_dir, i.e. removing strip_prefix
+    __modified_files+=("${line#$strip_prefix/}")
+  done < <(git diff --name-only --diff-filter=MA "$base"..HEAD "$src_dir" | grep "$filter")
+}
+
+run_clang_tidy_fix() {
+  local SOURCE_PKGS COMPILED_PKGS counter pkg file src_dir
+  SOURCE_PKGS=($(colcon list $BASEDIR/target_ws/src/$TARGET_REPO_NAME --only-names 2> /dev/null))
+
+  # filter repository packages for those which have a compile_commands.json file in their build folder
+  declare -A PKGS  # associative array
+  for pkg in ${SOURCE_PKGS[@]} ; do
+    file="$BASEDIR/target_ws/build/$pkg/compile_commands.json"
+    test -r "$file" && PKGS[$pkg]=$(dirname "$file")
+  done
+
+  for pkg in ${SOURCE_PKGS[@]} ; do  # process files in topological order
+    test -z "${PKGS[$pkg]:-}" && continue  # skip pkgs without compile_commands.json
+    echo -e $(colorize GREEN "start clang.tidy: $pkg")
+
+    # Find all .cpp files in pkg's src_dir that were added or modified in this pull request
+    modified_files=()
+    if [ "${GITHUB_BASE_REF:-false}" != false ]; then
+      src_dir=$(grep "^CMAKE_HOME_DIRECTORY:INTERNAL=" "${PKGS[$pkg]}/CMakeCache.txt")
+      collect_modified_files modified_files "\.cpp$" $(realpath "${src_dir#*=}") $GITHUB_BASE_REF
+      if [ ${#modified_files[@]} -eq 0 ]; then
+        echo "No modified .cpp files"
+        continue
+      fi
+    fi
+
+    clang-tidy-10 -fix -header-filter='$BASEDIR/target_ws/src/$TARGET_REPO_NAME/.*' -p "${PKGS[$pkg]}" ${modified_files[@]:-} 2> /dev/null
+    # if there are workspace changes, print broken pkg to file descriptor 3
+    travis_have_fixes && 1>&3 echo $pkg || true  # should always succeed ;-)
+    travis_fold end clang.tidy
+  done
+}
+
+pushd $BASEDIR/target_ws/src/$TARGET_REPO_NAME
+
+sudo apt-get install -qq git clang-tidy-10 grep
+
+echo "*********** START *************"
+
+echo -e $(colorize BLUE "GITHUB_HEAD_REF = $GITHUB_HEAD_REF")
+echo -e $(colorize BLUE "GITHUB_BASE_REF = $GITHUB_BASE_REF")
+
+# Ensure the base branch ($GITHUB_BASE_REF) is available
+if [ "$(git rev-parse --abbrev-ref HEAD)" != "$GITHUB_BASE_REF" ] ; then
+    git fetch origin "$GITHUB_BASE_REF"
+    git branch -f "$GITHUB_BASE_REF" FETCH_HEAD
+fi
+
+# Run run_clang_tidy_fix() and redirect file descriptor 3 to /tmp/clang-tidy.tainted to collect tainted pkgs
+run_clang_tidy_fix 3>/tmp/clang-tidy.tainted
+result=$?
+test $result -ne 0 && exit $result
+
+# Read content of /tmp/clang-tidy.tainted into variable TAINTED_PKGS
+TAINTED_PKGS=$(< /tmp/clang-tidy.tainted)
+
+if [ -z "$TAINTED_PKGS" ] ; then
+  echo -e $(colorize GREEN "Passed clang-tidy check")
+else
+  echo -e "$(colorize RED \"clang-tidy check failed for the following packages:\")\\n$(colorize YELLOW $(colorize THIN $TAINTED_PKGS))"
+  exit 2
+fi
+
+popd

.ci.prepare_codecov

@@ -4,6 +4,8 @@ pushd $BASEDIR
 BLUE='\033[0;34m'
 NOCOLOR='\033[0m'
 
+apt-get -qq install lcov
+
 echo -e "${BLUE}Capture coverage info${NOCOLOR}"
 lcov --capture --directory target_ws --output-file coverage.info
 

.github/workflows/industrial_ci_action.yml

@@ -10,34 +10,70 @@ jobs:
     strategy:
       matrix:
         env:
-          - {
-            ROS_DISTRO: foxy, ROS_REPO: main, CCOV_UPLOAD: true,
-            CMAKE_ARGS: "-DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS='--coverage' -DCMAKE_CXX_FLAGS='--coverage'",
-            AFTER_RUN_TARGET_TEST: './.ci.prepare_codecov',
-            ADDITIONAL_DEBS: 'curl lcov grep'
-          }
-          - {ROS_DISTRO: foxy, ROS_REPO: testing, CCOV_UPLOAD: false}
+          - IMAGE: 'foxy-ci'
+            CCOV: true
+            AFTER_RUN_TARGET_TEST: './.ci.prepare_codecov'
+          - IMAGE: 'foxy-ci-testing'
+            EXPORT_COMPILE_COMMANDS: true
+            AFTER_RUN_TARGET_TEST: './.ci.clang_tidy'
     env:
-      UPSTREAM_WORKSPACE: warehouse_ros_mongo.repos
-      CCACHE_DIR: /home/runner/.ccache
-      BASEDIR: .base
+      DOCKER_IMAGE: moveit/moveit2:${{ matrix.env.IMAGE }}
+      UPSTREAM_WORKSPACE: upstream.repos
+      CCOV: ${{ matrix.env.CCOV }}
+      AFTER_SETUP_UPSTREAM_WORKSPACE: 'vcs pull $BASEDIR/upstream_ws/src'
+      TARGET_CMAKE_ARGS: >
+        -DCMAKE_BUILD_TYPE=${{ matrix.env.CCOV && 'RelWithDebInfo' || 'Release'}}
+        ${{ matrix.env.CCOV && '-DCMAKE_CXX_FLAGS="--coverage" --no-warn-unused-cli' || '' }}
+        ${{ matrix.env.EXPORT_COMPILE_COMMANDS && '-DCMAKE_EXPORT_COMPILE_COMMANDS=1' || '' }}
+      CCACHE_DIR: "${{ github.workspace }}/.ccache"
+      BASEDIR: ${{ github.workspace }}/.work
+      CACHE_PREFIX: "${{ matrix.env.IMAGE }}${{ matrix.env.CCOV && '-ccov' || '' }}"
+
+    name: "${{ matrix.env.IMAGE }}${{ matrix.env.CCOV && ' + ccov' || ''}}${{ matrix.env.EXPORT_COMPILE_COMMANDS && ' + clang-tidy' || '' }}"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - name: cache upstream_ws
+        uses: pat-s/[email protected]
+        with:
+          path: ${{ env.BASEDIR }}/upstream_ws
+          key: upstream_ws-${{ env.CACHE_PREFIX }}-${{ hashFiles('moveit2.repos') }}-${{ github.run_id }}
+          restore-keys: |
+            upstream_ws-${{ env.CACHE_PREFIX }}-${{ hashFiles('moveit2.repos') }}
+      - name: cache target_ws
+        if: ${{ ! env.CCOV }}
+        uses: pat-s/[email protected]
+        with:
+          path: ${{ env.BASEDIR }}/target_ws
+          key: target_ws-${{ env.CACHE_PREFIX }}-${{ hashFiles('**/CMakeLists.txt', '**/package.xml') }}-${{ github.run_id }}
+          restore-keys: |
+            target_ws-${{ env.CACHE_PREFIX }}-${{ hashFiles('**/CMakeLists.txt', '**/package.xml') }}
+      - name: cache ccache
+        uses: pat-s/[email protected]
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ matrix.env.ROS_DISTRO }}-${{ matrix.env.ROS_REPO }}-${{ github.sha }}  
+          key: ccache-${{ env.CACHE_PREFIX }}-${{ github.sha }}-${{ github.run_id }}
           restore-keys: |
-            ccache-${{ matrix.env.ROS_DISTRO }}-${{ matrix.env.ROS_REPO }}
-      - uses: 'ros-industrial/industrial_ci@master'
+            ccache-${{ env.CACHE_PREFIX }}-${{ github.sha }}
+            ccache-${{ env.CACHE_PREFIX }}
+      - name: industrial_ci
+        uses: 'ros-industrial/industrial_ci@master'
         env: ${{ matrix.env }}
-      - uses: actions/upload-artifact@v2
+      - name: upload test artifacts (on failure)
+        uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: test-results
           path: ${{ env.BASEDIR }}/target_ws/**/test_results/**/*.xml
-      - uses: codecov/codecov-action@v1
-        if: ${{ matrix.env.CCOV_UPLOAD }} 
+      - name: upload codecov report
+        uses: codecov/codecov-action@v1
+        if: ${{ env.CCOV }}
         with:
           files: ${{ env.BASEDIR }}/coverage.info
+      - name: prepare target_ws for cache
+        if: ${{ always() && ! env.CCOV }}
+        run: |
+          du -sh ${{ env.BASEDIR }}/target_ws
+          sudo find ${{ env.BASEDIR }}/target_ws -wholename '*/test_results/*' -delete
+          sudo rm -rf ${{ env.BASEDIR }}/target_ws/src
+          du -sh ${{ env.BASEDIR }}/target_ws

README.md

@@ -6,8 +6,6 @@ Code for persisting ROS message data using MongoDB.  Contains C++ and Python lib
 
 [![Format](https://github.com/ros-planning/warehouse_ros_mongo/actions/workflows/format.yml/badge.svg?branch=ros2)](https://github.com/ros-planning/warehouse_ros_mongo/actions/workflows/format.yml?branch=ros2) [![BuildAndTest](https://github.com/ros-planning/warehouse_ros_mongo/actions/workflows/industrial_ci_action.yml/badge.svg?branch=ros2)](https://github.com/ros-planning/warehouse_ros_mongo/actions/workflows/industrial_ci_action.yml?branch=ros2) [![codecov](https://codecov.io/gh/ros-planning/warehouse_ros_mongo/branch/ros2/graph/badge.svg?token=W7uHKcY0ly)](https://codecov.io/gh/ros-planning/warehouse_ros_mongo)
 
-[![Code Coverage Grid](https://codecov.io/gh/ros-planning/warehouse_ros_mongo/branch/ros2/graphs/tree.svg)](https://codecov.io/gh/ros-planning/warehouse_ros_mongo/branch/ros2/graphs/tree.svg)
-
 ## Building from source
 
 ### ROS Jade  / Kinetic