tests: enable codecov (akash-network#21)

fixes ovrclk/engineering#350

Signed-off-by: Artur Troian <[email protected]>

.codecov.yml

@@ -0,0 +1,25 @@
+comment: false
+codecov:
+  require_ci_to_pass: true
+  branch: main
+
+coverage:
+  precision: 2
+  round: down
+  range: 50...100
+
+  status:
+    # Learn more at https://docs.codecov.io/docs/commit-status
+    project:
+      default:
+        threshold: 1% # allow this much decrease on project
+    changes: false
+
+ignore:
+  - "**/*mock/.*"
+  - "pkg/client"
+  - "**/zz_generated.deepcopy.go"
+  - "_docs"
+  - "_run"
+  - "script"
+  - "make"

.env

@@ -11,4 +11,5 @@ AP_DEVCACHE_INCLUDE=${AP_DEVCACHE}/include
 AP_DEVCACHE_VERSIONS=${AP_DEVCACHE}/versions
 AP_DEVCACHE_NODE_MODULES=${AP_DEVCACHE}
 AP_DEVCACHE_NODE_BIN=${AP_DEVCACHE_NODE_MODULES}/node_modules/.bin
+AP_DEVCACHE_TESTS=${AP_DEVCACHE}/tests
 DEVCACHE_RUN=${AP_DEVCACHE}/run

.github/workflows/codecov.yaml

@@ -0,0 +1,26 @@
+name: codecov
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest:
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: c-py/action-dotenv-to-setenv@v3
+        with:
+          env-file: .env
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "${{ env.GOLANG_VERSION }}"
+      - run: make modvendor
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: .cache/tests/coverage.txt

.github/workflows/standardize-yaml.yaml

@@ -12,4 +12,4 @@ jobs:
       - uses: actions/checkout@v3
       - name: check-yml-count
         run: |
-          if [[ $(git ls-files '*.yml' ':!:codecov.yml' | wc -l) -ne 0 ]]; then git ls-files '*.yml' ':!:codecov.yml' && exit 1;fi
+          if [[ $(git ls-files '*.yml' ':!:.codecov.yml' | wc -l) -ne 0 ]]; then git ls-files '*.yml' ':!:.codecov.yml' && exit 1;fi

.github/workflows/tests.yaml

@@ -58,4 +58,4 @@ jobs:
       - name: lint all
         run: make test-lint-all
       - name: lint make-sublinters
-        run: make test-sublinters 
+        run: make test-sublinters

make/setup-cache.mk

@@ -9,6 +9,7 @@ $(AP_DEVCACHE):
 	mkdir -p $(AP_DEVCACHE_INCLUDE)
 	mkdir -p $(AP_DEVCACHE_VERSIONS)
 	mkdir -p $(AP_DEVCACHE_NODE_MODULES)
+	mkdir -p $(AP_DEVCACHE_TESTS)
 	mkdir -p $(AP_DEVCACHE)/run
 
 .INTERMEDIATE: cache

make/test-integration.mk

@@ -1,4 +1,7 @@
-COVER_PACKAGES = $(shell go list ./... | grep -v 'mock\|pkg/client' | paste -sd, -)
+BUILD_TAGS_K8S_INTEGRATION := k8s_integration
+BUILD_TAGS_E2E := e2e integration
+
+BUILD_TAGS_ALL := "$(BUILD_TAGS_K8S_INTEGRATION) $(BUILD_TAGS_E2E)"
 
 # This is statically specified in the vagrant configuration
 # todo @troian check it still necessary
@@ -23,16 +26,16 @@ test-e2e-integration-k8s:
 
 .PHONY: test-query-app
 test-query-app:
-	 $(INTEGRATION_VARS) $(KIND_VARS) go test -mod=readonly -p 4 -tags "e2e integration" -v ./integration/... -run TestQueryApp
+	 $(INTEGRATION_VARS) $(KIND_VARS) go test -mod=readonly -p 4 -tags "$(BUILD_TAGS_E2E)" -v ./integration/... -run TestQueryApp
 
 .PHONY: test-k8s-integration
 test-k8s-integration:
 	# Assumes cluster created and configured:
 	# ```
 	# KUSTOMIZE_INSTALLS=akash-operator-inventory make kind-cluster-setup-e2e
 	# ```
-	go test -count=1 -v -tags k8s_integration ./pkg/apis/akash.network/v2beta1
-	go test -count=1 -v -tags k8s_integration ./cluster/kube
+	go test -count=1 -v -tags "$(BUILD_TAGS_K8S_INTEGRATION)" ./pkg/apis/akash.network/v2beta1
+	go test -count=1 -v -tags "$(BUILD_TAGS_K8S_INTEGRATION)" ./cluster/kube
 
 
 ###############################################################################
@@ -60,11 +63,8 @@ test-full:
 	$(GO) test -tags=$(BUILD_TAGS) -race ./...
 
 .PHONY: test-coverage
-test-coverage:
-	$(GO) test -tags=$(BUILD_MAINNET) -coverprofile=coverage.txt \
-		-covermode=count \
-		-coverpkg="$(COVER_PACKAGES)" \
-		./...
+test-coverage: $(AP_DEVCACHE)
+	./script/codecov.sh "$(AP_DEVCACHE_TESTS)" $(BUILD_TAGS_ALL)
 
 .PHONY: test-vet
 test-vet:

script/codecov.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -e
+
+OUTDIR=$1
+TAGS=$2
+
+coverage="$OUTDIR/coverage.txt"
+profile="$OUTDIR/profile.out"
+
+set -e
+echo "mode: atomic" > "$coverage"
+while IFS=$'\n' read -r pkg; do
+    go test -timeout 30m -race -coverprofile="$profile" -covermode=atomic -tags="$TAGS" "$pkg"
+    if [ -f "$profile" ]; then
+        tail -n +2 "$profile" >> "$coverage";
+        rm "$profile"
+    fi
+done < <(go list ./... | grep -v 'mock\|pkg/client')

script/is_local_gomod.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-MOD_PATH=$(go list -mod=readonly -m -f '{{ .Replace }}' "$1")
+MOD_PATH=$(go list -mod=readonly -m -f '{{ .Replace }}' "$1" 2>/dev/null)
 
 if [[ "${MOD_PATH}" == "<nil>" ]]; then
 	echo false