Capture test code coverage stats (#3135)

.github/workflows/ci.yml

@@ -36,7 +36,14 @@ jobs:
       - name: Test code
         # we're passing -short so that we skip the integration tests, which will be run in parallel below
         run: |
-          go test ./... -short
+          mkdir -p /tmp/code_coverage
+          go test ./... -short -cover -args "-test.gocoverdir=/tmp/code_coverage"
+      - name: Upload code coverage artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-unit-${{ matrix.os }}-${{ github.run_id }}
+          path: /tmp/code_coverage
+
   integration-tests:
     strategy:
       fail-fast: false
@@ -86,8 +93,17 @@ jobs:
       - name: Print git version
         run: git --version
       - name: Test code
+        env:
+          # See https://go.dev/blog/integration-test-coverage
+          LAZYGIT_GOCOVERDIR: /tmp/code_coverage
         run: |
+          mkdir -p /tmp/code_coverage
           ./scripts/run_integration_tests.sh
+      - name: Upload code coverage artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-integration-${{ matrix.git-version }}-${{ github.run_id }}
+          path: /tmp/code_coverage
   build:
     runs-on: ubuntu-latest
     env:
@@ -169,3 +185,31 @@ jobs:
           mode: exactly
           count: 1
           labels: "ignore-for-release, feature, enhancement, bug, maintenance, docs, i18n"
+  upload-coverage:
+    # List all jobs that produce coverage files
+    needs: [unit-tests, integration-tests]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Download all coverage artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: /tmp/code_coverage
+
+      - name: Combine coverage files
+        run: |
+          # Find all directories in /tmp/code_coverage and create a comma-separated list
+          COVERAGE_DIRS=$(find /tmp/code_coverage -mindepth 1 -maxdepth 1 -type d -printf '/tmp/code_coverage/%f,' | sed 's/,$//')
+          echo "Coverage directories: $COVERAGE_DIRS"
+          # Run the combine command with the generated list
+          go tool covdata textfmt -i=$COVERAGE_DIRS -o coverage.out
+          echo "Combined coverage:"
+          go tool cover -func coverage.out | tail -1 | awk '{print $3}'
+
+      - name: Upload to Codacy
+        run: |
+          CODACY_PROJECT_TOKEN=${{ secrets.CODACY_PROJECT_TOKEN }} \
+            bash <(curl -Ls https://coverage.codacy.com/get.sh) report \
+            --force-coverage-parser go -r coverage.out

.gitignore

@@ -42,3 +42,5 @@ __debug_bin
 
 .worktrees
 demo/output/*
+
+coverage.out

pkg/integration/clients/cli.go

@@ -29,17 +29,18 @@ func RunCLI(testNames []string, slow bool, sandbox bool, waitForDebugger bool, r
 		inputDelay = SLOW_INPUT_DELAY
 	}
 
-	err := components.RunTests(
-		getTestsToRun(testNames),
-		log.Printf,
-		runCmdInTerminal,
-		runAndPrintFatalError,
-		sandbox,
-		waitForDebugger,
-		raceDetector,
-		inputDelay,
-		1,
-	)
+	err := components.RunTests(components.RunTestArgs{
+		Tests:           getTestsToRun(testNames),
+		Logf:            log.Printf,
+		RunCmd:          runCmdInTerminal,
+		TestWrapper:     runAndPrintFatalError,
+		Sandbox:         sandbox,
+		WaitForDebugger: waitForDebugger,
+		RaceDetector:    raceDetector,
+		CodeCoverageDir: "",
+		InputDelay:      inputDelay,
+		MaxAttempts:     1,
+	})
 	if err != nil {
 		log.Print(err.Error())
 	}

pkg/integration/clients/go_test.go

@@ -28,13 +28,17 @@ func TestIntegration(t *testing.T) {
 	parallelTotal := tryConvert(os.Getenv("PARALLEL_TOTAL"), 1)
 	parallelIndex := tryConvert(os.Getenv("PARALLEL_INDEX"), 0)
 	raceDetector := os.Getenv("LAZYGIT_RACE_DETECTOR") != ""
+	// LAZYGIT_GOCOVERDIR is the directory where we write coverage files to. If this directory
+	// is defined, go binaries built with the -cover flag will write coverage files to
+	// to it.
+	codeCoverageDir := os.Getenv("LAZYGIT_GOCOVERDIR")
 	testNumber := 0
 
-	err := components.RunTests(
-		tests.GetTests(),
-		t.Logf,
-		runCmdHeadless,
-		func(test *components.IntegrationTest, f func() error) {
+	err := components.RunTests(components.RunTestArgs{
+		Tests:  tests.GetTests(),
+		Logf:   t.Logf,
+		RunCmd: runCmdHeadless,
+		TestWrapper: func(test *components.IntegrationTest, f func() error) {
 			defer func() { testNumber += 1 }()
 			if testNumber%parallelTotal != parallelIndex {
 				return
@@ -52,13 +56,14 @@ func TestIntegration(t *testing.T) {
 				assert.NoError(t, err)
 			})
 		},
-		false,
-		false,
-		raceDetector,
-		0,
+		Sandbox:         false,
+		WaitForDebugger: false,
+		RaceDetector:    raceDetector,
+		CodeCoverageDir: codeCoverageDir,
+		InputDelay:      0,
 		// Allow two attempts at each test to get around flakiness
-		2,
-	)
+		MaxAttempts: 2,
+	})
 
 	assert.NoError(t, err)
 }

pkg/integration/clients/tui.go

@@ -385,17 +385,18 @@ func quit(g *gocui.Gui, v *gocui.View) error {
 }
 
 func runTuiTest(test *components.IntegrationTest, sandbox bool, waitForDebugger bool, raceDetector bool, inputDelay int) {
-	err := components.RunTests(
-		[]*components.IntegrationTest{test},
-		log.Printf,
-		runCmdInTerminal,
-		runAndPrintError,
-		sandbox,
-		waitForDebugger,
-		raceDetector,
-		inputDelay,
-		1,
-	)
+	err := components.RunTests(components.RunTestArgs{
+		Tests:           []*components.IntegrationTest{test},
+		Logf:            log.Printf,
+		RunCmd:          runCmdInTerminal,
+		TestWrapper:     runAndPrintError,
+		Sandbox:         sandbox,
+		WaitForDebugger: waitForDebugger,
+		RaceDetector:    raceDetector,
+		CodeCoverageDir: "",
+		InputDelay:      inputDelay,
+		MaxAttempts:     1,
+	})
 	if err != nil {
 		log.Println(err.Error())
 	}

pkg/integration/components/runner.go

@@ -20,30 +20,32 @@ const (
 	GIT_CONFIG_GLOBAL_ENV_VAR = "GIT_CONFIG_GLOBAL"
 )
 
+type RunTestArgs struct {
+	Tests           []*IntegrationTest
+	Logf            func(format string, formatArgs ...interface{})
+	RunCmd          func(cmd *exec.Cmd) (int, error)
+	TestWrapper     func(test *IntegrationTest, f func() error)
+	Sandbox         bool
+	WaitForDebugger bool
+	RaceDetector    bool
+	CodeCoverageDir string
+	InputDelay      int
+	MaxAttempts     int
+}
+
 // This function lets you run tests either from within `go test` or from a regular binary.
 // The reason for having two separate ways of testing is that `go test` isn't great at
 // showing what's actually happening during the test, but it's still good at running
 // tests in telling you about their results.
-func RunTests(
-	tests []*IntegrationTest,
-	logf func(format string, formatArgs ...interface{}),
-	runCmd func(cmd *exec.Cmd) (int, error),
-	testWrapper func(test *IntegrationTest, f func() error),
-	sandbox bool,
-	waitForDebugger bool,
-	raceDetector bool,
-	inputDelay int,
-	maxAttempts int,
-) error {
+func RunTests(args RunTestArgs) error {
 	projectRootDir := lazycoreUtils.GetLazyRootDirectory()
 	err := os.Chdir(projectRootDir)
 	if err != nil {
 		return err
 	}
 
 	testDir := filepath.Join(projectRootDir, "test", "_results")
-
-	if err := buildLazygit(waitForDebugger, raceDetector); err != nil {
+	if err := buildLazygit(args); err != nil {
 		return err
 	}
 
@@ -52,21 +54,21 @@ func RunTests(
 		return err
 	}
 
-	for _, test := range tests {
+	for _, test := range args.Tests {
 		test := test
 
-		testWrapper(test, func() error { //nolint: thelper
+		args.TestWrapper(test, func() error { //nolint: thelper
 			paths := NewPaths(
 				filepath.Join(testDir, test.Name()),
 			)
 
-			for i := 0; i < maxAttempts; i++ {
-				err := runTest(test, paths, projectRootDir, logf, runCmd, sandbox, waitForDebugger, raceDetector, inputDelay, gitVersion)
+			for i := 0; i < args.MaxAttempts; i++ {
+				err := runTest(test, args, paths, projectRootDir, gitVersion)
 				if err != nil {
-					if i == maxAttempts-1 {
+					if i == args.MaxAttempts-1 {
 						return err
 					}
-					logf("retrying test %s", test.Name())
+					args.Logf("retrying test %s", test.Name())
 				} else {
 					break
 				}
@@ -81,42 +83,37 @@ func RunTests(
 
 func runTest(
 	test *IntegrationTest,
+	args RunTestArgs,
 	paths Paths,
 	projectRootDir string,
-	logf func(format string, formatArgs ...interface{}),
-	runCmd func(cmd *exec.Cmd) (int, error),
-	sandbox bool,
-	waitForDebugger bool,
-	raceDetector bool,
-	inputDelay int,
 	gitVersion *git_commands.GitVersion,
 ) error {
 	if test.Skip() {
-		logf("Skipping test %s", test.Name())
+		args.Logf("Skipping test %s", test.Name())
 		return nil
 	}
 
 	if !test.ShouldRunForGitVersion(gitVersion) {
-		logf("Skipping test %s for git version %d.%d.%d", test.Name(), gitVersion.Major, gitVersion.Minor, gitVersion.Patch)
+		args.Logf("Skipping test %s for git version %d.%d.%d", test.Name(), gitVersion.Major, gitVersion.Minor, gitVersion.Patch)
 		return nil
 	}
 
 	if err := prepareTestDir(test, paths, projectRootDir); err != nil {
 		return err
 	}
 
-	cmd, err := getLazygitCommand(test, paths, projectRootDir, sandbox, waitForDebugger, inputDelay)
+	cmd, err := getLazygitCommand(test, args, paths, projectRootDir)
 	if err != nil {
 		return err
 	}
 
-	pid, err := runCmd(cmd)
+	pid, err := args.RunCmd(cmd)
 
 	// Print race detector log regardless of the command's exit status
-	if raceDetector {
+	if args.RaceDetector {
 		logPath := fmt.Sprintf("%s.%d", raceDetectorLogsPath(), pid)
 		if bytes, err := os.ReadFile(logPath); err == nil {
-			logf("Race detector log:\n" + string(bytes))
+			args.Logf("Race detector log:\n" + string(bytes))
 		}
 	}
 
@@ -139,20 +136,19 @@ func prepareTestDir(
 	return createFixture(test, paths, rootDir)
 }
 
-func buildLazygit(debug bool, raceDetector bool) error {
-	// // TODO: remove this line!
-	// // skipping this because I'm not making changes to the app code atm.
-	// return nil
-
+func buildLazygit(testArgs RunTestArgs) error {
 	args := []string{"go", "build"}
-	if debug {
+	if testArgs.WaitForDebugger {
 		// Disable compiler optimizations (-N) and inlining (-l) because this
 		// makes debugging work better
 		args = append(args, "-gcflags=all=-N -l")
 	}
-	if raceDetector {
+	if testArgs.RaceDetector {
 		args = append(args, "-race")
 	}
+	if testArgs.CodeCoverageDir != "" {
+		args = append(args, "-cover")
+	}
 	args = append(args, "-o", tempLazygitPath(), filepath.FromSlash("pkg/integration/clients/injector/main.go"))
 	osCommand := oscommands.NewDummyOSCommand()
 	return osCommand.Cmd.New(args).Run()
@@ -183,7 +179,7 @@ func getGitVersion() (*git_commands.GitVersion, error) {
 	return git_commands.ParseGitVersion(versionStr)
 }
 
-func getLazygitCommand(test *IntegrationTest, paths Paths, rootDir string, sandbox bool, waitForDebugger bool, inputDelay int) (*exec.Cmd, error) {
+func getLazygitCommand(test *IntegrationTest, args RunTestArgs, paths Paths, rootDir string) (*exec.Cmd, error) {
 	osCommand := oscommands.NewDummyOSCommand()
 
 	err := os.RemoveAll(paths.Config())
@@ -211,11 +207,18 @@ func getLazygitCommand(test *IntegrationTest, paths Paths, rootDir string, sandb
 
 	cmdObj := osCommand.Cmd.New(cmdArgs)
 
+	if args.CodeCoverageDir != "" {
+		// We set this explicitly here rather than inherit it from the test runner's
+		// environment because the test runner has its own coverage directory that
+		// it writes to and so if we pass GOCOVERDIR to that, it will be overwritten.
+		cmdObj.AddEnvVars("GOCOVERDIR=" + args.CodeCoverageDir)
+	}
+
 	cmdObj.AddEnvVars(fmt.Sprintf("%s=%s", TEST_NAME_ENV_VAR, test.Name()))
-	if sandbox {
+	if args.Sandbox {
 		cmdObj.AddEnvVars(fmt.Sprintf("%s=%s", SANDBOX_ENV_VAR, "true"))
 	}
-	if waitForDebugger {
+	if args.WaitForDebugger {
 		cmdObj.AddEnvVars(fmt.Sprintf("%s=true", WAIT_FOR_DEBUGGER_ENV_VAR))
 	}
 	// Set a race detector log path only to avoid spamming the terminal with the
@@ -227,8 +230,8 @@ func getLazygitCommand(test *IntegrationTest, paths Paths, rootDir string, sandb
 		}
 	}
 
-	if inputDelay > 0 {
-		cmdObj.AddEnvVars(fmt.Sprintf("INPUT_DELAY=%d", inputDelay))
+	if args.InputDelay > 0 {
+		cmdObj.AddEnvVars(fmt.Sprintf("INPUT_DELAY=%d", args.InputDelay))
 	}
 
 	cmdObj.AddEnvVars(fmt.Sprintf("%s=%s", GIT_CONFIG_GLOBAL_ENV_VAR, globalGitConfigPath(rootDir)))

scripts/run_integration_tests.sh

@@ -10,7 +10,25 @@ fi
 
 cp test/global_git_config ~/.gitconfig
 
-go test pkg/integration/clients/*.go
+# if the LAZYGIT_GOCOVERDIR env var is set, we'll capture code coverage data
+if [ -n "$LAZYGIT_GOCOVERDIR" ]; then
+  # Go expects us to either be running the test binary directly or running `go test`, but because
+  # we're doing both and because we want to combine coverage data for both, we need to be a little
+  # hacky. To capture the coverage data for the test runner we pass the test.gocoverdir positional
+  # arg, but if we do that then the GOCOVERDIR env var (which you typically pass to the test binary) will be overwritten by the test runner. So we're passing LAZYGIT_COCOVERDIR instead
+  # and then internally passing that to the test binary as GOCOVERDIR.
+  go test -cover -coverpkg=github.com/jesseduffield/lazygit/pkg/... pkg/integration/clients/*.go -args -test.gocoverdir="/tmp/code_coverage"
+
+  # We're merging the coverage data for the sake of having fewer artefacts to upload.
+  # We can't merge inline so we're merging to a tmp dir then moving back to the original.
+  mkdir -p /tmp/code_coverage_merged
+  go tool covdata merge -i=/tmp/code_coverage -o=/tmp/code_coverage_merged
+  rm -rf /tmp/code_coverage
+  mv /tmp/code_coverage_merged /tmp/code_coverage
+else
+  go test pkg/integration/clients/*.go
+fi
+
 EXITCODE=$?
 
 if test -f ~/.gitconfig.lazygit.bak; then