ci: Out of the tarpit

.github/actions/go-test-setup/action.yml

@@ -9,3 +9,9 @@ runs:
       shell: bash
       # This matches only tests with "NoCover" in their test name to avoid running all tests again.
       run: go test -tags nocover -run NoCover -v ./...
+    - name: Install testing tools
+      shell: bash
+      run: cd scripts/test_analysis && go install ./cmd/gotest2sql
+    - name: Install test_analysis
+      shell: bash
+      run: cd scripts/test_analysis && go install .

.github/workflows/go-check.yml

@@ -17,4 +17,5 @@ jobs:
   go-check:
     uses: ipdxco/unified-github-workflows/.github/workflows/[email protected]
     with:
+      go-version: "1.22.x"
       go-generate-ignore-protoc-version-comments: true

.github/workflows/go-test-template.yml

@@ -0,0 +1,154 @@
+name: Go Test
+on:
+  workflow_call:
+    inputs:
+      go-versions:
+        required: false
+        type: string
+        default: '["this", "next"]'
+    secrets:
+      CODECOV_TOKEN:
+        required: false
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  unit:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu", "macos", "windows"]
+        go: ${{ fromJSON(inputs.go-versions) }}
+    env:
+      GOTESTFLAGS: -cover -coverprofile=module-coverage.txt -coverpkg=./...
+      GO386FLAGS: ""
+      GORACEFLAGS: ""
+    runs-on: ${{ fromJSON(vars[format('UCI_GO_TEST_RUNNER_{0}', matrix.os)] || format('"{0}-latest"', matrix.os)) }}
+    name: ${{ matrix.os }} (go ${{ matrix.go }})
+    steps:
+      - name: Use msys2 on windows
+        if: matrix.os == 'windows'
+        # The executable for msys2 is also called bash.cmd
+        #   https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#shells
+        # If we prepend its location to the PATH
+        #   subsequent 'shell: bash' steps will use msys2 instead of gitbash
+        run: echo "C:/msys64/usr/bin" >> $GITHUB_PATH
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Check out the latest stable version of Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: ${{ matrix.os != 'windows' }} # Windows VMs are slow to use caching. Can add ~15m to the job
+      - name: Read the Unified GitHub Workflows configuration
+        id: config
+        uses: ipdxco/unified-github-workflows/.github/actions/read-config@main
+      - name: Read the go.mod file
+        id: go-mod
+        uses: ipdxco/unified-github-workflows/.github/actions/read-go-mod@main
+      - name: Determine the Go version to use based on the go.mod file
+        id: go
+        env:
+          MATRIX_GO: ${{ matrix.go }}
+          GO_MOD_VERSION: ${{ fromJSON(steps.go-mod.outputs.json).Go }}
+        run: |
+          if [[ "$MATRIX_GO" == "this" ]]; then
+            echo "version=$GO_MOD_VERSION.x" >> $GITHUB_OUTPUT
+          elif [[ "$MATRIX_GO" == "next" ]]; then
+            MAJOR="${GO_MOD_VERSION%.[0-9]*}"
+            MINOR="${GO_MOD_VERSION#[0-9]*.}"
+            echo "version=$MAJOR.$(($MINOR+1)).x" >> $GITHUB_OUTPUT
+          elif [[ "$MATRIX_GO" == "prev" ]]; then
+            MAJOR="${GO_MOD_VERSION%.[0-9]*}"
+            MINOR="${GO_MOD_VERSION#[0-9]*.}"
+            echo "version=$MAJOR.$(($MINOR-1)).x" >> $GITHUB_OUTPUT
+          else
+            echo "version=$MATRIX_GO" >> $GITHUB_OUTPUT
+          fi
+      - name: Enable shuffle flag for go test command
+        if: toJSON(fromJSON(steps.config.outputs.json).shuffle) != 'false'
+        run: |
+          echo "GOTESTFLAGS=-shuffle=on $GOTESTFLAGS" >> $GITHUB_ENV
+          echo "GO386FLAGS=-shuffle=on $GO386FLAGS" >> $GITHUB_ENV
+          echo "GORACEFLAGS=-shuffle=on $GORACEFLAGS" >> $GITHUB_ENV
+      - name: Enable verbose flag for go test command
+        if: toJSON(fromJSON(steps.config.outputs.json).verbose) != 'false'
+        run: |
+          echo "GOTESTFLAGS=-v $GOTESTFLAGS" >> $GITHUB_ENV
+          echo "GO386FLAGS=-v $GO386FLAGS" >> $GITHUB_ENV
+          echo "GORACEFLAGS=-v $GORACEFLAGS" >> $GITHUB_ENV
+      - name: Set extra flags for go test command
+        if: fromJSON(steps.config.outputs.json).gotestflags != ''
+        run: |
+          echo "GOTESTFLAGS=${{ fromJSON(steps.config.outputs.json).gotestflags }} $GOTESTFLAGS" >> $GITHUB_ENV
+      - name: Set extra flags for go test race command
+        if: fromJSON(steps.config.outputs.json).goraceflags != ''
+        run: |
+          echo "GORACEFLAGS=${{ fromJSON(steps.config.outputs.json).goraceflags }} $GORACEFLAGS" >> $GITHUB_ENV
+      - name: Set up the Go version read from the go.mod file
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache: ${{ matrix.os != 'windows' }} # Windows VMs are slow to use caching. Can add ~15m to the job
+      - name: Display the Go version and environment
+        run: |
+          go version
+          go env
+      - name: Run repo-specific setup
+        uses: ./.github/actions/go-test-setup
+        if: hashFiles('./.github/actions/go-test-setup') != ''
+      - name: Run tests
+        id: test
+        if: contains(fromJSON(steps.config.outputs.json).skipOSes, matrix.os) == false
+        uses: protocol/[email protected]
+        with:
+          run: test_analysis ${{ env.GOTESTFLAGS }}
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.os }}_${{ matrix.go }}_test_results.db
+          path: ./test_results.db
+      - name: Add failure summary
+        if: always()
+        run: |
+          echo "### Failure Summary" >> $GITHUB_STEP_SUMMARY
+          test_analysis summarize >> $GITHUB_STEP_SUMMARY
+      - name: Remove test results
+        run: rm ./test_results.db
+      - name: Run tests with race detector
+        # speed things up. Windows and OSX VMs are slow
+        if: matrix.os == 'ubuntu' &&
+          fromJSON(steps.config.outputs.json).skipRace != true &&
+          contains(fromJSON(steps.config.outputs.json).skipOSes, matrix.os) == false
+        uses: protocol/[email protected]
+        id: race
+        with:
+          run: test_analysis -race ${{ env.GORACEFLAGS }} ./...
+      - name: Upload test results (Race)
+        if: (steps.race.conclusion == 'success' || steps.race.conclusion == 'failure')
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.os }}_${{ matrix.go }}_test_results_race.db
+          path: ./test_results.db
+      - name: Add failure summary
+        if: (steps.race.conclusion == 'success' || steps.race.conclusion == 'failure')
+        run: |
+          echo "# Tests with race detector failure summary" >> $GITHUB_STEP_SUMMARY
+          test_analysis summarize >> $GITHUB_STEP_SUMMARY
+      - name: Adding Link to Run Analysis
+        run: echo "### [Test flakiness analysis](https://observablehq.com/d/d74435ea5bbf24c7?run-id=$GITHUB_RUN_ID)" >> $GITHUB_STEP_SUMMARY
+      - name: Collect coverage files
+        id: coverages
+        run: echo "files=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_OUTPUT
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4.1.0
+        with:
+          files: ${{ steps.coverages.outputs.files }}
+          env_vars: OS=${{ matrix.os }}, GO=${{ steps.go.outputs.version }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false

.github/workflows/go-test.yml

@@ -15,7 +15,7 @@ concurrency:
 
 jobs:
   go-test:
-    uses: libp2p/uci/.github/workflows/go-test.yml@v1.0
+    uses: ./.github/workflows/go-test-template.yml
     with:
       go-versions: '["1.21.x", "1.22.x"]'
     secrets:

libp2p_test.go

@@ -5,6 +5,8 @@ import (
 	"crypto/rand"
 	"errors"
 	"fmt"
+	"net"
+	"net/netip"
 	"regexp"
 	"strconv"
 	"strings"
@@ -488,10 +490,23 @@ func TestHostAddrsFactoryAddsCerthashes(t *testing.T) {
 	h.Close()
 }
 
+func newRandomPort(t *testing.T) string {
+	t.Helper()
+	// Find an available port
+	c, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
+	require.NoError(t, err)
+	c.LocalAddr().Network()
+	ipPort := netip.MustParseAddrPort(c.LocalAddr().String())
+	port := strconv.Itoa(int(ipPort.Port()))
+	require.NoError(t, c.Close())
+	return port
+}
+
 func TestWebRTCReuseAddrWithQUIC(t *testing.T) {
+	port := newRandomPort(t)
 	order := [][]string{
-		{"/ip4/127.0.0.1/udp/54322/quic-v1", "/ip4/127.0.0.1/udp/54322/webrtc-direct"},
-		{"/ip4/127.0.0.1/udp/54322/webrtc-direct", "/ip4/127.0.0.1/udp/54322/quic-v1"},
+		{"/ip4/127.0.0.1/udp/" + port + "/quic-v1", "/ip4/127.0.0.1/udp/" + port + "/webrtc-direct"},
+		{"/ip4/127.0.0.1/udp/" + port + "/webrtc-direct", "/ip4/127.0.0.1/udp/" + port + "/quic-v1"},
 		// We do not support WebRTC automatically reusing QUIC addresses if port is not specified, yet.
 		// {"/ip4/127.0.0.1/udp/0/webrtc-direct", "/ip4/127.0.0.1/udp/0/quic-v1"},
 	}
@@ -542,16 +557,18 @@ func TestWebRTCReuseAddrWithQUIC(t *testing.T) {
 		})
 	}
 
-	swapPort := func(addrStrs []string, newPort string) []string {
+	swapPort := func(addrStrs []string, oldPort, newPort string) []string {
 		out := make([]string, 0, len(addrStrs))
 		for _, addrStr := range addrStrs {
-			out = append(out, strings.Replace(addrStr, "54322", newPort, 1))
+			out = append(out, strings.Replace(addrStr, oldPort, newPort, 1))
 		}
 		return out
 	}
 
 	t.Run("setup with no reuseport. Should fail", func(t *testing.T) {
-		h1, err := New(ListenAddrStrings(swapPort(order[0], "54323")...), Transport(quic.NewTransport), Transport(libp2pwebrtc.New), QUICReuse(quicreuse.NewConnManager, quicreuse.DisableReuseport()))
+		oldPort := port
+		newPort := newRandomPort(t)
+		h1, err := New(ListenAddrStrings(swapPort(order[0], oldPort, newPort)...), Transport(quic.NewTransport), Transport(libp2pwebrtc.New), QUICReuse(quicreuse.NewConnManager, quicreuse.DisableReuseport()))
 		require.NoError(t, err) // It's a bug/feature that swarm.Listen does not error if at least one transport succeeds in listening.
 		defer h1.Close()
 		// Check that webrtc did fail to listen
@@ -560,7 +577,9 @@ func TestWebRTCReuseAddrWithQUIC(t *testing.T) {
 	})
 
 	t.Run("setup with autonat", func(t *testing.T) {
-		h1, err := New(EnableAutoNATv2(), ListenAddrStrings(swapPort(order[0], "54324")...), Transport(quic.NewTransport), Transport(libp2pwebrtc.New), QUICReuse(quicreuse.NewConnManager, quicreuse.DisableReuseport()))
+		oldPort := port
+		newPort := newRandomPort(t)
+		h1, err := New(EnableAutoNATv2(), ListenAddrStrings(swapPort(order[0], oldPort, newPort)...), Transport(quic.NewTransport), Transport(libp2pwebrtc.New), QUICReuse(quicreuse.NewConnManager, quicreuse.DisableReuseport()))
 		require.NoError(t, err) // It's a bug/feature that swarm.Listen does not error if at least one transport succeeds in listening.
 		defer h1.Close()
 		// Check that webrtc did fail to listen

p2p/protocol/identify/id.go

@@ -997,7 +997,7 @@ func (ids *idService) addConnWithLock(c network.Conn) {
 }
 
 func signedPeerRecordFromMessage(msg *pb.Identify) (*record.Envelope, error) {
-	if msg.SignedPeerRecord == nil || len(msg.SignedPeerRecord) == 0 {
+	if len(msg.SignedPeerRecord) == 0 {
 		return nil, nil
 	}
 	env, _, err := record.ConsumeEnvelope(msg.SignedPeerRecord, peer.PeerRecordEnvelopeDomain)

p2p/protocol/identify/id_test.go

@@ -960,7 +960,7 @@ func waitForAddrInStream(t *testing.T, s <-chan ma.Multiaddr, expected ma.Multia
 			}
 			continue
 		case <-time.After(timeout):
-			t.Fatalf(failMsg)
+			t.Fatal(failMsg)
 		}
 	}
 }

p2p/transport/quic/cmd/client/main.go

@@ -14,6 +14,6 @@ func main() {
 		return
 	}
 	if err := cmdlib.RunClient(os.Args[1], os.Args[2]); err != nil {
-		log.Fatalf(err.Error())
+		log.Fatal(err.Error())
 	}
 }

p2p/transport/quic/cmd/server/main.go

@@ -14,6 +14,6 @@ func main() {
 		return
 	}
 	if err := cmdlib.RunServer(os.Args[1], nil); err != nil {
-		log.Fatalf(err.Error())
+		log.Fatal(err.Error())
 	}
 }

scripts/test_analysis/cmd/gotest2sql/main.go

@@ -0,0 +1,100 @@
+// gotest2sql inserts the output of go test -json ./... into a sqlite database
+package main
+
+import (
+	"bufio"
+	"database/sql"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	_ "github.com/glebarez/go-sqlite"
+)
+
+type TestEvent struct {
+	Time    time.Time // encodes as an RFC3339-format string
+	Action  string
+	Package string
+	Test    string
+	Elapsed float64 // seconds
+	Output  string
+}
+
+func main() {
+	outputPath := flag.String("output", "", "output db file")
+	verbose := flag.Bool("v", false, "Print test output to stdout")
+	flag.Parse()
+
+	if *outputPath == "" {
+		log.Fatal("-output path is required")
+	}
+
+	db, err := sql.Open("sqlite", *outputPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Create a table to store test results.
+	_, err = db.Exec(`
+		CREATE TABLE IF NOT EXISTS test_results (
+			Time TEXT,
+			Action TEXT,
+			Package TEXT,
+			Test TEXT,
+			Elapsed REAL,
+			Output TEXT,
+			BatchInsertTime TEXT
+	)`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	tx, err := db.Begin()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Prepare the insert statement once
+	insertTime := time.Now().Format(time.RFC3339Nano)
+	stmt, err := tx.Prepare(`
+    INSERT INTO test_results (Time, Action, Package, Test, Elapsed, Output, BatchInsertTime)
+    VALUES (?, ?, ?, ?, ?, ?, ?)`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer stmt.Close() // Ensure the statement is closed after use
+
+	s := bufio.NewScanner(os.Stdin)
+	for s.Scan() {
+		line := s.Bytes()
+		var ev TestEvent
+		err = json.Unmarshal(line, &ev)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if *verbose && ev.Action == "output" {
+			fmt.Print(ev.Output)
+		}
+
+		_, err = stmt.Exec(
+			ev.Time.Format(time.RFC3339Nano),
+			ev.Action,
+			ev.Package,
+			ev.Test,
+			ev.Elapsed,
+			ev.Output,
+			insertTime,
+		)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	// Commit the transaction
+	if err := tx.Commit(); err != nil {
+		log.Fatal(err)
+	}
+}