[release-1.30] Bump k3s version for master and add/enhance tests

.github/workflows/test-suite.yaml

@@ -66,7 +66,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        itest: [etcdsnapshot]
+        itest: [startup, etcdsnapshot]
       max-parallel: 3
     env:
       GOCOVERDIR: /tmp/rke2cov
@@ -96,7 +96,7 @@ jobs:
     - name: Run Integration Tests
       run: | 
         mkdir -p $GOCOVERDIR
-        sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration
+        sudo -E env "PATH=$PATH" go test -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration -ginkgo.v -test.v
     - name: Generate coverage report
       run:  go tool covdata textfmt -i $GOCOVERDIR -o ${{ matrix.itest }}.out
     - name: Upload Results To Codecov
@@ -114,7 +114,7 @@ jobs:
     name: "E2E Tests"
     needs: build
     runs-on: ubuntu-24.04
-    timeout-minutes: 40
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
@@ -161,4 +161,4 @@ jobs:
     - name: Run ${{ matrix.etest }} Test
       run: | 
         cd tests/e2e/${{ matrix.etest }}
-        go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local
+        go test -timeout=45m ./${{ matrix.etest}}_test.go -ci -local -ginkgo.v -test.v

go.mod

@@ -80,7 +80,7 @@ require (
 	github.com/google/go-containerregistry v0.20.2
 	github.com/iamacarpet/go-win64api v0.0.0-20210311141720-fe38760bed28
 	github.com/k3s-io/helm-controller v0.16.5
-	github.com/k3s-io/k3s v1.30.9-0.20250113231420-9515436e80e4 // release-1.30
+	github.com/k3s-io/k3s v1.30.9-0.20250115234947-be4b1bcdba33 // release-1.30
 	github.com/k3s-io/kine v0.13.5
 	github.com/libp2p/go-netroute v0.2.1
 	github.com/natefinch/lumberjack v2.0.0+incompatible

go.sum

@@ -1198,8 +1198,8 @@ github.com/k3s-io/etcd/server/v3 v3.5.16-k3s1 h1:9c0DChFw6WRz6r+eCuVLBltZcRwT6h1
 github.com/k3s-io/etcd/server/v3 v3.5.16-k3s1/go.mod h1:ynhyZZpdDp1Gq49jkUg5mfkDWZwXnn3eIqCqtJnrD/s=
 github.com/k3s-io/helm-controller v0.16.5 h1:SsUHfksQXNwePkswv4a970EGD2h0Exsf6t3IdXhpXRo=
 github.com/k3s-io/helm-controller v0.16.5/go.mod h1:AcSxEhOIUgeVvBTnJOAwcezBZXtYew/RhKwO5xp3RlM=
-github.com/k3s-io/k3s v1.30.9-0.20250113231420-9515436e80e4 h1:GlB6C7J5qTnAFEl5LEuGi+SM0rWK6QBknOPpNpBd6iM=
-github.com/k3s-io/k3s v1.30.9-0.20250113231420-9515436e80e4/go.mod h1:fVAVGfz7U891Z4sckLBkV9vOAsCgpjO6bvmW3s8Qf9s=
+github.com/k3s-io/k3s v1.30.9-0.20250115234947-be4b1bcdba33 h1:FSB6wLh9N3lYMiFwa4SpFkd9WITCVYVAqlVPdCyCb7Q=
+github.com/k3s-io/k3s v1.30.9-0.20250115234947-be4b1bcdba33/go.mod h1:fVAVGfz7U891Z4sckLBkV9vOAsCgpjO6bvmW3s8Qf9s=
 github.com/k3s-io/kine v0.13.5 h1:bVTcxCHJVUqeyz0mJenO+WFHkxtgD2aRwLt1JcJ0pq8=
 github.com/k3s-io/kine v0.13.5/go.mod h1:OIG+kPc9cU+NDHnVm+aR0PyZYExdZWU+vlJynhH4haU=
 github.com/k3s-io/klog v1.0.0-k3s2/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=

tests/e2e/splitserver/splitserver_test.go

@@ -49,7 +49,7 @@ func createSplitCluster(nodeOS string, etcdCount, controlPlaneCount, agentCount
 		}
 	}
 
-	cmd := fmt.Sprintf(`E2E_NODE_ROLES="%s" E2E_NODE_BOXES="%s" %s vagrant up &> vagrant.log`, nodeRoles, nodeBoxes, testOptions)
+	cmd := fmt.Sprintf(`E2E_NODE_ROLES="%s" E2E_NODE_BOXES="%s" %s vagrant up --no-tty &> vagrant.log`, nodeRoles, nodeBoxes, testOptions)
 	fmt.Println(cmd)
 	if _, err := e2e.RunCommand(cmd); err != nil {
 		fmt.Println("Error Creating Cluster", err)

tests/e2e/testutils.go

@@ -117,7 +117,7 @@ func CreateCluster(nodeOS string, serverCount int, agentCount int) ([]string, []
 	}
 
 	// Bring up the first server node
-	cmd := fmt.Sprintf(`%s %s vagrant up %s &> vagrant.log`, nodeEnvs, testOptions, serverNodeNames[0])
+	cmd := fmt.Sprintf(`%s %s vagrant up --no-tty %s &> vagrant.log`, nodeEnvs, testOptions, serverNodeNames[0])
 
 	fmt.Println(cmd)
 	if _, err := RunCommand(cmd); err != nil {
@@ -127,7 +127,7 @@ func CreateCluster(nodeOS string, serverCount int, agentCount int) ([]string, []
 	// Bring up the rest of the nodes in parallel
 	errg, _ := errgroup.WithContext(context.Background())
 	for _, node := range append(serverNodeNames[1:], agentNodeNames...) {
-		cmd := fmt.Sprintf(`%s %s vagrant up %s &>> vagrant.log`, nodeEnvs, testOptions, node)
+		cmd := fmt.Sprintf(`%s %s vagrant up --no-tty %s &>> vagrant.log`, nodeEnvs, testOptions, node)
 		fmt.Println(cmd)
 		errg.Go(func() error {
 			if _, err := RunCommand(cmd); err != nil {
@@ -155,7 +155,7 @@ func CreateMixedCluster(nodeOS string, serverCount, linuxAgentCount, windowsAgen
 		}
 	}
 
-	cmd := fmt.Sprintf("%s %s vagrant up &> vagrant.log", nodeEnvs, testOptions)
+	cmd := fmt.Sprintf("%s %s vagrant up --no-tty &> vagrant.log", nodeEnvs, testOptions)
 	fmt.Println(cmd)
 	if _, err := RunCommand(cmd); err != nil {
 		fmt.Println("Error Creating Cluster", err)
@@ -213,7 +213,7 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [
 	testOptions += " E2E_RELEASE_VERSION=skip"
 
 	// Standup all VMs. In GitHub Actions, this also imports the VM image into libvirt, which takes time to complete.
-	cmd := fmt.Sprintf(`%s %s E2E_STANDUP_PARALLEL=true vagrant up --no-provision &> vagrant.log`, nodeEnvs, testOptions)
+	cmd := fmt.Sprintf(`%s %s E2E_STANDUP_PARALLEL=true vagrant up --no-tty --no-provision &> vagrant.log`, nodeEnvs, testOptions)
 	fmt.Println(cmd)
 	if _, err := RunCommand(cmd); err != nil {
 		return nil, nil, newNodeError(cmd, serverNodeNames[0], err)
@@ -296,7 +296,7 @@ func CreateLocalMixedCluster(nodeOS string, serverCount, linuxAgentCount, window
 	testOptions += " E2E_RELEASE_VERSION=skip"
 
 	// Standup all nodes, relying on vagrant-libvirt native parallel provisioning
-	cmd := fmt.Sprintf(`%s %s E2E_STANDUP_PARALLEL=true vagrant up --no-provision &> vagrant.log`, nodeEnvs, testOptions)
+	cmd := fmt.Sprintf(`%s %s E2E_STANDUP_PARALLEL=true vagrant up --no-tty --no-provision &> vagrant.log`, nodeEnvs, testOptions)
 	fmt.Println(cmd)
 	if _, err := RunCommand(cmd); err != nil {
 		return nil, nil, nil, err
@@ -419,7 +419,7 @@ func GenReport(specReport ginkgo.SpecReport) {
 }
 
 // GetVagrantLog returns the logs of on vagrant commands that initialize the nodes and provision RKE2 on each node.
-// It also attempts to fetch the systemctl logs of RKE2 on nodes where the rke2.service failed.
+// It also attempts to fetch the systemctl and kubelet/containerd logs of RKE2 on nodes where the rke2.service failed.
 func GetVagrantLog(cErr error) string {
 	var nodeErr *NodeError
 	nodeJournal := ""
@@ -430,6 +430,12 @@ func GetVagrantLog(cErr error) string {
 			nodeJournal, _ = RunCmdOnNode("sudo journalctl -u rke2* --no-pager", nodeErr.Node)
 		}
 		nodeJournal = "\nNode Journal Logs:\n" + nodeJournal
+
+		paths := []string{"/var/lib/rancher/rke2/agent/logs/kubelet.log", "/var/lib/rancher/rke2/agent/containerd/containerd.log"}
+		for _, path := range paths {
+			out, _ := RunCmdOnNode("sudo cat "+path, nodeErr.Node)
+			nodeJournal += "\n" + path + ":\n" + out + "\n"
+		}
 	}
 
 	log, err := os.Open("vagrant.log")
@@ -546,7 +552,9 @@ func RunCmdOnWindowsNode(cmd string, nodename string) (string, error) {
 
 // RunCommand execute a command on the host
 func RunCommand(cmd string) (string, error) {
-	c := exec.Command("bash", "-c", cmd)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*15)
+	defer cancel()
+	c := exec.CommandContext(ctx, "bash", "-c", cmd)
 	out, err := c.CombinedOutput()
 	return string(out), err
 }

tests/integration/etcdsnapshot/etcd_int_test.go

@@ -1,4 +1,4 @@
-package tests
+package etcdsnapshot
 
 import (
 	"fmt"

tests/integration/integration.go

@@ -162,7 +162,10 @@ func SaveLog(log *os.File, dump bool) error {
 // for use with the next test.
 func Cleanup(rke2TestLock int) error {
 	// Save the agent/images directory
-	cmd := exec.Command("cp", "-r", "/var/lib/rancher/rke2/agent/images", "/tmp/images-backup")
+	if err := os.MkdirAll("/tmp/images-backup", 0755); err != nil && !errors.Is(err, os.ErrExist) {
+		return fmt.Errorf("failed to make backup images directory: %w", err)
+	}
+	cmd := exec.Command("sh", "-c", "mv /var/lib/rancher/rke2/agent/images/* /tmp/images-backup/")
 	if res, err := cmd.CombinedOutput(); err != nil {
 		return fmt.Errorf("error backing up images directory: %s: %w", res, err)
 	}
@@ -179,7 +182,7 @@ func Cleanup(rke2TestLock int) error {
 	if err := os.MkdirAll("/var/lib/rancher/rke2/agent", 0755); err != nil {
 		return fmt.Errorf("failed to make agent directory: %w", err)
 	}
-	cmd = exec.Command("mv", "/tmp/images-backup", "/var/lib/rancher/rke2/agent/images")
+	cmd = exec.Command("sh", "-c", "mv /tmp/images-backup/* /var/lib/rancher/rke2/agent/images/")
 	if res, err := cmd.CombinedOutput(); err != nil {
 		return fmt.Errorf("error restoring images directory: %s: %w", res, err)
 	}

tests/integration/startup/startup_test.go

@@ -0,0 +1,87 @@
+package startup
+
+import (
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	testutil "github.com/rancher/rke2/tests/integration"
+	"github.com/sirupsen/logrus"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+)
+
+var (
+	serverLog  *os.File
+	serverArgs = []string{"--debug"}
+	testLock   int
+)
+
+var _ = BeforeSuite(func() {
+	var err error
+	testLock, err = testutil.AcquireTestLock()
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = Describe("startup tests", Ordered, func() {
+	When("a default server is created", func() {
+		It("starts successfully", func() {
+			var err error
+			serverLog, err = testutil.StartServer(serverArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default components deployed", func() {
+			Eventually(func() error {
+				err := testutil.ServerReady()
+				if err != nil {
+					logrus.Info(err)
+				}
+				return err
+			}, "240s", "15s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.KillServer(serverLog)).To(Succeed())
+			Expect(testutil.Cleanup(testLock)).To(Succeed())
+		})
+	})
+	When("a server is created with bind-address", func() {
+		It("starts successfully", func() {
+			hostIP, _ := utilnet.ChooseHostInterface()
+			var err error
+			serverLog, err = testutil.StartServer(append(serverArgs, "--bind-address", hostIP.String())...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default components deployed", func() {
+			Eventually(func() error {
+				err := testutil.ServerReady()
+				if err != nil {
+					logrus.Info(err)
+				}
+				return err
+			}, "240s", "15s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.KillServer(serverLog)).To(Succeed())
+			Expect(testutil.Cleanup(testLock)).To(Succeed())
+		})
+	})
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
+var _ = AfterSuite(func() {
+	if failed {
+		testutil.SaveLog(serverLog, false)
+		serverLog = nil
+	}
+	Expect(testutil.KillServer(serverLog)).To(Succeed())
+	Expect(testutil.Cleanup(testLock)).To(Succeed())
+})
+
+func Test_IntegrationStartup(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Startup Suite")
+}