Merge pull request rancher#44150 from nicholasSUSE/git-ssh-custom-por…

…t-v2.7

Git ssh custom port [Backport - relase/v2.7]

pkg/catalogv2/git/download.go

@@ -1,94 +1,111 @@
 package git
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/pem"
 	"fmt"
-	"net/url"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
 
 	"github.com/rancher/rancher/pkg/settings"
 	corev1 "k8s.io/api/core/v1"
 )
 
-const (
-	stateDir  = "management-state/git-repo"
-	staticDir = "/var/lib/rancher-data/local-catalogs/v2"
-	localDir  = "../rancher-data/local-catalogs/v2" // identical to helm.InternalCatalog
-)
+// Ensure runs git clone, clean DIRTY contents and fetch the latest commit
+func Ensure(secret *corev1.Secret, namespace, name, gitURL, commit string, insecureSkipTLS bool, caBundle []byte) error {
+	git, err := gitForRepo(secret, namespace, name, gitURL, insecureSkipTLS, caBundle)
+	if err != nil {
+		return fmt.Errorf("ensure failure: %w", err)
+	}
+
+	// If the repositories are rancher managed and if bundled is set
+	// don't fetch anything from upstream.
+	if isBundled(git) && settings.SystemCatalog.Get() == "bundled" {
+		return nil
+	}
 
-func gitDir(namespace, name, gitURL string) string {
-	staticDir := filepath.Join(staticDir, namespace, name, hash(gitURL))
-	if s, err := os.Stat(staticDir); err == nil && s.IsDir() {
-		return staticDir
+	if err := git.clone(""); err != nil {
+		return fmt.Errorf("ensure failure: %w", err)
 	}
-	localDir := filepath.Join(localDir, namespace, name, hash(gitURL))
-	if s, err := os.Stat(localDir); err == nil && s.IsDir() {
-		return localDir
+
+	if err := git.reset(commit); err == nil {
+		return nil
 	}
-	return filepath.Join(stateDir, namespace, name, hash(gitURL))
+
+	if err := git.fetchAndReset(commit); err != nil {
+		return fmt.Errorf("ensure failure: %w", err)
+	}
+	return nil
 }
 
+// Head runs git clone on directory(if not exist), reset dirty content and return the HEAD commit
 func Head(secret *corev1.Secret, namespace, name, gitURL, branch string, insecureSkipTLS bool, caBundle []byte) (string, error) {
 	git, err := gitForRepo(secret, namespace, name, gitURL, insecureSkipTLS, caBundle)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("head failure: %w", err)
+	}
+
+	if err := git.clone(branch); err != nil {
+		return "", fmt.Errorf("head failure: %w", err)
 	}
 
-	return git.Head(branch)
+	if err := git.reset("HEAD"); err != nil {
+		return "", fmt.Errorf("head failure: %w", err)
+	}
+
+	commit, err := git.currentCommit()
+	if err != nil {
+		return "", fmt.Errorf("head failure: %w", err)
+	}
+
+	return commit, nil
 }
 
+// Update updates git repo if remote sha has changed
 func Update(secret *corev1.Secret, namespace, name, gitURL, branch string, insecureSkipTLS bool, caBundle []byte) (string, error) {
 	git, err := gitForRepo(secret, namespace, name, gitURL, insecureSkipTLS, caBundle)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("update failure: %w", err)
 	}
 
 	if isBundled(git) && settings.SystemCatalog.Get() == "bundled" {
 		return Head(secret, namespace, name, gitURL, branch, insecureSkipTLS, caBundle)
 	}
 
-	commit, err := git.Update(branch)
-	if err != nil && isBundled(git) {
-		return Head(secret, namespace, name, gitURL, branch, insecureSkipTLS, caBundle)
+	if err := git.clone(branch); err != nil {
+		return "", nil
 	}
-	return commit, err
-}
 
-func Ensure(secret *corev1.Secret, namespace, name, gitURL, commit string, insecureSkipTLS bool, caBundle []byte) error {
-	if commit == "" {
-		return nil
+	if err := git.reset("HEAD"); err != nil {
+		return "", fmt.Errorf("update failure: %w", err)
 	}
-	git, err := gitForRepo(secret, namespace, name, gitURL, insecureSkipTLS, caBundle)
+
+	commit, err := git.currentCommit()
 	if err != nil {
-		return err
+		return commit, fmt.Errorf("update failure: %w", err)
 	}
 
-	return git.Ensure(commit)
-}
+	changed, err := git.remoteSHAChanged(branch, commit)
+	if err != nil {
+		return commit, fmt.Errorf("update failure: %w", err)
+	}
+	if !changed {
+		return commit, nil
+	}
 
-func isBundled(git *git) bool {
-	return strings.HasPrefix(git.Directory, staticDir) || strings.HasPrefix(git.Directory, localDir)
+	if err := git.fetchAndReset(branch); err != nil {
+		return "", fmt.Errorf("update failure: %w", err)
+	}
+
+	lastCommit, err := git.currentCommit()
+	if err != nil && isBundled(git) {
+		return Head(secret, namespace, name, gitURL, branch, insecureSkipTLS, caBundle)
+	}
+	return lastCommit, nil
 }
 
 func gitForRepo(secret *corev1.Secret, namespace, name, gitURL string, insecureSkipTLS bool, caBundle []byte) (*git, error) {
-	isGitSSH, err := isGitSSH(gitURL)
+	err := validateURL(gitURL)
 	if err != nil {
-		return nil, fmt.Errorf("failed to verify the type of URL %s: %w", gitURL, err)
-	}
-	if !isGitSSH {
-		u, err := url.Parse(gitURL)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse URL %s: %w", gitURL, err)
-		}
-		if u.Scheme != "http" && u.Scheme != "https" {
-			return nil, fmt.Errorf("invalid git URL scheme %s, only http(s) and git supported", u.Scheme)
-		}
+		return nil, fmt.Errorf("%w: only http(s) or ssh:// supported", err)
 	}
+
 	dir := gitDir(namespace, name, gitURL)
 	headers := map[string]string{}
 	if settings.InstallUUID.Get() != "" {
@@ -106,22 +123,3 @@ func gitForRepo(secret *corev1.Secret, namespace, name, gitURL string, insecureS
 		CABundle:          caBundle,
 	})
 }
-
-func isGitSSH(gitURL string) (bool, error) {
-	// Matches URLs with the format [anything]@[anything]:[anything]
-	return regexp.MatchString("(.+)@(.+):(.+)", gitURL)
-}
-
-func hash(gitURL string) string {
-	b := sha256.Sum256([]byte(gitURL))
-	return hex.EncodeToString(b[:])
-}
-
-// convertDERToPEM converts a src DER certificate into PEM with line breaks, header, and footer.
-func convertDERToPEM(src []byte) []byte {
-	return pem.EncodeToMemory(&pem.Block{
-		Type:    "CERTIFICATE",
-		Headers: map[string]string{},
-		Bytes:   src,
-	})
-}

pkg/catalogv2/git/download_test.go

@@ -1,57 +1,179 @@
 package git
 
 import (
+	"os"
 	"testing"
 
-	assertlib "github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
 )
 
-func Test_isGitSSH(t *testing.T) {
+const chartsSmallForkURL = "https://github.com/rancher/charts-small-fork"
+const mainBranch = "main"
+const lastBranch = "test-1"
+
+func TestMain(m *testing.M) {
+	// Run all the tests
+	exitCode := m.Run()
+
+	// Cleanup after tests
+	cleanup()
+
+	// Exit with the proper code
+	os.Exit(exitCode)
+}
+
+func cleanup() {
+	// Delete the management-state directory
+	os.RemoveAll("management-state")
+}
+
+func Test_Ensure(t *testing.T) {
 	testCases := []struct {
-		gitURL   string
-		expected bool
+		test            string
+		secret          *corev1.Secret
+		namespace       string
+		name            string
+		gitURL          string
+		commit          string
+		insecureSkipTLS bool
+		caBundle        []byte
+		branch          string
+		expectedError   error
 	}{
-		// True cases
-		{"[email protected]:user/repo.git", true},
-		{"[email protected]:user/repo.git", true},
-		{"[email protected]:user/repo", true},
-		{"[email protected]:user/repo-with-dashes.git", true},
-		{"[email protected]:user/repo.git", true},
-		{"[email protected]:user/repo-with-dashes.git", true},
-		{"[email protected]:user/repo", true},
-		// False cases
-		{"https://github.com/user/repo.git", false},
-		{"http://gitlab.com/user/repo.git", false},
-		{"http://gitlab.com/user/repo", false},
-		{"http://gitlab.com", false},
-		{"[email protected]", false},
+		{
+			test:            "#1 TestCase: Success - Clone, Reset And Exit",
+			secret:          nil,
+			namespace:       "cattle-test",
+			name:            "small-fork-test",
+			gitURL:          chartsSmallForkURL,
+			commit:          "0e2b9da9ddde5c1e502bba6474119856496e5026",
+			insecureSkipTLS: false,
+			caBundle:        []byte{},
+			branch:          mainBranch,
+			expectedError:   nil,
+		},
+		{
+			test:            "#2 TestCase: Success - Clone, Reset And Fetch Last Branch",
+			secret:          nil,
+			namespace:       "cattle-test",
+			name:            "small-fork-test",
+			gitURL:          chartsSmallForkURL,
+			commit:          "0e2b9da9ddde5c1e502bba6474119856496e5026",
+			insecureSkipTLS: false,
+			caBundle:        []byte{},
+			branch:          lastBranch,
+			expectedError:   nil,
+		},
 	}
-	assert := assertlib.New(t)
+
 	for _, tc := range testCases {
-		actual, err := isGitSSH(tc.gitURL)
-		if err != nil {
-			t.Errorf("unexpected error: %s", err)
-		}
-		assert.Equalf(tc.expected, actual, "testcase: %v", tc)
+		t.Run(tc.name, func(t *testing.T) {
+			err := Ensure(tc.secret, tc.namespace, tc.name, tc.gitURL, tc.commit, tc.insecureSkipTLS, tc.caBundle)
+			// Check the error
+			if tc.expectedError == nil && tc.expectedError != err {
+				t.Errorf("Expected error: %v |But got: %v", tc.expectedError, err)
+			}
+
+			// Check the error
+			if tc.expectedError == nil && tc.expectedError != err {
+				t.Errorf("Expected error: %v |But got: %v", tc.expectedError, err)
+			}
+			// Only testing error in some cases
+			if err != nil {
+				assert.EqualError(t, tc.expectedError, err.Error())
+			}
+		})
 	}
 }
 
-func Test_gitDir(t *testing.T) {
-	assert := assertlib.New(t)
+func Test_Head(t *testing.T) {
 	testCases := []struct {
-		namespace string
-		name      string
-		gitURL    string
-		expected  string
+		test            string
+		secret          *corev1.Secret
+		namespace       string
+		name            string
+		gitURL          string
+		insecureSkipTLS bool
+		caBundle        []byte
+		branch          string
+		expectedCommit  string
+		expectedError   error
 	}{
 		{
-			"namespace", "name", "https://git.rancher.io/charts",
-			"management-state/git-repo/namespace/name/4b40cac650031b74776e87c1a726b0484d0877c3ec137da0872547ff9b73a721",
+			test:            "#1 TestCase: Success - Clone, Reset And Return Commit",
+			secret:          nil,
+			namespace:       "cattle-test",
+			name:            "small-fork-test",
+			gitURL:          chartsSmallForkURL,
+			insecureSkipTLS: false,
+			caBundle:        []byte{},
+			branch:          mainBranch,
+			expectedCommit:  "226d544def39de56db210e96d2b0b535badf9bdd",
+			expectedError:   nil,
 		},
-		// NOTE(manno): cannot test the other cases without poluting the filesystem
 	}
+
 	for _, tc := range testCases {
-		actual := gitDir(tc.namespace, tc.name, tc.gitURL)
-		assert.Equalf(tc.expected, actual, "testcase: %v", tc)
+		t.Run(tc.name, func(t *testing.T) {
+			commit, err := Head(tc.secret, tc.namespace, tc.name, tc.gitURL, tc.branch, tc.insecureSkipTLS, tc.caBundle)
+			// Check the error
+			if tc.expectedError == nil && tc.expectedError != err {
+				t.Errorf("Expected error: %v |But got: %v", tc.expectedError, err)
+			}
+			// Only testing error in some cases
+			if err != nil {
+				assert.EqualError(t, tc.expectedError, err.Error())
+			}
+
+			assert.Equal(t, len(commit), len(tc.expectedCommit))
+		})
+	}
+}
+
+func Test_Update(t *testing.T) {
+	testCases := []struct {
+		test              string
+		secret            *corev1.Secret
+		namespace         string
+		name              string
+		gitURL            string
+		insecureSkipTLS   bool
+		caBundle          []byte
+		branch            string
+		systemCatalogMode string
+		expectedCommit    string
+		expectedError     error
+	}{
+		{
+			test:              "#1 TestCase: Success ",
+			secret:            nil,
+			namespace:         "cattle-test",
+			name:              "small-fork-test",
+			gitURL:            chartsSmallForkURL,
+			insecureSkipTLS:   false,
+			caBundle:          []byte{},
+			branch:            lastBranch,
+			systemCatalogMode: "",
+			expectedCommit:    "226d544def39de56db210e96d2b0b535badf9bdd",
+			expectedError:     nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			commit, err := Update(tc.secret, tc.namespace, tc.name, tc.gitURL, tc.branch, tc.insecureSkipTLS, tc.caBundle)
+			// Check the error
+			if tc.expectedError == nil && tc.expectedError != err {
+				t.Errorf("Expected error: %v |But got: %v", tc.expectedError, err)
+			}
+
+			// Only testing error in some cases
+			if err != nil {
+				assert.EqualError(t, tc.expectedError, err.Error())
+			}
+
+			assert.Equal(t, len(commit), len(tc.expectedCommit))
+		})
 	}
 }