diff --git a/pkg/catalogv2/git/download.go b/pkg/catalogv2/git/download.go index 92c012f861d..3ddd948316e 100644 --- a/pkg/catalogv2/git/download.go +++ b/pkg/catalogv2/git/download.go @@ -1,94 +1,111 @@ package git import ( - "crypto/sha256" - "encoding/hex" - "encoding/pem" "fmt" - "net/url" - "os" - "path/filepath" - "regexp" - "strings" "github.com/rancher/rancher/pkg/settings" corev1 "k8s.io/api/core/v1" ) -const ( - stateDir = "management-state/git-repo" - staticDir = "/var/lib/rancher-data/local-catalogs/v2" - localDir = "../rancher-data/local-catalogs/v2" // identical to helm.InternalCatalog -) +// Ensure runs git clone, clean DIRTY contents and fetch the latest commit +func Ensure(secret *corev1.Secret, namespace, name, gitURL, commit string, insecureSkipTLS bool, caBundle []byte) error { + git, err := gitForRepo(secret, namespace, name, gitURL, insecureSkipTLS, caBundle) + if err != nil { + return fmt.Errorf("ensure failure: %w", err) + } + + // If the repositories are rancher managed and if bundled is set + // don't fetch anything from upstream. + if isBundled(git) && settings.SystemCatalog.Get() == "bundled" { + return nil + } -func gitDir(namespace, name, gitURL string) string { - staticDir := filepath.Join(staticDir, namespace, name, hash(gitURL)) - if s, err := os.Stat(staticDir); err == nil && s.IsDir() { - return staticDir + if err := git.clone(""); err != nil { + return fmt.Errorf("ensure failure: %w", err) } - localDir := filepath.Join(localDir, namespace, name, hash(gitURL)) - if s, err := os.Stat(localDir); err == nil && s.IsDir() { - return localDir + + if err := git.reset(commit); err == nil { + return nil } - return filepath.Join(stateDir, namespace, name, hash(gitURL)) + + if err := git.fetchAndReset(commit); err != nil { + return fmt.Errorf("ensure failure: %w", err) + } + return nil } +// Head runs git clone on directory(if not exist), reset dirty content and return the HEAD commit func Head(secret *corev1.Secret, namespace, name, gitURL, branch string, insecureSkipTLS bool, caBundle []byte) (string, error) { git, err := gitForRepo(secret, namespace, name, gitURL, insecureSkipTLS, caBundle) if err != nil { - return "", err + return "", fmt.Errorf("head failure: %w", err) + } + + if err := git.clone(branch); err != nil { + return "", fmt.Errorf("head failure: %w", err) } - return git.Head(branch) + if err := git.reset("HEAD"); err != nil { + return "", fmt.Errorf("head failure: %w", err) + } + + commit, err := git.currentCommit() + if err != nil { + return "", fmt.Errorf("head failure: %w", err) + } + + return commit, nil } +// Update updates git repo if remote sha has changed func Update(secret *corev1.Secret, namespace, name, gitURL, branch string, insecureSkipTLS bool, caBundle []byte) (string, error) { git, err := gitForRepo(secret, namespace, name, gitURL, insecureSkipTLS, caBundle) if err != nil { - return "", err + return "", fmt.Errorf("update failure: %w", err) } if isBundled(git) && settings.SystemCatalog.Get() == "bundled" { return Head(secret, namespace, name, gitURL, branch, insecureSkipTLS, caBundle) } - commit, err := git.Update(branch) - if err != nil && isBundled(git) { - return Head(secret, namespace, name, gitURL, branch, insecureSkipTLS, caBundle) + if err := git.clone(branch); err != nil { + return "", nil } - return commit, err -} -func Ensure(secret *corev1.Secret, namespace, name, gitURL, commit string, insecureSkipTLS bool, caBundle []byte) error { - if commit == "" { - return nil + if err := git.reset("HEAD"); err != nil { + return "", fmt.Errorf("update failure: %w", err) } - git, err := gitForRepo(secret, namespace, name, gitURL, insecureSkipTLS, caBundle) + + commit, err := git.currentCommit() if err != nil { - return err + return commit, fmt.Errorf("update failure: %w", err) } - return git.Ensure(commit) -} + changed, err := git.remoteSHAChanged(branch, commit) + if err != nil { + return commit, fmt.Errorf("update failure: %w", err) + } + if !changed { + return commit, nil + } -func isBundled(git *git) bool { - return strings.HasPrefix(git.Directory, staticDir) || strings.HasPrefix(git.Directory, localDir) + if err := git.fetchAndReset(branch); err != nil { + return "", fmt.Errorf("update failure: %w", err) + } + + lastCommit, err := git.currentCommit() + if err != nil && isBundled(git) { + return Head(secret, namespace, name, gitURL, branch, insecureSkipTLS, caBundle) + } + return lastCommit, nil } func gitForRepo(secret *corev1.Secret, namespace, name, gitURL string, insecureSkipTLS bool, caBundle []byte) (*git, error) { - isGitSSH, err := isGitSSH(gitURL) + err := validateURL(gitURL) if err != nil { - return nil, fmt.Errorf("failed to verify the type of URL %s: %w", gitURL, err) - } - if !isGitSSH { - u, err := url.Parse(gitURL) - if err != nil { - return nil, fmt.Errorf("failed to parse URL %s: %w", gitURL, err) - } - if u.Scheme != "http" && u.Scheme != "https" { - return nil, fmt.Errorf("invalid git URL scheme %s, only http(s) and git supported", u.Scheme) - } + return nil, fmt.Errorf("%w: only http(s) or ssh:// supported", err) } + dir := gitDir(namespace, name, gitURL) headers := map[string]string{} if settings.InstallUUID.Get() != "" { @@ -106,22 +123,3 @@ func gitForRepo(secret *corev1.Secret, namespace, name, gitURL string, insecureS CABundle: caBundle, }) } - -func isGitSSH(gitURL string) (bool, error) { - // Matches URLs with the format [anything]@[anything]:[anything] - return regexp.MatchString("(.+)@(.+):(.+)", gitURL) -} - -func hash(gitURL string) string { - b := sha256.Sum256([]byte(gitURL)) - return hex.EncodeToString(b[:]) -} - -// convertDERToPEM converts a src DER certificate into PEM with line breaks, header, and footer. -func convertDERToPEM(src []byte) []byte { - return pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Headers: map[string]string{}, - Bytes: src, - }) -} diff --git a/pkg/catalogv2/git/download_test.go b/pkg/catalogv2/git/download_test.go index 5c5622fd56f..322cdb23abb 100644 --- a/pkg/catalogv2/git/download_test.go +++ b/pkg/catalogv2/git/download_test.go @@ -1,57 +1,179 @@ package git import ( + "os" "testing" - assertlib "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" ) -func Test_isGitSSH(t *testing.T) { +const chartsSmallForkURL = "https://github.com/rancher/charts-small-fork" +const mainBranch = "main" +const lastBranch = "test-1" + +func TestMain(m *testing.M) { + // Run all the tests + exitCode := m.Run() + + // Cleanup after tests + cleanup() + + // Exit with the proper code + os.Exit(exitCode) +} + +func cleanup() { + // Delete the management-state directory + os.RemoveAll("management-state") +} + +func Test_Ensure(t *testing.T) { testCases := []struct { - gitURL string - expected bool + test string + secret *corev1.Secret + namespace string + name string + gitURL string + commit string + insecureSkipTLS bool + caBundle []byte + branch string + expectedError error }{ - // True cases - {"customusername@github.com:user/repo.git", true}, - {"customusername@gitlab.com:user/repo.git", true}, - {"customusername@gitlab.com:user/repo", true}, - {"customusername@gitlab.com:user/repo-with-dashes.git", true}, - {"git@github.com:user/repo.git", true}, - {"git@gitlab.com:user/repo-with-dashes.git", true}, - {"git@gitlab.com:user/repo", true}, - // False cases - {"https://github.com/user/repo.git", false}, - {"http://gitlab.com/user/repo.git", false}, - {"http://gitlab.com/user/repo", false}, - {"http://gitlab.com", false}, - {"git@gitlab.com", false}, + { + test: "#1 TestCase: Success - Clone, Reset And Exit", + secret: nil, + namespace: "cattle-test", + name: "small-fork-test", + gitURL: chartsSmallForkURL, + commit: "0e2b9da9ddde5c1e502bba6474119856496e5026", + insecureSkipTLS: false, + caBundle: []byte{}, + branch: mainBranch, + expectedError: nil, + }, + { + test: "#2 TestCase: Success - Clone, Reset And Fetch Last Branch", + secret: nil, + namespace: "cattle-test", + name: "small-fork-test", + gitURL: chartsSmallForkURL, + commit: "0e2b9da9ddde5c1e502bba6474119856496e5026", + insecureSkipTLS: false, + caBundle: []byte{}, + branch: lastBranch, + expectedError: nil, + }, } - assert := assertlib.New(t) + for _, tc := range testCases { - actual, err := isGitSSH(tc.gitURL) - if err != nil { - t.Errorf("unexpected error: %s", err) - } - assert.Equalf(tc.expected, actual, "testcase: %v", tc) + t.Run(tc.name, func(t *testing.T) { + err := Ensure(tc.secret, tc.namespace, tc.name, tc.gitURL, tc.commit, tc.insecureSkipTLS, tc.caBundle) + // Check the error + if tc.expectedError == nil && tc.expectedError != err { + t.Errorf("Expected error: %v |But got: %v", tc.expectedError, err) + } + + // Check the error + if tc.expectedError == nil && tc.expectedError != err { + t.Errorf("Expected error: %v |But got: %v", tc.expectedError, err) + } + // Only testing error in some cases + if err != nil { + assert.EqualError(t, tc.expectedError, err.Error()) + } + }) } } -func Test_gitDir(t *testing.T) { - assert := assertlib.New(t) +func Test_Head(t *testing.T) { testCases := []struct { - namespace string - name string - gitURL string - expected string + test string + secret *corev1.Secret + namespace string + name string + gitURL string + insecureSkipTLS bool + caBundle []byte + branch string + expectedCommit string + expectedError error }{ { - "namespace", "name", "https://git.rancher.io/charts", - "management-state/git-repo/namespace/name/4b40cac650031b74776e87c1a726b0484d0877c3ec137da0872547ff9b73a721", + test: "#1 TestCase: Success - Clone, Reset And Return Commit", + secret: nil, + namespace: "cattle-test", + name: "small-fork-test", + gitURL: chartsSmallForkURL, + insecureSkipTLS: false, + caBundle: []byte{}, + branch: mainBranch, + expectedCommit: "226d544def39de56db210e96d2b0b535badf9bdd", + expectedError: nil, }, - // NOTE(manno): cannot test the other cases without poluting the filesystem } + for _, tc := range testCases { - actual := gitDir(tc.namespace, tc.name, tc.gitURL) - assert.Equalf(tc.expected, actual, "testcase: %v", tc) + t.Run(tc.name, func(t *testing.T) { + commit, err := Head(tc.secret, tc.namespace, tc.name, tc.gitURL, tc.branch, tc.insecureSkipTLS, tc.caBundle) + // Check the error + if tc.expectedError == nil && tc.expectedError != err { + t.Errorf("Expected error: %v |But got: %v", tc.expectedError, err) + } + // Only testing error in some cases + if err != nil { + assert.EqualError(t, tc.expectedError, err.Error()) + } + + assert.Equal(t, len(commit), len(tc.expectedCommit)) + }) + } +} + +func Test_Update(t *testing.T) { + testCases := []struct { + test string + secret *corev1.Secret + namespace string + name string + gitURL string + insecureSkipTLS bool + caBundle []byte + branch string + systemCatalogMode string + expectedCommit string + expectedError error + }{ + { + test: "#1 TestCase: Success ", + secret: nil, + namespace: "cattle-test", + name: "small-fork-test", + gitURL: chartsSmallForkURL, + insecureSkipTLS: false, + caBundle: []byte{}, + branch: lastBranch, + systemCatalogMode: "", + expectedCommit: "226d544def39de56db210e96d2b0b535badf9bdd", + expectedError: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + commit, err := Update(tc.secret, tc.namespace, tc.name, tc.gitURL, tc.branch, tc.insecureSkipTLS, tc.caBundle) + // Check the error + if tc.expectedError == nil && tc.expectedError != err { + t.Errorf("Expected error: %v |But got: %v", tc.expectedError, err) + } + + // Only testing error in some cases + if err != nil { + assert.EqualError(t, tc.expectedError, err.Error()) + } + + assert.Equal(t, len(commit), len(tc.expectedCommit)) + }) } } diff --git a/pkg/catalogv2/git/git.go b/pkg/catalogv2/git/git.go index a5fec325570..8daeb7c4eec 100644 --- a/pkg/catalogv2/git/git.go +++ b/pkg/catalogv2/git/git.go @@ -1,13 +1,11 @@ package git import ( - "bufio" "bytes" "crypto/tls" "crypto/x509" "fmt" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -33,22 +31,6 @@ type Options struct { Headers map[string]string } -func newGit(directory, url string, opts *Options) (*git, error) { - if opts == nil { - opts = &Options{} - } - - g := &git{ - URL: url, - Directory: directory, - caBundle: opts.CABundle, - insecureTLSVerify: opts.InsecureTLSVerify, - secret: opts.Credential, - headers: opts.Headers, - } - return g, g.setCredential(opts.Credential) -} - type git struct { URL string Directory string @@ -61,84 +43,101 @@ type git struct { knownHosts []byte } -// LsRemote runs ls-remote on git repo and returns the HEAD commit SHA -func (g *git) LsRemote(branch string, commit string) (string, error) { - if changed, err := g.remoteSHAChanged(branch, commit); err != nil || !changed { - return commit, err - } - - output := &bytes.Buffer{} - if err := g.gitCmd(output, "ls-remote", "--", g.URL, formatRefForBranch(branch)); err != nil { - return "", err +func newGit(directory, url string, opts *Options) (*git, error) { + if opts == nil { + opts = &Options{} } - var lines []string - s := bufio.NewScanner(output) - for s.Scan() { - lines = append(lines, s.Text()) + g := &git{ + URL: url, + Directory: directory, + caBundle: opts.CABundle, + insecureTLSVerify: opts.InsecureTLSVerify, + secret: opts.Credential, + headers: opts.Headers, } - - return firstField(lines, fmt.Sprintf("no commit for branch: %s", branch)) + return g, g.setCredential(opts.Credential) } -// Head runs git clone on directory(if not exist), reset dirty content and return the HEAD commit -func (g *git) Head(branch string) (string, error) { - if err := g.clone(branch); err != nil { - return "", err - } - - if err := g.reset("HEAD"); err != nil { - return "", err +func (g *git) setCredential(cred *corev1.Secret) error { + if cred == nil { + return nil } - return g.currentCommit() -} + switch cred.Type { + case corev1.SecretTypeBasicAuth: + username, password := cred.Data[corev1.BasicAuthUsernameKey], cred.Data[corev1.BasicAuthPasswordKey] + if len(password) == 0 && len(username) == 0 { + return nil + } -// Clone runs git clone with depth 1 -func (g *git) Clone(branch string) error { - if branch == "" { - return g.git("clone", "--depth=1", "-n", "--", g.URL, g.Directory) + u, err := url.Parse(g.URL) + if err != nil { + return err + } + u.User = url.User(string(username)) + g.URL = u.String() + g.password = string(password) + case corev1.SecretTypeSSHAuth: + key, err := ssh.ParseRawPrivateKey(cred.Data[corev1.SSHAuthPrivateKey]) + if err != nil { + return err + } + sshAgent := agent.NewKeyring() + err = sshAgent.Add(agent.AddedKey{ + PrivateKey: key, + }) + if err != nil { + return err + } + g.knownHosts = cred.Data["known_hosts"] + g.agent = &sshAgent } - return g.git("clone", "--depth=1", "-n", "--branch="+branch, "--", g.URL, g.Directory) -} -// Update updates git repo if remote sha has changed -func (g *git) Update(branch string) (string, error) { - if err := g.clone(branch); err != nil { - return "", nil - } + return nil +} - if err := g.reset("HEAD"); err != nil { - return "", err +func (g *git) injectAgent(cmd *exec.Cmd) (io.Closer, error) { + r, err := randomtoken.Generate() + if err != nil { + return nil, err } - commit, err := g.currentCommit() + tmpDir, err := os.MkdirTemp("", "ssh-agent") if err != nil { - return commit, err + return nil, err } - if changed, err := g.remoteSHAChanged(branch, commit); err != nil || !changed { - return commit, err + addr := &net.UnixAddr{ + Name: filepath.Join(tmpDir, r), + Net: "unix", } - if err := g.fetchAndReset(branch); err != nil { - return "", err + l, err := net.ListenUnix(addr.Net, addr) + if err != nil { + return nil, err } - return g.currentCommit() -} - -// Ensure runs git clone, clean DIRTY contents and fetch the latest commit -func (g *git) Ensure(commit string) error { - if err := g.clone(""); err != nil { - return err - } + cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+addr.Name) - if err := g.reset(commit); err == nil { - return nil - } + go func() { + defer os.RemoveAll(tmpDir) + defer l.Close() + for { + conn, err := l.Accept() + if err != nil { + if !k8snet.IsProbableEOF(err) { + logrus.Errorf("failed to accept ssh-agent client connection: %v", err) + } + return + } + if err := agent.ServeAgent(*g.agent, conn); err != nil && err != io.EOF { + logrus.Errorf("failed to handle ssh-agent client connection: %v", err) + } + } + }() - return g.fetchAndReset(commit) + return l, nil } func (g *git) httpClientWithCreds() (*http.Client, error) { @@ -194,6 +193,42 @@ func (g *git) httpClientWithCreds() (*http.Client, error) { return client, nil } +// Clone runs git clone with depth 1 +func (g *git) Clone(branch string) error { + if branch == "" { + return g.git("clone", "--depth=1", "-n", "--", g.URL, g.Directory) + } + return g.git("clone", "--depth=1", "-n", "--branch="+branch, "--", g.URL, g.Directory) +} + +func (g *git) clone(branch string) error { + gitDir := filepath.Join(g.Directory, ".git") + if dir, err := os.Stat(gitDir); err == nil && dir.IsDir() { + return nil + } + + if err := os.RemoveAll(g.Directory); err != nil { + return fmt.Errorf("failed to remove directory %s: %v", g.Directory, err) + } + + return g.Clone(branch) +} + +func (g *git) fetchAndReset(rev string) error { + if err := g.git("-C", g.Directory, "fetch", "origin", "--", rev); err != nil { + return err + } + return g.reset("FETCH_HEAD") +} + +func (g *git) reset(rev string) error { + return g.git("-C", g.Directory, "reset", "--hard", rev) +} + +func (g *git) currentCommit() (string, error) { + return g.gitOutput("-C", g.Directory, "rev-parse", "HEAD") +} + func (g *git) remoteSHAChanged(branch, sha string) (bool, error) { formattedURL := formatGitURL(g.URL, branch) if formattedURL == "" { @@ -251,71 +286,6 @@ func (g *git) gitOutput(args ...string) (string, error) { return strings.TrimSpace(output.String()), err } -func (g *git) setCredential(cred *corev1.Secret) error { - if cred == nil { - return nil - } - - if cred.Type == corev1.SecretTypeBasicAuth { - username, password := cred.Data[corev1.BasicAuthUsernameKey], cred.Data[corev1.BasicAuthPasswordKey] - if len(password) == 0 && len(username) == 0 { - return nil - } - - u, err := url.Parse(g.URL) - if err != nil { - return err - } - u.User = url.User(string(username)) - g.URL = u.String() - g.password = string(password) - } else if cred.Type == corev1.SecretTypeSSHAuth { - key, err := ssh.ParseRawPrivateKey(cred.Data[corev1.SSHAuthPrivateKey]) - if err != nil { - return err - } - sshAgent := agent.NewKeyring() - err = sshAgent.Add(agent.AddedKey{ - PrivateKey: key, - }) - if err != nil { - return err - } - g.knownHosts = cred.Data["known_hosts"] - g.agent = &sshAgent - } - - return nil -} - -func (g *git) clone(branch string) error { - gitDir := filepath.Join(g.Directory, ".git") - if dir, err := os.Stat(gitDir); err == nil && dir.IsDir() { - return nil - } - - if err := os.RemoveAll(g.Directory); err != nil { - return fmt.Errorf("failed to remove directory %s: %v", g.Directory, err) - } - - return g.Clone(branch) -} - -func (g *git) fetchAndReset(rev string) error { - if err := g.git("-C", g.Directory, "fetch", "origin", "--", rev); err != nil { - return err - } - return g.reset("FETCH_HEAD") -} - -func (g *git) reset(rev string) error { - return g.git("-C", g.Directory, "reset", "--hard", rev) -} - -func (g *git) currentCommit() (string, error) { - return g.gitOutput("-C", g.Directory, "rev-parse", "HEAD") -} - func (g *git) gitCmd(output io.Writer, args ...string) error { kv := fmt.Sprintf("credential.helper=%s", `/bin/sh -c 'echo "password=$GIT_PASSWORD"'`) cmd := exec.Command("git", append([]string{"-c", kv}, args...)...) @@ -333,7 +303,7 @@ func (g *git) gitCmd(output io.Writer, args ...string) error { } if len(g.knownHosts) != 0 { - f, err := ioutil.TempFile("", "known_hosts") + f, err := os.CreateTemp("", "known_hosts") if err != nil { return err } @@ -358,7 +328,7 @@ func (g *git) gitCmd(output io.Writer, args ...string) error { } if len(g.caBundle) > 0 { - f, err := ioutil.TempFile("", "ca-pem-") + f, err := os.CreateTemp("", "ca-pem-") if err != nil { return err } @@ -380,101 +350,3 @@ func (g *git) gitCmd(output io.Writer, args ...string) error { } return nil } - -func (g *git) injectAgent(cmd *exec.Cmd) (io.Closer, error) { - r, err := randomtoken.Generate() - if err != nil { - return nil, err - } - - tmpDir, err := ioutil.TempDir("", "ssh-agent") - if err != nil { - return nil, err - } - - addr := &net.UnixAddr{ - Name: filepath.Join(tmpDir, r), - Net: "unix", - } - - l, err := net.ListenUnix(addr.Net, addr) - if err != nil { - return nil, err - } - - cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+addr.Name) - - go func() { - defer os.RemoveAll(tmpDir) - defer l.Close() - for { - conn, err := l.Accept() - if err != nil { - if !k8snet.IsProbableEOF(err) { - logrus.Errorf("failed to accept ssh-agent client connection: %v", err) - } - return - } - if err := agent.ServeAgent(*g.agent, conn); err != nil && err != io.EOF { - logrus.Errorf("failed to handle ssh-agent client connection: %v", err) - } - } - }() - - return l, nil -} - -func formatGitURL(endpoint, branch string) string { - u, err := url.Parse(endpoint) - if err != nil { - return "" - } - - pathParts := strings.Split(u.Path, "/") - switch u.Hostname() { - case "github.com": - if len(pathParts) >= 3 { - org := pathParts[1] - repo := strings.TrimSuffix(pathParts[2], ".git") - return fmt.Sprintf("https://api.github.com/repos/%s/%s/commits/%s", org, repo, branch) - } - case "git.rancher.io": - repo := strings.TrimSuffix(pathParts[1], ".git") - u.Path = fmt.Sprintf("/repos/%s/commits/%s", repo, branch) - return u.String() - } - - return "" -} - -func firstField(lines []string, errText string) (string, error) { - if len(lines) == 0 { - return "", errors.New(errText) - } - - fields := strings.Fields(lines[0]) - if len(fields) == 0 { - return "", errors.New(errText) - } - - if len(fields[0]) == 0 { - return "", errors.New(errText) - } - - return fields[0], nil -} - -func formatRefForBranch(branch string) string { - return fmt.Sprintf("refs/heads/%s", branch) -} - -type basicRoundTripper struct { - username string - password string - next http.RoundTripper -} - -func (b *basicRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { - request.SetBasicAuth(b.username, b.password) - return b.next.RoundTrip(request) -} diff --git a/pkg/catalogv2/git/utils.go b/pkg/catalogv2/git/utils.go new file mode 100644 index 00000000000..6c164afd372 --- /dev/null +++ b/pkg/catalogv2/git/utils.go @@ -0,0 +1,148 @@ +package git + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/pem" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/pkg/errors" +) + +const ( + stateDir = "management-state/git-repo" + staticDir = "/var/lib/rancher-data/local-catalogs/v2" + localDir = "../rancher-data/local-catalogs/v2" // identical to helm.InternalCatalog +) + +func gitDir(namespace, name, gitURL string) string { + staticDir := filepath.Join(staticDir, namespace, name, hash(gitURL)) + if s, err := os.Stat(staticDir); err == nil && s.IsDir() { + return staticDir + } + localDir := filepath.Join(localDir, namespace, name, hash(gitURL)) + if s, err := os.Stat(localDir); err == nil && s.IsDir() { + return localDir + } + return filepath.Join(stateDir, namespace, name, hash(gitURL)) +} + +func isBundled(git *git) bool { + return strings.HasPrefix(git.Directory, staticDir) || strings.HasPrefix(git.Directory, localDir) +} + +// isGitSSH checks if the URL is in the SSH URL format using regular expressions. +// [anything]@[anything]:[anything] +// ssh://@:// +func isGitSSH(gitURL string) bool { + pattern1 := `^[^:/]+@[^:]+:[a-zA-Z]+/[^/]+$` + pattern2 := `^ssh://[^@]+@[^:]+:\d+/.+$` + + // Check if the input matches either of the two patterns. + valid, err := regexp.MatchString(pattern1, gitURL) + if err != nil { + return false + } + if valid { + return true + } + valid, err = regexp.MatchString(pattern2, gitURL) + if err != nil { + return false + } + + return valid +} + +// validateURL will validate if the provided URL is in one of the expected patterns +// for the supported protocols http(s) or ssh. +// - if Valid: returns nil +// - if Invalid: returns an error +func validateURL(gitURL string) error { + valid := isGitSSH(gitURL) + if valid { + return nil + } + // not ssh; validate http(s) + u, err := url.Parse(gitURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") { + return fmt.Errorf("invalid git URL: %s", gitURL) + } + + return nil +} + +func hash(gitURL string) string { + b := sha256.Sum256([]byte(gitURL)) + return hex.EncodeToString(b[:]) +} + +// convertDERToPEM converts a src DER certificate into PEM with line breaks, header, and footer. +func convertDERToPEM(src []byte) []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Headers: map[string]string{}, + Bytes: src, + }) +} + +func formatGitURL(endpoint, branch string) string { + u, err := url.Parse(endpoint) + if err != nil { + return "" + } + + pathParts := strings.Split(u.Path, "/") + switch u.Hostname() { + case "github.com": + if len(pathParts) >= 3 { + org := pathParts[1] + repo := strings.TrimSuffix(pathParts[2], ".git") + return fmt.Sprintf("https://api.github.com/repos/%s/%s/commits/%s", org, repo, branch) + } + case "git.rancher.io": + repo := strings.TrimSuffix(pathParts[1], ".git") + u.Path = fmt.Sprintf("/repos/%s/commits/%s", repo, branch) + return u.String() + } + + return "" +} + +func firstField(lines []string, errText string) (string, error) { + if len(lines) == 0 { + return "", errors.New(errText) + } + + fields := strings.Fields(lines[0]) + if len(fields) == 0 { + return "", errors.New(errText) + } + + if len(fields[0]) == 0 { + return "", errors.New(errText) + } + + return fields[0], nil +} + +func formatRefForBranch(branch string) string { + return fmt.Sprintf("refs/heads/%s", branch) +} + +type basicRoundTripper struct { + username string + password string + next http.RoundTripper +} + +func (b *basicRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + request.SetBasicAuth(b.username, b.password) + return b.next.RoundTrip(request) +} diff --git a/pkg/catalogv2/git/utils_test.go b/pkg/catalogv2/git/utils_test.go new file mode 100644 index 00000000000..b4341257218 --- /dev/null +++ b/pkg/catalogv2/git/utils_test.go @@ -0,0 +1,56 @@ +package git + +import ( + "testing" + + assertlib "github.com/stretchr/testify/assert" +) + +func Test_isGitSSH(t *testing.T) { + testCases := []struct { + gitURL string + expected bool + }{ + // True cases + {"customusername@github.com:user/repo.git", true}, + {"customusername@gitlab.com:user/repo.git", true}, + {"customusername@gitlab.com:user/repo", true}, + {"customusername@gitlab.com:user/repo-with-dashes.git", true}, + {"git@github.com:user/repo.git", true}, + {"git@gitlab.com:user/repo-with-dashes.git", true}, + {"git@gitlab.com:user/repo", true}, + {"ssh://git@git.domain.com:443/repo", true}, + // False cases + {"https://github.com/user/repo.git", false}, + {"http://gitlab.com/user/repo.git", false}, + {"http://gitlab.com/user/repo", false}, + {"http://gitlab.com", false}, + {"git@gitlab.com", false}, + {"ftp://git@gitlab.com:22/repo", false}, + } + assert := assertlib.New(t) + for _, tc := range testCases { + actual := isGitSSH(tc.gitURL) + assert.Equalf(tc.expected, actual, "testcase: %v", tc) + } +} + +func Test_gitDir(t *testing.T) { + assert := assertlib.New(t) + testCases := []struct { + namespace string + name string + gitURL string + expected string + }{ + { + "namespace", "name", "https://git.rancher.io/charts", + "management-state/git-repo/namespace/name/4b40cac650031b74776e87c1a726b0484d0877c3ec137da0872547ff9b73a721", + }, + // NOTE(manno): cannot test the other cases without poluting the filesystem + } + for _, tc := range testCases { + actual := gitDir(tc.namespace, tc.name, tc.gitURL) + assert.Equalf(tc.expected, actual, "testcase: %v", tc) + } +}