Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft for dependencies cache reuse within same deploy #2769

Closed
wants to merge 22 commits into from
Closed

Draft for dependencies cache reuse within same deploy #2769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
94c4680
Exampel code for dependencies cache reuse within same deploy
Andrioden Dec 4, 2023
8e11a8b
fix: pull for cached dependencies
Andrioden Dec 4, 2023
274ca63
More and edited comments
Andrioden Jan 29, 2024
99c212a
Moved git Pull out of Clone
Andrioden Jan 29, 2024
ab4ad96
fix: nil pointer in ensure namespace
FabianKramm Dec 1, 2023
02f6745
tests: add regression tests for nil pointer on invalid kubeconfig
89luca89 Dec 1, 2023
d43987c
fix: try to resolve test flaking
89luca89 Dec 1, 2023
07c01cd
fix: disable upx for devspace-helper on linux/arm64
xvzf Dec 12, 2023
cfa63da
fix: fix typo
Dec 20, 2023
c3d0bdf
fix: use string hash instead of whole name
FabianKramm Dec 21, 2023
6826302
added video tutorial series playlist to introduction to DevSpace
mpetason Jan 2, 2024
7f111cc
test: add flake attempts and fail-fast for e2e tests
lizardruss Jan 2, 2024
6f7ad36
chore(deps): bump follow-redirects from 1.15.2 to 1.15.4 in /docs
dependabot[bot] Jan 10, 2024
50f9a3b
chore(deps): bump follow-redirects from 1.14.8 to 1.15.4 in /ui
dependabot[bot] Jan 9, 2024
04ae491
Bugfix: also run persistPaths() for containers
Jan 6, 2024
c267eac
bugfix: make a DelayedContainerStarter per container instead of per p…
Jan 6, 2024
a563de1
Keep linter happy
Jan 9, 2024
ac12588
test: update SSH tests to wait for start_dev before canceling the com…
lizardruss Jan 12, 2024
f53f338
fix: use string hash for volumeName
Jan 22, 2024
528181a
restart helper: give container process some time to shutdown
Jan 10, 2024
f52d3cb
Merge branch 'devspace-sh:main' into main
Andrioden Jan 29, 2024
ec81def
Merge branch 'main' into dep-skip
Andrioden Jan 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion pkg/devspace/dependency/resolver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type ResolverInterface interface {
// Resolver implements the resolver interface
type resolver struct {
DependencyGraph *graph.Graph
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you may be able to use the DependencyGraph to avoid re-downloading / re-pulling dependencies.

DownloadedIds []string

BaseCache localcache.Cache
BaseConfig *latest.Config
Expand Down Expand Up @@ -130,7 +131,7 @@ func (r *resolver) resolveRecursive(ctx devspacecontext.Context, basePath, paren
continue
}

dependencyConfigPath, err := util.DownloadDependency(ctx.Context(), basePath, dependencyConfig.Source, ctx.Log())
dependencyConfigPath, err := util.DownloadDependency(ctx.Context(), basePath, dependencyConfig.Source, ctx.Log(), r.DownloadedIds)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You might be able to skip this by doing a check similar to this line.

if err != nil {
return err
}
Expand Down
77 changes: 46 additions & 31 deletions pkg/devspace/dependency/util/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,59 +78,63 @@ func switchURLType(gitPath string) string {
return newGitURL
}

func DownloadDependency(ctx context.Context, workingDirectory string, source *latest.SourceConfig, log log.Logger) (configPath string, err error) {
func DownloadDependency(ctx context.Context, workingDirectory string, source *latest.SourceConfig, log log.Logger, downloadedIds string[]) (configPath string, err error) {
downloadMutex.Lock()
defer downloadMutex.Unlock()

ID, err := GetDependencyID(source)
if contains(downloadedIds, ID) {
log.Debugf("Using depdendencies download cache for ", ID)
return "", nil
}

if err != nil {
return "", err
}

// Resolve source
var localPath string

// Resolve git source
if source.Git != "" {
fmt.Println("--- DownloadDependency", source.Git, source.Branch)
gitPath := strings.TrimSpace(source.Git)

_ = os.MkdirAll(DependencyFolderPath, 0755)
localPath = filepath.Join(DependencyFolderPath, ID)

// Check if dependency exists
// Check if dependency are cached locally
_, statErr := os.Stat(localPath)

// Update dependency
if !source.DisablePull && statErr != nil {
repo, err := git.NewGitCLIRepository(ctx, localPath)
if err != nil {
if statErr == nil {
log.Warnf("Error creating git cli: %v", err)
return getDependencyConfigPath(localPath, source)
}

return "", err
// Verify git cli works
repo, err := git.NewGitCLIRepository(ctx, localPath)
if err != nil {
if statErr == nil {
log.Warnf("Error creating git cli: %v", err)
return getDependencyConfigPath(localPath, source)
}
return "", err
}

err = repo.Clone(ctx, git.CloneOptions{
URL: gitPath,
Tag: source.Tag,
Branch: source.Branch,
Commit: source.Revision,
Args: source.CloneArgs,
DisableShallow: source.DisableShallow,
})
// Create git clone options
var gitCloneOptions = git.CloneOptions{
URL: gitPath,
Tag: source.Tag,
Branch: source.Branch,
Commit: source.Revision,
Args: source.CloneArgs,
DisableShallow: source.DisableShallow,
}

// Git clone
if statErr != nil {
err = repo.Clone(ctx, gitCloneOptions)

if err != nil {
log.Warn("Error cloning repo: ", err)
newGitURL := switchURLType(gitPath)
log.Infof("Switching URL from %s to %s and will try cloning again", gitPath, newGitURL)
err = repo.Clone(ctx, git.CloneOptions{
URL: newGitURL,
Tag: source.Tag,
Branch: source.Branch,
Commit: source.Revision,
Args: source.CloneArgs,
DisableShallow: source.DisableShallow,
})

gitCloneOptions.URL = switchURLType(gitPath)
log.Infof("Switching URL from %s to %s and will try cloning again", gitPath, gitCloneOptions.URL)
err = repo.Clone(ctx, gitCloneOptions)

if err != nil {
log.Warn("Failed to clone repo with both HTTPS and SSH URL. Please make sure if your git login or ssh setup is correct.")
Expand All @@ -142,8 +146,19 @@ func DownloadDependency(ctx context.Context, workingDirectory string, source *la
return "", errors.Wrap(err, "clone repository")
}
}

downloadedIds.append(??)

log.Debugf("Cloned %s", gitPath)
}

// Git pull
if !source.DisablePull && source.Revision == "" {
repo.Pull(ctx)
log.Debugf("Pulled %s", gitPath)
}

// Resolve local source
} else if source.Path != "" {
if isURL(source.Path) {
localPath = filepath.Join(DependencyFolderPath, ID)
Expand Down
2 changes: 2 additions & 0 deletions pkg/devspace/plugin/installer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ func (i *installer) DownloadBinary(metadataPath, version, binaryPath, outFile st
if err != nil {
return err
}
repo.Pull(context.Background())

return copy.Copy(filepath.Join(tempDir, binaryPath), outFile)
}
Expand Down Expand Up @@ -132,6 +133,7 @@ func (i *installer) DownloadMetadata(path, version string) (*Metadata, error) {
if err != nil {
return nil, err
}
repo.Pull(context.Background())

out, err := os.ReadFile(filepath.Join(tempDir, pluginYaml))
if err != nil {
Expand Down
21 changes: 11 additions & 10 deletions pkg/util/git/git.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func (gr *GitCLIRepository) Clone(ctx context.Context, options CloneOptions) err
args = append(args, "--branch", options.Tag)
}

// do a shallow clone by default
// Do a shallow clone by default
if options.Commit == "" && !options.DisableShallow {
args = append(args, "--depth", "1")
}
Expand All @@ -76,7 +76,7 @@ func (gr *GitCLIRepository) Clone(ctx context.Context, options CloneOptions) err
return errors.Errorf("Error running 'git %s': %v -> %s", strings.Join(args, " "), err, string(out))
}

// checkout the commit if necessary
// Checkout the commit if necessary
if options.Commit != "" {
out, err := command.CombinedOutput(ctx, gr.LocalPath, expand.ListEnviron(os.Environ()...), "git", "-C", gr.LocalPath, "checkout", options.Commit)
if err != nil {
Expand All @@ -87,13 +87,14 @@ func (gr *GitCLIRepository) Clone(ctx context.Context, options CloneOptions) err
return nil
}

// make sure the repo is up-to-date
if options.Commit == "" {
out, err := command.CombinedOutput(ctx, gr.LocalPath, expand.ListEnviron(os.Environ()...), "git", "-C", gr.LocalPath, "pull")
if err != nil {
return errors.Errorf("Error running 'git pull %s': %v -> %s", options.URL, err, string(out))
}
}

return nil
}

func (gr *GitCLIRepository) Pull(ctx context.Context) error {
out, err := command.CombinedOutput(ctx, gr.LocalPath, expand.ListEnviron(os.Environ()...), "git", "-C", gr.LocalPath, "pull")
if err != nil {
return errors.Errorf("Error running 'git pull %s': %v -> %s", gr.LocalPath, err, string(out))
} else {
return nil
}
}