From 836726740fd15f98d7057200499d1bb1bdbcd44f Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Wed, 22 Nov 2023 14:42:44 +0100 Subject: [PATCH 01/17] feat: fix bug Signed-off-by: Kasper J. Hermansen --- pkg/git/git.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/git/git.go b/pkg/git/git.go index 7c52b0f..5346a12 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -164,7 +164,7 @@ func GetGitPlan( panic(fmt.Sprintf("Unknown protocol '%s'", parsedGitPlan.Protocol)) } - uii.Infoln("Cloning plan %s", cloneArg) + uii.Errorln("Cloning plan %s", cloneArg) err = gitCmd(fmt.Sprintf("clone %v --branch %v plan", cloneArg, parsedGitPlan.Head), localShuttleDirectoryPath, uii) if err != nil { return "", err From 0649eaa7b10ca0f31a5bcf15bae8d4d87b3f59f2 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Wed, 22 Nov 2023 15:55:38 +0100 Subject: [PATCH 02/17] fix: set all output to stderr from ui Signed-off-by: Kasper J. Hermansen --- pkg/ui/ui.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ui/ui.go b/pkg/ui/ui.go index 11ea996..2140cb3 100644 --- a/pkg/ui/ui.go +++ b/pkg/ui/ui.go @@ -21,7 +21,7 @@ func Create(out, err io.Writer) *UI { EffectiveLevel: LevelInfo, DefaultLevel: LevelInfo, UserLevelSet: false, - Out: out, + Out: err, Err: err, } } From 33e0771170dc6aa40eb64c8357dcb4ad6670677b Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Mon, 4 Dec 2023 16:04:04 +0100 Subject: [PATCH 03/17] feat: with stderr instead Signed-off-by: Kasper J. Hermansen --- pkg/git/git.go | 2 +- pkg/ui/ui.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/git/git.go b/pkg/git/git.go index 5346a12..7c52b0f 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -164,7 +164,7 @@ func GetGitPlan( panic(fmt.Sprintf("Unknown protocol '%s'", parsedGitPlan.Protocol)) } - uii.Errorln("Cloning plan %s", cloneArg) + uii.Infoln("Cloning plan %s", cloneArg) err = gitCmd(fmt.Sprintf("clone %v --branch %v plan", cloneArg, parsedGitPlan.Head), localShuttleDirectoryPath, uii) if err != nil { return "", err diff --git a/pkg/ui/ui.go b/pkg/ui/ui.go index 2140cb3..11ea996 100644 --- a/pkg/ui/ui.go +++ b/pkg/ui/ui.go @@ -21,7 +21,7 @@ func Create(out, err io.Writer) *UI { EffectiveLevel: LevelInfo, DefaultLevel: LevelInfo, UserLevelSet: false, - Out: err, + Out: out, Err: err, } } From f5f32f2519c4e0c6b6fce60f94081b9d99ac8612 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Tue, 5 Dec 2023 12:18:20 +0100 Subject: [PATCH 04/17] feat: with local dir Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch.go | 130 ++++++++++++++++++ pkg/executors/golang/codegen/patch_test.go | 32 +++++ .../codegen/testdata/patch/root_module/go.mod | 3 + pkg/executors/golang/compile/compile.go | 11 +- 4 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 pkg/executors/golang/codegen/patch.go create mode 100644 pkg/executors/golang/codegen/patch_test.go create mode 100644 pkg/executors/golang/codegen/testdata/patch/root_module/go.mod diff --git a/pkg/executors/golang/codegen/patch.go b/pkg/executors/golang/codegen/patch.go new file mode 100644 index 0000000..b056d0f --- /dev/null +++ b/pkg/executors/golang/codegen/patch.go @@ -0,0 +1,130 @@ +package codegen + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "os" + "path" + "strings" +) + +type writeFileFunc = func(name string, contents []byte, permissions fs.FileMode) error + +func PatchGoMod(rootDir string, shuttleLocalDir string) error { + fmt.Printf("dirs: %s %s\n", rootDir, shuttleLocalDir) + + return patchGoMod( + rootDir, + shuttleLocalDir, + os.WriteFile, + ) +} + +func patchGoMod(rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) error { + packages := make(map[string]string, 0) + + if rootModExists(rootDir) { + moduleName, modulePath, err := GetRootModule(rootDir) + if err != nil { + return fmt.Errorf("failed to parse go.mod in root of project: %w", err) + } + + packages[moduleName] = modulePath + } + + if err := patchPackagesUsed(rootDir, shuttleLocalDir, packages, writeFileFunc); err != nil { + return err + } + + return nil + +} + +func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[string]string, writeFileFunc writeFileFunc) error { + actionsModFilePath := path.Join(shuttleLocalDir, "tmp/go.mod") + segmentsToRoot := strings.Count(path.Join(strings.TrimPrefix(rootDir, shuttleLocalDir), "tmp/go.mod"), "/") + actionsModFileContents, err := os.ReadFile(actionsModFilePath) + if err != nil { + return err + } + actionsModFilePermissions, err := os.Stat(actionsModFilePath) + if err != nil { + return err + } + + actionsModFile := string(actionsModFileContents) + actionsFileWriter := bytes.NewBufferString(actionsModFile) + + actionsModFileContainsModule := func(moduleName string) bool { + return strings.Contains(actionsModFile, moduleName) + } + + for moduleName, modulePath := range packages { + if !actionsModFileContainsModule(moduleName) { + continue + } + + relativeToActionsModulePath := path.Join(strings.Repeat("../", segmentsToRoot), modulePath) + + _, err := fmt.Fprintf(actionsFileWriter, "\nreplace %s => %s\n", moduleName, relativeToActionsModulePath) + if err != nil { + return err + } + } + + err = writeFileFunc(actionsModFilePath, actionsFileWriter.Bytes(), actionsModFilePermissions.Mode()) + if err != nil { + return err + } + + return nil +} + +func GetRootModule(rootDir string) (moduleName string, modulePath string, err error) { + modFile, err := os.ReadFile(path.Join(rootDir, "go.mod")) + if err != nil { + return "", "", err + } + + modFileContent := string(modFile) + lines := strings.Split(modFileContent, "\n") + if len(lines) == 0 { + return "", "", errors.New("go mod is empty") + } + + for _, line := range lines { + modFileLine := strings.TrimSpace(line) + if strings.HasPrefix(modFileLine, "module") { + sections := strings.Split(modFileLine, " ") + if len(sections) < 2 { + return "", "", fmt.Errorf("invalid module line: %s", modFileLine) + } + + moduleName := sections[1] + + return moduleName, ".", nil + } + } + + return "", "", errors.New("failed to find a valid go.mod file") +} + +func rootModExists(rootDir string) bool { + goMod := path.Join(rootDir, "go.mod") + if _, err := os.Stat(goMod); errors.Is(err, os.ErrNotExist) { + return false + } + + return true +} + +func rootWorkspaceExists() bool { + goWork := "go.work" + if _, err := os.Stat(goWork); errors.Is(err, os.ErrNotExist) { + return false + } + + return true +} diff --git a/pkg/executors/golang/codegen/patch_test.go b/pkg/executors/golang/codegen/patch_test.go new file mode 100644 index 0000000..3c93e83 --- /dev/null +++ b/pkg/executors/golang/codegen/patch_test.go @@ -0,0 +1,32 @@ +package codegen + +import ( + "io/fs" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPatchGoMod(t *testing.T) { + t.Parallel() + + t.Run("finds root module adds to actions plan", func(t *testing.T) { + err := patchGoMod("testdata/patch/root_module/", ".shuttle/actions", func(name string, contents []byte, permissions fs.FileMode) error { + assert.Equal(t, "testdata/patch/root_module/.shuttle/actions/tmp/go.mod", name) + assert.Equal(t, `module actions + +require ( + root_module +) + +go 1.21.4 + +replace root_module => ../../.. +`, string(contents)) + + return nil + }) + require.NoError(t, err) + }) +} diff --git a/pkg/executors/golang/codegen/testdata/patch/root_module/go.mod b/pkg/executors/golang/codegen/testdata/patch/root_module/go.mod new file mode 100644 index 0000000..0db8771 --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/root_module/go.mod @@ -0,0 +1,3 @@ +module root_module + +go 1.21.4 diff --git a/pkg/executors/golang/compile/compile.go b/pkg/executors/golang/compile/compile.go index 3d38513..5f2bd72 100644 --- a/pkg/executors/golang/compile/compile.go +++ b/pkg/executors/golang/compile/compile.go @@ -113,14 +113,19 @@ func compile(ctx context.Context, ui *ui.UI, actions *discover.ActionsDiscovered var binarypath string + if err := codegen.PatchGoMod(actions.ParentDir, shuttlelocaldir); err != nil { + return "", fmt.Errorf("failed to patch generated go.mod: %w", err) + } + if goInstalled() { + if err = codegen.ModTidy(ctx, ui, shuttlelocaldir); err != nil { + return "", fmt.Errorf("go mod tidy failed: %w", err) + } + if err = codegen.Format(ctx, ui, shuttlelocaldir); err != nil { return "", fmt.Errorf("go fmt failed: %w", err) } - if err = codegen.ModTidy(ctx, ui, shuttlelocaldir); err != nil { - return "", fmt.Errorf("go mod tidy failed: %w", err) - } binarypath, err = codegen.CompileBinary(ctx, ui, shuttlelocaldir) if err != nil { return "", fmt.Errorf("go build failed: %w", err) From 02471058d2d6750b7828de8c4496ffc90cd8bb24 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Tue, 5 Dec 2023 12:19:38 +0100 Subject: [PATCH 05/17] feat: with go mod Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/executors/golang/codegen/patch_test.go b/pkg/executors/golang/codegen/patch_test.go index 3c93e83..6eb2e96 100644 --- a/pkg/executors/golang/codegen/patch_test.go +++ b/pkg/executors/golang/codegen/patch_test.go @@ -12,7 +12,7 @@ func TestPatchGoMod(t *testing.T) { t.Parallel() t.Run("finds root module adds to actions plan", func(t *testing.T) { - err := patchGoMod("testdata/patch/root_module/", ".shuttle/actions", func(name string, contents []byte, permissions fs.FileMode) error { + err := patchGoMod("testdata/patch/root_module/", "testdata/patch/root_module/.shuttle/actions", func(name string, contents []byte, permissions fs.FileMode) error { assert.Equal(t, "testdata/patch/root_module/.shuttle/actions/tmp/go.mod", name) assert.Equal(t, `module actions @@ -22,7 +22,7 @@ require ( go 1.21.4 -replace root_module => ../../.. +replace root_module => ../../../.. `, string(contents)) return nil From 65e43ab3d9271b24f88516f6a4bcdcca6ef88be5 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Tue, 5 Dec 2023 12:53:44 +0100 Subject: [PATCH 06/17] feat: with root module Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch.go | 27 +++++++++++++++---- pkg/executors/golang/codegen/patch_test.go | 21 ++++++++++++++- .../testdata/patch/replace_existing/go.mod | 3 +++ 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 pkg/executors/golang/codegen/testdata/patch/replace_existing/go.mod diff --git a/pkg/executors/golang/codegen/patch.go b/pkg/executors/golang/codegen/patch.go index b056d0f..155f219 100644 --- a/pkg/executors/golang/codegen/patch.go +++ b/pkg/executors/golang/codegen/patch.go @@ -13,8 +13,6 @@ import ( type writeFileFunc = func(name string, contents []byte, permissions fs.FileMode) error func PatchGoMod(rootDir string, shuttleLocalDir string) error { - fmt.Printf("dirs: %s %s\n", rootDir, shuttleLocalDir) - return patchGoMod( rootDir, shuttleLocalDir, @@ -44,7 +42,7 @@ func patchGoMod(rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) er func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[string]string, writeFileFunc writeFileFunc) error { actionsModFilePath := path.Join(shuttleLocalDir, "tmp/go.mod") - segmentsToRoot := strings.Count(path.Join(strings.TrimPrefix(rootDir, shuttleLocalDir), "tmp/go.mod"), "/") + segmentsToRoot := strings.Count(path.Join(strings.TrimPrefix(shuttleLocalDir, rootDir), "tmp/go.mod"), "/") - 1 actionsModFileContents, err := os.ReadFile(actionsModFilePath) if err != nil { return err @@ -55,7 +53,7 @@ func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[stri } actionsModFile := string(actionsModFileContents) - actionsFileWriter := bytes.NewBufferString(actionsModFile) + actionsModFileLines := strings.Split(actionsModFile, "\n") actionsModFileContainsModule := func(moduleName string) bool { return strings.Contains(actionsModFile, moduleName) @@ -68,12 +66,31 @@ func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[stri relativeToActionsModulePath := path.Join(strings.Repeat("../", segmentsToRoot), modulePath) - _, err := fmt.Fprintf(actionsFileWriter, "\nreplace %s => %s\n", moduleName, relativeToActionsModulePath) + foundReplace := false + for i, line := range actionsModFileLines { + lineTrim := strings.TrimSpace(line) + + if strings.Contains(lineTrim, fmt.Sprintf("replace %s", moduleName)) { + actionsModFileLines[i] = fmt.Sprintf("replace %s => %s", moduleName, relativeToActionsModulePath) + foundReplace = true + break + } + } + + if !foundReplace { + actionsModFileLines = append( + actionsModFileLines, + fmt.Sprintf("\nreplace %s => %s", moduleName, relativeToActionsModulePath), + ) + + } + if err != nil { return err } } + actionsFileWriter := bytes.NewBufferString(strings.Join(actionsModFileLines, "\n")) err = writeFileFunc(actionsModFilePath, actionsFileWriter.Bytes(), actionsModFilePermissions.Mode()) if err != nil { return err diff --git a/pkg/executors/golang/codegen/patch_test.go b/pkg/executors/golang/codegen/patch_test.go index 6eb2e96..beab40a 100644 --- a/pkg/executors/golang/codegen/patch_test.go +++ b/pkg/executors/golang/codegen/patch_test.go @@ -22,7 +22,26 @@ require ( go 1.21.4 -replace root_module => ../../../.. + +replace root_module => ../..`, string(contents)) + + return nil + }) + require.NoError(t, err) + }) + + t.Run("finds root module replaces existing", func(t *testing.T) { + err := patchGoMod("testdata/patch/replace_existing/", "testdata/patch/replace_existing/.shuttle/actions", func(name string, contents []byte, permissions fs.FileMode) error { + assert.Equal(t, "testdata/patch/replace_existing/.shuttle/actions/tmp/go.mod", name) + assert.Equal(t, `module actions + +require ( + replace_existing v0.0.0 +) + +go 1.21.4 + +replace replace_existing => ../.. `, string(contents)) return nil diff --git a/pkg/executors/golang/codegen/testdata/patch/replace_existing/go.mod b/pkg/executors/golang/codegen/testdata/patch/replace_existing/go.mod new file mode 100644 index 0000000..4dbd3f8 --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/replace_existing/go.mod @@ -0,0 +1,3 @@ +module replace_existing + +go 1.21.4 From c662fce907544df08b3234bc105107015a78d34e Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Tue, 5 Dec 2023 13:48:05 +0100 Subject: [PATCH 07/17] feat: add workspace Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch.go | 83 ++++++++++++++++++- pkg/executors/golang/codegen/patch_test.go | 29 ++++++- .../testdata/patch/root_workspace/go.mod | 3 + .../testdata/patch/root_workspace/go.work | 8 ++ .../root_workspace/other/subpackage/go.mod | 3 + .../patch/root_workspace/subpackage/go.mod | 3 + 6 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 pkg/executors/golang/codegen/testdata/patch/root_workspace/go.mod create mode 100644 pkg/executors/golang/codegen/testdata/patch/root_workspace/go.work create mode 100644 pkg/executors/golang/codegen/testdata/patch/root_workspace/other/subpackage/go.mod create mode 100644 pkg/executors/golang/codegen/testdata/patch/root_workspace/subpackage/go.mod diff --git a/pkg/executors/golang/codegen/patch.go b/pkg/executors/golang/codegen/patch.go index 155f219..94ba959 100644 --- a/pkg/executors/golang/codegen/patch.go +++ b/pkg/executors/golang/codegen/patch.go @@ -23,7 +23,21 @@ func PatchGoMod(rootDir string, shuttleLocalDir string) error { func patchGoMod(rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) error { packages := make(map[string]string, 0) - if rootModExists(rootDir) { + if rootWorkspaceExists(rootDir) { + modules, err := GetWorkspaceModules(rootDir) + if err != nil { + return fmt.Errorf("failed to parse go.mod in root of project: %w", err) + } + + for _, module := range modules { + moduleName, modulePath, err := GetWorkspaceModule(rootDir, module) + if err != nil { + return err + } + packages[moduleName] = modulePath + } + + } else if rootModExists(rootDir) { moduleName, modulePath, err := GetRootModule(rootDir) if err != nil { return fmt.Errorf("failed to parse go.mod in root of project: %w", err) @@ -42,7 +56,9 @@ func patchGoMod(rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) er func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[string]string, writeFileFunc writeFileFunc) error { actionsModFilePath := path.Join(shuttleLocalDir, "tmp/go.mod") - segmentsToRoot := strings.Count(path.Join(strings.TrimPrefix(shuttleLocalDir, rootDir), "tmp/go.mod"), "/") - 1 + relativeActionsModFilePath := strings.TrimPrefix(path.Join(strings.TrimPrefix(shuttleLocalDir, rootDir), "tmp/go.mod"), "/") + + segmentsToRoot := strings.Count(relativeActionsModFilePath, "/") actionsModFileContents, err := os.ReadFile(actionsModFilePath) if err != nil { return err @@ -128,6 +144,65 @@ func GetRootModule(rootDir string) (moduleName string, modulePath string, err er return "", "", errors.New("failed to find a valid go.mod file") } +func GetWorkspaceModule(rootDir string, absoluteModulePath string) (moduleName string, modulePath string, err error) { + modFile, err := os.ReadFile(path.Join(rootDir, absoluteModulePath, "go.mod")) + if err != nil { + return "", "", fmt.Errorf("failed to find go.mod at: %s: %w", absoluteModulePath, err) + } + + modFileContent := string(modFile) + lines := strings.Split(modFileContent, "\n") + if len(lines) == 0 { + return "", "", errors.New("go mod is empty") + } + + for _, line := range lines { + modFileLine := strings.TrimSpace(line) + if strings.HasPrefix(modFileLine, "module") { + sections := strings.Split(modFileLine, " ") + if len(sections) < 2 { + return "", "", fmt.Errorf("invalid module line: %s", modFileLine) + } + + moduleName := sections[1] + modulePath = strings.TrimPrefix(absoluteModulePath, rootDir) + + return moduleName, modulePath, nil + } + } + + return "", "", errors.New("failed to find a valid go.mod file") +} + +func GetWorkspaceModules(rootDir string) (modules []string, err error) { + workFile, err := os.ReadFile(path.Join(rootDir, "go.work")) + if err != nil { + return nil, err + } + + workFileContent := string(workFile) + lines := strings.Split(workFileContent, "\n") + if len(lines) == 0 { + return nil, errors.New("go work is empty") + } + + modules = make([]string, 0) + for _, line := range lines { + modFileLine := strings.Trim(strings.TrimSpace(line), "\t") + if strings.HasPrefix(modFileLine, ".") && modFileLine != "./actions" { + modules = append( + modules, + strings.TrimPrefix( + strings.TrimPrefix(modFileLine, "."), + "/", + ), + ) + } + } + + return modules, nil +} + func rootModExists(rootDir string) bool { goMod := path.Join(rootDir, "go.mod") if _, err := os.Stat(goMod); errors.Is(err, os.ErrNotExist) { @@ -137,8 +212,8 @@ func rootModExists(rootDir string) bool { return true } -func rootWorkspaceExists() bool { - goWork := "go.work" +func rootWorkspaceExists(rootDir string) bool { + goWork := path.Join(rootDir, "go.work") if _, err := os.Stat(goWork); errors.Is(err, os.ErrNotExist) { return false } diff --git a/pkg/executors/golang/codegen/patch_test.go b/pkg/executors/golang/codegen/patch_test.go index beab40a..f5edd6c 100644 --- a/pkg/executors/golang/codegen/patch_test.go +++ b/pkg/executors/golang/codegen/patch_test.go @@ -23,7 +23,7 @@ require ( go 1.21.4 -replace root_module => ../..`, string(contents)) +replace root_module => ../../..`, string(contents)) return nil }) @@ -41,11 +41,36 @@ require ( go 1.21.4 -replace replace_existing => ../.. +replace replace_existing => ../../.. `, string(contents)) return nil }) require.NoError(t, err) }) + + t.Run("finds root workspace adds entries", func(t *testing.T) { + err := patchGoMod("testdata/patch/root_workspace/", "testdata/patch/root_workspace/.shuttle/actions", func(name string, contents []byte, permissions fs.FileMode) error { + assert.Equal(t, "testdata/patch/root_workspace/.shuttle/actions/tmp/go.mod", name) + assert.Equal(t, `module actions + +require ( + root_workspace v0.0.0 + subpackage v0.0.0 + othersubpackage v0.0.0 +) + +go 1.21.4 + + +replace root_workspace => ../../.. + +replace subpackage => ../../../subpackage + +replace othersubpackage => ../../../other/subpackage`, string(contents)) + + return nil + }) + require.NoError(t, err) + }) } diff --git a/pkg/executors/golang/codegen/testdata/patch/root_workspace/go.mod b/pkg/executors/golang/codegen/testdata/patch/root_workspace/go.mod new file mode 100644 index 0000000..6f27c4c --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/root_workspace/go.mod @@ -0,0 +1,3 @@ +module root_workspace + +go 1.21.4 diff --git a/pkg/executors/golang/codegen/testdata/patch/root_workspace/go.work b/pkg/executors/golang/codegen/testdata/patch/root_workspace/go.work new file mode 100644 index 0000000..d9f95d8 --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/root_workspace/go.work @@ -0,0 +1,8 @@ +go 1.21.4 + +use ( + . + ./subpackage + ./other/subpackage + ignored +) diff --git a/pkg/executors/golang/codegen/testdata/patch/root_workspace/other/subpackage/go.mod b/pkg/executors/golang/codegen/testdata/patch/root_workspace/other/subpackage/go.mod new file mode 100644 index 0000000..0e20b67 --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/root_workspace/other/subpackage/go.mod @@ -0,0 +1,3 @@ +module othersubpackage + +go 1.21.4 diff --git a/pkg/executors/golang/codegen/testdata/patch/root_workspace/subpackage/go.mod b/pkg/executors/golang/codegen/testdata/patch/root_workspace/subpackage/go.mod new file mode 100644 index 0000000..d1edf00 --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/root_workspace/subpackage/go.mod @@ -0,0 +1,3 @@ +module subpackage + +go 1.21.4 From 0c6b81a9f598bc0930173bc583948091fa8c7833 Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Thu, 7 Dec 2023 15:17:23 +0100 Subject: [PATCH 08/17] feat: add more logs (#200) Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch.go | 16 +++++++++++----- pkg/executors/golang/compile/compile.go | 6 +++++- pkg/executors/golang/compile/matcher/matcher.go | 2 ++ pkg/executors/golang/executer/prepare.go | 6 ++++++ pkg/executors/golang/executer/run.go | 1 + pkg/executors/task.go | 2 +- 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pkg/executors/golang/codegen/patch.go b/pkg/executors/golang/codegen/patch.go index 94ba959..5cbee94 100644 --- a/pkg/executors/golang/codegen/patch.go +++ b/pkg/executors/golang/codegen/patch.go @@ -8,22 +8,26 @@ import ( "os" "path" "strings" + + "github.com/lunarway/shuttle/pkg/ui" ) type writeFileFunc = func(name string, contents []byte, permissions fs.FileMode) error -func PatchGoMod(rootDir string, shuttleLocalDir string) error { +func PatchGoMod(rootDir string, shuttleLocalDir string, ui *ui.UI) error { return patchGoMod( rootDir, shuttleLocalDir, + ui, os.WriteFile, ) } -func patchGoMod(rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) error { +func patchGoMod(rootDir, shuttleLocalDir string, ui *ui.UI, writeFileFunc writeFileFunc) error { packages := make(map[string]string, 0) if rootWorkspaceExists(rootDir) { + ui.Verboseln("patching go action using workspace file") modules, err := GetWorkspaceModules(rootDir) if err != nil { return fmt.Errorf("failed to parse go.mod in root of project: %w", err) @@ -38,6 +42,7 @@ func patchGoMod(rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) er } } else if rootModExists(rootDir) { + ui.Verboseln("patching go action using mod file") moduleName, modulePath, err := GetRootModule(rootDir) if err != nil { return fmt.Errorf("failed to parse go.mod in root of project: %w", err) @@ -46,7 +51,7 @@ func patchGoMod(rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) er packages[moduleName] = modulePath } - if err := patchPackagesUsed(rootDir, shuttleLocalDir, packages, writeFileFunc); err != nil { + if err := patchPackagesUsed(rootDir, shuttleLocalDir, packages, ui, writeFileFunc); err != nil { return err } @@ -54,11 +59,11 @@ func patchGoMod(rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) er } -func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[string]string, writeFileFunc writeFileFunc) error { +func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[string]string, ui *ui.UI, writeFileFunc writeFileFunc) error { actionsModFilePath := path.Join(shuttleLocalDir, "tmp/go.mod") relativeActionsModFilePath := strings.TrimPrefix(path.Join(strings.TrimPrefix(shuttleLocalDir, rootDir), "tmp/go.mod"), "/") - segmentsToRoot := strings.Count(relativeActionsModFilePath, "/") + actionsModFileContents, err := os.ReadFile(actionsModFilePath) if err != nil { return err @@ -79,6 +84,7 @@ func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[stri if !actionsModFileContainsModule(moduleName) { continue } + ui.Verboseln("golang binary patch adding: moduleName: %s and modulePath: %s", moduleName, modulePath) relativeToActionsModulePath := path.Join(strings.Repeat("../", segmentsToRoot), modulePath) diff --git a/pkg/executors/golang/compile/compile.go b/pkg/executors/golang/compile/compile.go index 5f2bd72..9723ff2 100644 --- a/pkg/executors/golang/compile/compile.go +++ b/pkg/executors/golang/compile/compile.go @@ -48,6 +48,8 @@ func Compile(ctx context.Context, ui *ui.UI, discovered *discover.Discovered) (* binaries := &Binaries{} if discovered.Local != nil { egrp.Go(func() error { + ui.Verboseln("compiling golang actions binary for: %s", discovered.Local.DirPath) + path, err := compile(ctx, ui, discovered.Local) if err != nil { return err @@ -59,6 +61,8 @@ func Compile(ctx context.Context, ui *ui.UI, discovered *discover.Discovered) (* } if discovered.Plan != nil { egrp.Go(func() error { + ui.Verboseln("compiling golang actions binary for: %s", discovered.Plan.DirPath) + path, err := compile(ctx, ui, discovered.Plan) if err != nil { return err @@ -113,7 +117,7 @@ func compile(ctx context.Context, ui *ui.UI, actions *discover.ActionsDiscovered var binarypath string - if err := codegen.PatchGoMod(actions.ParentDir, shuttlelocaldir); err != nil { + if err := codegen.PatchGoMod(actions.ParentDir, shuttlelocaldir, ui); err != nil { return "", fmt.Errorf("failed to patch generated go.mod: %w", err) } diff --git a/pkg/executors/golang/compile/matcher/matcher.go b/pkg/executors/golang/compile/matcher/matcher.go index 9974b0d..61a1e39 100644 --- a/pkg/executors/golang/compile/matcher/matcher.go +++ b/pkg/executors/golang/compile/matcher/matcher.go @@ -12,6 +12,7 @@ import ( "github.com/lunarway/shuttle/pkg/executors/golang/discover" "github.com/lunarway/shuttle/pkg/ui" + "golang.org/x/exp/slices" "golang.org/x/mod/sumdb/dirhash" ) @@ -66,6 +67,7 @@ func GetHash(ctx context.Context, actions *discover.ActionsDiscovered) (string, return io.NopCloser(bytes.NewReader(b)), nil } + slices.Sort(entries) hash, err := dirhash.Hash1(entries, open) if err != nil { return "", err diff --git a/pkg/executors/golang/executer/prepare.go b/pkg/executors/golang/executer/prepare.go index 7a342e3..6c0710e 100644 --- a/pkg/executors/golang/executer/prepare.go +++ b/pkg/executors/golang/executer/prepare.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "time" "github.com/lunarway/shuttle/pkg/config" "github.com/lunarway/shuttle/pkg/executors/golang/compile" @@ -17,6 +18,8 @@ func prepare( path string, c *config.ShuttleProjectContext, ) (*compile.Binaries, error) { + ui.Verboseln("preparing shuttle golang actions") + start := time.Now() log.SetFlags(log.LstdFlags | log.Lshortfile) disc, err := discover.Discover(ctx, path, c) @@ -29,5 +32,8 @@ func prepare( return nil, fmt.Errorf("failed to compile binaries: %v", err) } + elapsed := time.Since(start) + ui.Verboseln("preparing shuttle golang actions took: %d ms", elapsed.Milliseconds()) + return binaries, nil } diff --git a/pkg/executors/golang/executer/run.go b/pkg/executors/golang/executer/run.go index fdce12c..9435c16 100644 --- a/pkg/executors/golang/executer/run.go +++ b/pkg/executors/golang/executer/run.go @@ -20,6 +20,7 @@ func Run( return err } + ui.Verboseln("executing shuttle golang actions") if err := executeAction(ctx, binaries, args...); err != nil { return err } diff --git a/pkg/executors/task.go b/pkg/executors/task.go index 0f8725a..ce5aa66 100644 --- a/pkg/executors/task.go +++ b/pkg/executors/task.go @@ -26,7 +26,7 @@ func executeTask(ctx context.Context, ui *ui.UI, context ActionExecutionContext) args = append(args, value) } - err := executer.Run(ctx, ui, &context.ScriptContext.Project, "shuttle.yaml", args...) + err := executer.Run(ctx, ui, &context.ScriptContext.Project, fmt.Sprintf("%s/shuttle.yaml", context.ScriptContext.Project.ProjectPath), args...) if err != nil { return err } From d09a755f4f21cab039da3ef24cfa2d6ceb59619d Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 7 Dec 2023 16:41:42 +0100 Subject: [PATCH 09/17] refactor: into different files Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch.go | 166 +++--------------- pkg/executors/golang/codegen/patch_default.go | 14 ++ pkg/executors/golang/codegen/patch_finder.go | 32 ++++ .../golang/codegen/patch_gomodule.go | 74 ++++++++ .../golang/codegen/patch_workspace.go | 110 ++++++++++++ 5 files changed, 258 insertions(+), 138 deletions(-) create mode 100644 pkg/executors/golang/codegen/patch_default.go create mode 100644 pkg/executors/golang/codegen/patch_finder.go create mode 100644 pkg/executors/golang/codegen/patch_gomodule.go create mode 100644 pkg/executors/golang/codegen/patch_workspace.go diff --git a/pkg/executors/golang/codegen/patch.go b/pkg/executors/golang/codegen/patch.go index 5cbee94..478001d 100644 --- a/pkg/executors/golang/codegen/patch.go +++ b/pkg/executors/golang/codegen/patch.go @@ -2,7 +2,7 @@ package codegen import ( "bytes" - "errors" + "context" "fmt" "io/fs" "os" @@ -14,8 +14,9 @@ import ( type writeFileFunc = func(name string, contents []byte, permissions fs.FileMode) error -func PatchGoMod(rootDir string, shuttleLocalDir string, ui *ui.UI) error { +func PatchGoMod(ctx context.Context, rootDir string, shuttleLocalDir string) error { return patchGoMod( + ctx, rootDir, shuttleLocalDir, ui, @@ -23,35 +24,17 @@ func PatchGoMod(rootDir string, shuttleLocalDir string, ui *ui.UI) error { ) } -func patchGoMod(rootDir, shuttleLocalDir string, ui *ui.UI, writeFileFunc writeFileFunc) error { - packages := make(map[string]string, 0) - - if rootWorkspaceExists(rootDir) { - ui.Verboseln("patching go action using workspace file") - modules, err := GetWorkspaceModules(rootDir) - if err != nil { - return fmt.Errorf("failed to parse go.mod in root of project: %w", err) - } - - for _, module := range modules { - moduleName, modulePath, err := GetWorkspaceModule(rootDir, module) - if err != nil { - return err - } - packages[moduleName] = modulePath - } - - } else if rootModExists(rootDir) { - ui.Verboseln("patching go action using mod file") - moduleName, modulePath, err := GetRootModule(rootDir) - if err != nil { - return fmt.Errorf("failed to parse go.mod in root of project: %w", err) - } - - packages[moduleName] = modulePath +func patchGoMod(ctx context.Context, rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) error { + packages, err := newChainedwPatchFinder( + newWorkspaceFinder(rootDir), + newGoModuleFinder(rootDir), + newDefaultFinder(), + ).findPackages(ctx) + if err != nil { + return err } - if err := patchPackagesUsed(rootDir, shuttleLocalDir, packages, ui, writeFileFunc); err != nil { + if err := newGoModPatcher(writeFileFunc).patchPackagesUsed(rootDir, shuttleLocalDir, packages); err != nil { return err } @@ -59,7 +42,21 @@ func patchGoMod(rootDir, shuttleLocalDir string, ui *ui.UI, writeFileFunc writeF } -func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[string]string, ui *ui.UI, writeFileFunc writeFileFunc) error { +// packageFinder exists to find whatever patches are required for a given shuttle golang action to function +type packageFinder interface { + // Find should return how many packages are required to function + Find(ctx context.Context) (packages map[string]string, ok bool, err error) +} + +type goModPatcher struct { + writeFileFunc writeFileFunc +} + +func newGoModPatcher(writeFileFunc writeFileFunc) *goModPatcher { + return &goModPatcher{writeFileFunc: writeFileFunc} +} + +func (p *goModPatcher) patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[string]string) error { actionsModFilePath := path.Join(shuttleLocalDir, "tmp/go.mod") relativeActionsModFilePath := strings.TrimPrefix(path.Join(strings.TrimPrefix(shuttleLocalDir, rootDir), "tmp/go.mod"), "/") segmentsToRoot := strings.Count(relativeActionsModFilePath, "/") @@ -84,7 +81,6 @@ func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[stri if !actionsModFileContainsModule(moduleName) { continue } - ui.Verboseln("golang binary patch adding: moduleName: %s and modulePath: %s", moduleName, modulePath) relativeToActionsModulePath := path.Join(strings.Repeat("../", segmentsToRoot), modulePath) @@ -113,116 +109,10 @@ func patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[stri } actionsFileWriter := bytes.NewBufferString(strings.Join(actionsModFileLines, "\n")) - err = writeFileFunc(actionsModFilePath, actionsFileWriter.Bytes(), actionsModFilePermissions.Mode()) + err = p.writeFileFunc(actionsModFilePath, actionsFileWriter.Bytes(), actionsModFilePermissions.Mode()) if err != nil { return err } return nil } - -func GetRootModule(rootDir string) (moduleName string, modulePath string, err error) { - modFile, err := os.ReadFile(path.Join(rootDir, "go.mod")) - if err != nil { - return "", "", err - } - - modFileContent := string(modFile) - lines := strings.Split(modFileContent, "\n") - if len(lines) == 0 { - return "", "", errors.New("go mod is empty") - } - - for _, line := range lines { - modFileLine := strings.TrimSpace(line) - if strings.HasPrefix(modFileLine, "module") { - sections := strings.Split(modFileLine, " ") - if len(sections) < 2 { - return "", "", fmt.Errorf("invalid module line: %s", modFileLine) - } - - moduleName := sections[1] - - return moduleName, ".", nil - } - } - - return "", "", errors.New("failed to find a valid go.mod file") -} - -func GetWorkspaceModule(rootDir string, absoluteModulePath string) (moduleName string, modulePath string, err error) { - modFile, err := os.ReadFile(path.Join(rootDir, absoluteModulePath, "go.mod")) - if err != nil { - return "", "", fmt.Errorf("failed to find go.mod at: %s: %w", absoluteModulePath, err) - } - - modFileContent := string(modFile) - lines := strings.Split(modFileContent, "\n") - if len(lines) == 0 { - return "", "", errors.New("go mod is empty") - } - - for _, line := range lines { - modFileLine := strings.TrimSpace(line) - if strings.HasPrefix(modFileLine, "module") { - sections := strings.Split(modFileLine, " ") - if len(sections) < 2 { - return "", "", fmt.Errorf("invalid module line: %s", modFileLine) - } - - moduleName := sections[1] - modulePath = strings.TrimPrefix(absoluteModulePath, rootDir) - - return moduleName, modulePath, nil - } - } - - return "", "", errors.New("failed to find a valid go.mod file") -} - -func GetWorkspaceModules(rootDir string) (modules []string, err error) { - workFile, err := os.ReadFile(path.Join(rootDir, "go.work")) - if err != nil { - return nil, err - } - - workFileContent := string(workFile) - lines := strings.Split(workFileContent, "\n") - if len(lines) == 0 { - return nil, errors.New("go work is empty") - } - - modules = make([]string, 0) - for _, line := range lines { - modFileLine := strings.Trim(strings.TrimSpace(line), "\t") - if strings.HasPrefix(modFileLine, ".") && modFileLine != "./actions" { - modules = append( - modules, - strings.TrimPrefix( - strings.TrimPrefix(modFileLine, "."), - "/", - ), - ) - } - } - - return modules, nil -} - -func rootModExists(rootDir string) bool { - goMod := path.Join(rootDir, "go.mod") - if _, err := os.Stat(goMod); errors.Is(err, os.ErrNotExist) { - return false - } - - return true -} - -func rootWorkspaceExists(rootDir string) bool { - goWork := path.Join(rootDir, "go.work") - if _, err := os.Stat(goWork); errors.Is(err, os.ErrNotExist) { - return false - } - - return true -} diff --git a/pkg/executors/golang/codegen/patch_default.go b/pkg/executors/golang/codegen/patch_default.go new file mode 100644 index 0000000..f9ab2fa --- /dev/null +++ b/pkg/executors/golang/codegen/patch_default.go @@ -0,0 +1,14 @@ +package codegen + +import "context" + +type defaultFinder struct{} + +func newDefaultFinder() *defaultFinder { + return &defaultFinder{} +} + +func (s *defaultFinder) Find(ctx context.Context) (packages map[string]string, ok bool, err error) { + // We return true, as this should be placed last in the chain + return make(map[string]string, 0), true, nil +} diff --git a/pkg/executors/golang/codegen/patch_finder.go b/pkg/executors/golang/codegen/patch_finder.go new file mode 100644 index 0000000..2c13abb --- /dev/null +++ b/pkg/executors/golang/codegen/patch_finder.go @@ -0,0 +1,32 @@ +package codegen + +import ( + "context" + "errors" +) + +type chainedPackageFinder struct { + finders []packageFinder +} + +func newChainedwPatchFinder(finders ...packageFinder) *chainedPackageFinder { + return &chainedPackageFinder{ + finders: finders, + } +} + +// FindPackages is setup as a chain of responsibility, which means that from most significant it will attempt to find packages +// to be used. However, each finder needs to return how many packages it needs to function, as returning ok means that the finder has exclusive access to the packages +func (p *chainedPackageFinder) findPackages(ctx context.Context) (packages map[string]string, err error) { + for _, finder := range p.finders { + packages, ok, err := finder.Find(ctx) + if err != nil { + return nil, err + } + if ok { + return packages, nil + } + } + + return nil, errors.New("failed to find a valid patcher") +} diff --git a/pkg/executors/golang/codegen/patch_gomodule.go b/pkg/executors/golang/codegen/patch_gomodule.go new file mode 100644 index 0000000..fd5632a --- /dev/null +++ b/pkg/executors/golang/codegen/patch_gomodule.go @@ -0,0 +1,74 @@ +package codegen + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "strings" +) + +type goModuleFinder struct { + rootDir string +} + +func newGoModuleFinder(rootDir string) *goModuleFinder { + return &goModuleFinder{ + rootDir: rootDir, + } +} + +func (s *goModuleFinder) Find(ctx context.Context) (packages map[string]string, ok bool, err error) { + if !s.rootModExists() { + return nil, false, nil + } + + moduleName, modulePath, err := s.getRootModule() + if err != nil { + return nil, true, fmt.Errorf("failed to parse go.mod in root of project: %w", err) + } + + packages = make(map[string]string, 0) + packages[moduleName] = modulePath + + return packages, true, nil +} + +func (g *goModuleFinder) getRootModule() (moduleName string, modulePath string, err error) { + modFile, err := os.ReadFile(path.Join(g.rootDir, "go.mod")) + if err != nil { + return "", "", err + } + + modFileContent := string(modFile) + lines := strings.Split(modFileContent, "\n") + if len(lines) == 0 { + return "", "", errors.New("go mod is empty") + } + + for _, line := range lines { + modFileLine := strings.TrimSpace(line) + if strings.HasPrefix(modFileLine, "module") { + sections := strings.Split(modFileLine, " ") + if len(sections) < 2 { + return "", "", fmt.Errorf("invalid module line: %s", modFileLine) + } + + moduleName := sections[1] + + return moduleName, ".", nil + } + } + + return "", "", errors.New("failed to find a valid go.mod file") +} + +func (g *goModuleFinder) rootModExists() bool { + goMod := path.Join(g.rootDir, "go.mod") + if _, err := os.Stat(goMod); errors.Is(err, os.ErrNotExist) { + return false + } + + return true +} diff --git a/pkg/executors/golang/codegen/patch_workspace.go b/pkg/executors/golang/codegen/patch_workspace.go new file mode 100644 index 0000000..053b834 --- /dev/null +++ b/pkg/executors/golang/codegen/patch_workspace.go @@ -0,0 +1,110 @@ +package codegen + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "strings" +) + +type workspaceFinder struct { + rootDir string +} + +func newWorkspaceFinder(rootDir string) *workspaceFinder { + return &workspaceFinder{ + rootDir: rootDir, + } +} + +func (w *workspaceFinder) rootWorkspaceExists() bool { + goWork := path.Join(w.rootDir, "go.work") + if _, err := os.Stat(goWork); errors.Is(err, os.ErrNotExist) { + return false + } + + return true +} + +func (s *workspaceFinder) Find(ctx context.Context) (packages map[string]string, ok bool, err error) { + if !s.rootWorkspaceExists() { + return nil, false, nil + } + + modules, err := s.getWorkspaceModules() + if err != nil { + return nil, true, err + } + + packages = make(map[string]string, 0) + for _, module := range modules { + moduleName, modulePath, err := s.getWorkspaceModule(module) + if err != nil { + return nil, true, err + } + packages[moduleName] = modulePath + } + + return packages, true, nil +} + +func (w *workspaceFinder) getWorkspaceModules() (modules []string, err error) { + workFile, err := os.ReadFile(path.Join(w.rootDir, "go.work")) + if err != nil { + return nil, err + } + + workFileContent := string(workFile) + lines := strings.Split(workFileContent, "\n") + if len(lines) == 0 { + return nil, errors.New("go work is empty") + } + + modules = make([]string, 0) + for _, line := range lines { + modFileLine := strings.Trim(strings.TrimSpace(line), "\t") + if strings.HasPrefix(modFileLine, ".") && modFileLine != "./actions" { + modules = append( + modules, + strings.TrimPrefix( + strings.TrimPrefix(modFileLine, "."), + "/", + ), + ) + } + } + + return modules, nil +} + +func (w *workspaceFinder) getWorkspaceModule(absoluteModulePath string) (moduleName string, modulePath string, err error) { + modFile, err := os.ReadFile(path.Join(w.rootDir, absoluteModulePath, "go.mod")) + if err != nil { + return "", "", fmt.Errorf("failed to find go.mod at: %s: %w", absoluteModulePath, err) + } + + modFileContent := string(modFile) + lines := strings.Split(modFileContent, "\n") + if len(lines) == 0 { + return "", "", errors.New("go mod is empty") + } + + for _, line := range lines { + modFileLine := strings.TrimSpace(line) + if strings.HasPrefix(modFileLine, "module") { + sections := strings.Split(modFileLine, " ") + if len(sections) < 2 { + return "", "", fmt.Errorf("invalid module line: %s", modFileLine) + } + + moduleName := sections[1] + modulePath = strings.TrimPrefix(absoluteModulePath, w.rootDir) + + return moduleName, modulePath, nil + } + } + + return "", "", errors.New("failed to find a valid go.mod file") +} From f1a55d3e5f64d3e2c9aff6aa3f63fef2a4ace1c1 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 7 Dec 2023 17:09:21 +0100 Subject: [PATCH 10/17] refactor: into package finder Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch.go | 108 +++--------------- pkg/executors/golang/codegen/patch_default.go | 2 +- pkg/executors/golang/codegen/patch_finder.go | 10 +- .../golang/codegen/patch_gomodule.go | 24 ++-- pkg/executors/golang/codegen/patch_patcher.go | 100 ++++++++++++++++ pkg/executors/golang/codegen/patch_test.go | 22 +++- .../golang/codegen/patch_workspace.go | 32 +++--- pkg/executors/golang/compile/compile.go | 2 +- 8 files changed, 164 insertions(+), 136 deletions(-) create mode 100644 pkg/executors/golang/codegen/patch_patcher.go diff --git a/pkg/executors/golang/codegen/patch.go b/pkg/executors/golang/codegen/patch.go index 478001d..b477a79 100644 --- a/pkg/executors/golang/codegen/patch.go +++ b/pkg/executors/golang/codegen/patch.go @@ -1,116 +1,36 @@ package codegen import ( - "bytes" "context" - "fmt" "io/fs" "os" - "path" - "strings" - - "github.com/lunarway/shuttle/pkg/ui" ) type writeFileFunc = func(name string, contents []byte, permissions fs.FileMode) error -func PatchGoMod(ctx context.Context, rootDir string, shuttleLocalDir string) error { - return patchGoMod( - ctx, - rootDir, - shuttleLocalDir, - ui, - os.WriteFile, - ) +type Patcher struct { + patchFinder *chainedPackageFinder + patcher *goModPatcher } -func patchGoMod(ctx context.Context, rootDir, shuttleLocalDir string, writeFileFunc writeFileFunc) error { - packages, err := newChainedwPatchFinder( - newWorkspaceFinder(rootDir), - newGoModuleFinder(rootDir), - newDefaultFinder(), - ).findPackages(ctx) - if err != nil { - return err - } - - if err := newGoModPatcher(writeFileFunc).patchPackagesUsed(rootDir, shuttleLocalDir, packages); err != nil { - return err +func NewPatcher() *Patcher { + return &Patcher{ + patchFinder: newChainedwPatchFinder( + newWorkspaceFinder(), + newGoModuleFinder(), + newDefaultFinder(), + ), + patcher: newGoModPatcher(os.WriteFile), } - - return nil - -} - -// packageFinder exists to find whatever patches are required for a given shuttle golang action to function -type packageFinder interface { - // Find should return how many packages are required to function - Find(ctx context.Context) (packages map[string]string, ok bool, err error) -} - -type goModPatcher struct { - writeFileFunc writeFileFunc -} - -func newGoModPatcher(writeFileFunc writeFileFunc) *goModPatcher { - return &goModPatcher{writeFileFunc: writeFileFunc} } -func (p *goModPatcher) patchPackagesUsed(rootDir string, shuttleLocalDir string, packages map[string]string) error { - actionsModFilePath := path.Join(shuttleLocalDir, "tmp/go.mod") - relativeActionsModFilePath := strings.TrimPrefix(path.Join(strings.TrimPrefix(shuttleLocalDir, rootDir), "tmp/go.mod"), "/") - segmentsToRoot := strings.Count(relativeActionsModFilePath, "/") - - actionsModFileContents, err := os.ReadFile(actionsModFilePath) +func (p *Patcher) Patch(ctx context.Context, rootDir string, shuttleLocalDir string) error { + packages, err := p.patchFinder.findPackages(ctx, rootDir) if err != nil { return err } - actionsModFilePermissions, err := os.Stat(actionsModFilePath) - if err != nil { - return err - } - - actionsModFile := string(actionsModFileContents) - actionsModFileLines := strings.Split(actionsModFile, "\n") - - actionsModFileContainsModule := func(moduleName string) bool { - return strings.Contains(actionsModFile, moduleName) - } - - for moduleName, modulePath := range packages { - if !actionsModFileContainsModule(moduleName) { - continue - } - - relativeToActionsModulePath := path.Join(strings.Repeat("../", segmentsToRoot), modulePath) - foundReplace := false - for i, line := range actionsModFileLines { - lineTrim := strings.TrimSpace(line) - - if strings.Contains(lineTrim, fmt.Sprintf("replace %s", moduleName)) { - actionsModFileLines[i] = fmt.Sprintf("replace %s => %s", moduleName, relativeToActionsModulePath) - foundReplace = true - break - } - } - - if !foundReplace { - actionsModFileLines = append( - actionsModFileLines, - fmt.Sprintf("\nreplace %s => %s", moduleName, relativeToActionsModulePath), - ) - - } - - if err != nil { - return err - } - } - - actionsFileWriter := bytes.NewBufferString(strings.Join(actionsModFileLines, "\n")) - err = p.writeFileFunc(actionsModFilePath, actionsFileWriter.Bytes(), actionsModFilePermissions.Mode()) - if err != nil { + if err := p.patcher.patch(rootDir, shuttleLocalDir, packages); err != nil { return err } diff --git a/pkg/executors/golang/codegen/patch_default.go b/pkg/executors/golang/codegen/patch_default.go index f9ab2fa..b7f49ad 100644 --- a/pkg/executors/golang/codegen/patch_default.go +++ b/pkg/executors/golang/codegen/patch_default.go @@ -8,7 +8,7 @@ func newDefaultFinder() *defaultFinder { return &defaultFinder{} } -func (s *defaultFinder) Find(ctx context.Context) (packages map[string]string, ok bool, err error) { +func (s *defaultFinder) Find(ctx context.Context, _ string) (packages map[string]string, ok bool, err error) { // We return true, as this should be placed last in the chain return make(map[string]string, 0), true, nil } diff --git a/pkg/executors/golang/codegen/patch_finder.go b/pkg/executors/golang/codegen/patch_finder.go index 2c13abb..379a7bf 100644 --- a/pkg/executors/golang/codegen/patch_finder.go +++ b/pkg/executors/golang/codegen/patch_finder.go @@ -5,6 +5,12 @@ import ( "errors" ) +// packageFinder exists to find whatever patches are required for a given shuttle golang action to function +type packageFinder interface { + // Find should return how many packages are required to function + Find(ctx context.Context, rootDir string) (packages map[string]string, ok bool, err error) +} + type chainedPackageFinder struct { finders []packageFinder } @@ -17,9 +23,9 @@ func newChainedwPatchFinder(finders ...packageFinder) *chainedPackageFinder { // FindPackages is setup as a chain of responsibility, which means that from most significant it will attempt to find packages // to be used. However, each finder needs to return how many packages it needs to function, as returning ok means that the finder has exclusive access to the packages -func (p *chainedPackageFinder) findPackages(ctx context.Context) (packages map[string]string, err error) { +func (p *chainedPackageFinder) findPackages(ctx context.Context, rootDir string) (packages map[string]string, err error) { for _, finder := range p.finders { - packages, ok, err := finder.Find(ctx) + packages, ok, err := finder.Find(ctx, rootDir) if err != nil { return nil, err } diff --git a/pkg/executors/golang/codegen/patch_gomodule.go b/pkg/executors/golang/codegen/patch_gomodule.go index fd5632a..b65254b 100644 --- a/pkg/executors/golang/codegen/patch_gomodule.go +++ b/pkg/executors/golang/codegen/patch_gomodule.go @@ -9,22 +9,18 @@ import ( "strings" ) -type goModuleFinder struct { - rootDir string -} +type goModuleFinder struct{} -func newGoModuleFinder(rootDir string) *goModuleFinder { - return &goModuleFinder{ - rootDir: rootDir, - } +func newGoModuleFinder() *goModuleFinder { + return &goModuleFinder{} } -func (s *goModuleFinder) Find(ctx context.Context) (packages map[string]string, ok bool, err error) { - if !s.rootModExists() { +func (s *goModuleFinder) Find(ctx context.Context, rootDir string) (packages map[string]string, ok bool, err error) { + if !s.rootModExists(rootDir) { return nil, false, nil } - moduleName, modulePath, err := s.getRootModule() + moduleName, modulePath, err := s.getRootModule(rootDir) if err != nil { return nil, true, fmt.Errorf("failed to parse go.mod in root of project: %w", err) } @@ -35,8 +31,8 @@ func (s *goModuleFinder) Find(ctx context.Context) (packages map[string]string, return packages, true, nil } -func (g *goModuleFinder) getRootModule() (moduleName string, modulePath string, err error) { - modFile, err := os.ReadFile(path.Join(g.rootDir, "go.mod")) +func (g *goModuleFinder) getRootModule(rootDir string) (moduleName string, modulePath string, err error) { + modFile, err := os.ReadFile(path.Join(rootDir, "go.mod")) if err != nil { return "", "", err } @@ -64,8 +60,8 @@ func (g *goModuleFinder) getRootModule() (moduleName string, modulePath string, return "", "", errors.New("failed to find a valid go.mod file") } -func (g *goModuleFinder) rootModExists() bool { - goMod := path.Join(g.rootDir, "go.mod") +func (g *goModuleFinder) rootModExists(rootDir string) bool { + goMod := path.Join(rootDir, "go.mod") if _, err := os.Stat(goMod); errors.Is(err, os.ErrNotExist) { return false } diff --git a/pkg/executors/golang/codegen/patch_patcher.go b/pkg/executors/golang/codegen/patch_patcher.go new file mode 100644 index 0000000..891ae14 --- /dev/null +++ b/pkg/executors/golang/codegen/patch_patcher.go @@ -0,0 +1,100 @@ +package codegen + +import ( + "bytes" + "fmt" + "os" + "path" + "strings" + + "golang.org/x/exp/slices" +) + +type module struct { + name string + path string +} + +func modulesFromMap(packages map[string]string) []module { + modules := make([]module, 0, len(packages)) + for moduleName, modulePath := range packages { + modules = append(modules, module{ + name: moduleName, + path: modulePath, + }) + } + slices.SortFunc(modules, func(a, b module) int { + return strings.Compare(a.name, b.name) + }) + + return modules +} + +type goModPatcher struct { + writeFileFunc writeFileFunc +} + +func newGoModPatcher(writeFileFunc writeFileFunc) *goModPatcher { + return &goModPatcher{writeFileFunc: writeFileFunc} +} + +func (p *goModPatcher) patch(rootDir string, shuttleLocalDir string, packages map[string]string) error { + actionsModFilePath := path.Join(shuttleLocalDir, "tmp/go.mod") + relativeActionsModFilePath := strings.TrimPrefix(path.Join(strings.TrimPrefix(shuttleLocalDir, rootDir), "tmp/go.mod"), "/") + + segmentsToRoot := strings.Count(relativeActionsModFilePath, "/") + actionsModFileContents, err := os.ReadFile(actionsModFilePath) + if err != nil { + return err + } + actionsModFilePermissions, err := os.Stat(actionsModFilePath) + if err != nil { + return err + } + + actionsModFile := string(actionsModFileContents) + actionsModFileLines := strings.Split(actionsModFile, "\n") + + actionsModFileContainsModule := func(moduleName string) bool { + return strings.Contains(actionsModFile, moduleName) + } + + for _, module := range modulesFromMap(packages) { + if !actionsModFileContainsModule(module.name) { + continue + } + + relativeToActionsModulePath := path.Join(strings.Repeat("../", segmentsToRoot), module.path) + + foundReplace := false + for i, line := range actionsModFileLines { + lineTrim := strings.TrimSpace(line) + + if strings.Contains(lineTrim, fmt.Sprintf("replace %s", module.name)) { + actionsModFileLines[i] = fmt.Sprintf("replace %s => %s", module.name, relativeToActionsModulePath) + foundReplace = true + break + } + } + + if !foundReplace { + actionsModFileLines = append( + actionsModFileLines, + fmt.Sprintf("\nreplace %s => %s", module.name, relativeToActionsModulePath), + ) + + } + + if err != nil { + return err + } + } + + actionsFileWriter := bytes.NewBufferString(strings.Join(actionsModFileLines, "\n")) + err = p.writeFileFunc(actionsModFilePath, actionsFileWriter.Bytes(), actionsModFilePermissions.Mode()) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/executors/golang/codegen/patch_test.go b/pkg/executors/golang/codegen/patch_test.go index f5edd6c..4172ba3 100644 --- a/pkg/executors/golang/codegen/patch_test.go +++ b/pkg/executors/golang/codegen/patch_test.go @@ -1,6 +1,7 @@ package codegen import ( + "context" "io/fs" "testing" @@ -12,7 +13,8 @@ func TestPatchGoMod(t *testing.T) { t.Parallel() t.Run("finds root module adds to actions plan", func(t *testing.T) { - err := patchGoMod("testdata/patch/root_module/", "testdata/patch/root_module/.shuttle/actions", func(name string, contents []byte, permissions fs.FileMode) error { + sut := NewPatcher() + sut.patcher = newGoModPatcher(func(name string, contents []byte, permissions fs.FileMode) error { assert.Equal(t, "testdata/patch/root_module/.shuttle/actions/tmp/go.mod", name) assert.Equal(t, `module actions @@ -27,11 +29,14 @@ replace root_module => ../../..`, string(contents)) return nil }) + + err := sut.Patch(context.Background(), "testdata/patch/root_module/", "testdata/patch/root_module/.shuttle/actions") require.NoError(t, err) }) t.Run("finds root module replaces existing", func(t *testing.T) { - err := patchGoMod("testdata/patch/replace_existing/", "testdata/patch/replace_existing/.shuttle/actions", func(name string, contents []byte, permissions fs.FileMode) error { + sut := NewPatcher() + sut.patcher = newGoModPatcher(func(name string, contents []byte, permissions fs.FileMode) error { assert.Equal(t, "testdata/patch/replace_existing/.shuttle/actions/tmp/go.mod", name) assert.Equal(t, `module actions @@ -46,11 +51,14 @@ replace replace_existing => ../../.. return nil }) + + err := sut.Patch(context.Background(), "testdata/patch/replace_existing/", "testdata/patch/replace_existing/.shuttle/actions") require.NoError(t, err) }) t.Run("finds root workspace adds entries", func(t *testing.T) { - err := patchGoMod("testdata/patch/root_workspace/", "testdata/patch/root_workspace/.shuttle/actions", func(name string, contents []byte, permissions fs.FileMode) error { + sut := NewPatcher() + sut.patcher = newGoModPatcher(func(name string, contents []byte, permissions fs.FileMode) error { assert.Equal(t, "testdata/patch/root_workspace/.shuttle/actions/tmp/go.mod", name) assert.Equal(t, `module actions @@ -63,14 +71,16 @@ require ( go 1.21.4 -replace root_workspace => ../../.. +replace othersubpackage => ../../../other/subpackage -replace subpackage => ../../../subpackage +replace root_workspace => ../../.. -replace othersubpackage => ../../../other/subpackage`, string(contents)) +replace subpackage => ../../../subpackage`, string(contents)) return nil }) + + err := sut.Patch(context.Background(), "testdata/patch/root_workspace/", "testdata/patch/root_workspace/.shuttle/actions") require.NoError(t, err) }) } diff --git a/pkg/executors/golang/codegen/patch_workspace.go b/pkg/executors/golang/codegen/patch_workspace.go index 053b834..294842e 100644 --- a/pkg/executors/golang/codegen/patch_workspace.go +++ b/pkg/executors/golang/codegen/patch_workspace.go @@ -9,18 +9,14 @@ import ( "strings" ) -type workspaceFinder struct { - rootDir string -} +type workspaceFinder struct{} -func newWorkspaceFinder(rootDir string) *workspaceFinder { - return &workspaceFinder{ - rootDir: rootDir, - } +func newWorkspaceFinder() *workspaceFinder { + return &workspaceFinder{} } -func (w *workspaceFinder) rootWorkspaceExists() bool { - goWork := path.Join(w.rootDir, "go.work") +func (w *workspaceFinder) rootWorkspaceExists(rootDir string) bool { + goWork := path.Join(rootDir, "go.work") if _, err := os.Stat(goWork); errors.Is(err, os.ErrNotExist) { return false } @@ -28,19 +24,19 @@ func (w *workspaceFinder) rootWorkspaceExists() bool { return true } -func (s *workspaceFinder) Find(ctx context.Context) (packages map[string]string, ok bool, err error) { - if !s.rootWorkspaceExists() { +func (s *workspaceFinder) Find(ctx context.Context, rootDir string) (packages map[string]string, ok bool, err error) { + if !s.rootWorkspaceExists(rootDir) { return nil, false, nil } - modules, err := s.getWorkspaceModules() + modules, err := s.getWorkspaceModules(rootDir) if err != nil { return nil, true, err } packages = make(map[string]string, 0) for _, module := range modules { - moduleName, modulePath, err := s.getWorkspaceModule(module) + moduleName, modulePath, err := s.getWorkspaceModule(rootDir, module) if err != nil { return nil, true, err } @@ -50,8 +46,8 @@ func (s *workspaceFinder) Find(ctx context.Context) (packages map[string]string, return packages, true, nil } -func (w *workspaceFinder) getWorkspaceModules() (modules []string, err error) { - workFile, err := os.ReadFile(path.Join(w.rootDir, "go.work")) +func (w *workspaceFinder) getWorkspaceModules(rootDir string) (modules []string, err error) { + workFile, err := os.ReadFile(path.Join(rootDir, "go.work")) if err != nil { return nil, err } @@ -79,8 +75,8 @@ func (w *workspaceFinder) getWorkspaceModules() (modules []string, err error) { return modules, nil } -func (w *workspaceFinder) getWorkspaceModule(absoluteModulePath string) (moduleName string, modulePath string, err error) { - modFile, err := os.ReadFile(path.Join(w.rootDir, absoluteModulePath, "go.mod")) +func (w *workspaceFinder) getWorkspaceModule(rootDir string, absoluteModulePath string) (moduleName string, modulePath string, err error) { + modFile, err := os.ReadFile(path.Join(rootDir, absoluteModulePath, "go.mod")) if err != nil { return "", "", fmt.Errorf("failed to find go.mod at: %s: %w", absoluteModulePath, err) } @@ -100,7 +96,7 @@ func (w *workspaceFinder) getWorkspaceModule(absoluteModulePath string) (moduleN } moduleName := sections[1] - modulePath = strings.TrimPrefix(absoluteModulePath, w.rootDir) + modulePath = strings.TrimPrefix(absoluteModulePath, rootDir) return moduleName, modulePath, nil } diff --git a/pkg/executors/golang/compile/compile.go b/pkg/executors/golang/compile/compile.go index 9723ff2..c8ff062 100644 --- a/pkg/executors/golang/compile/compile.go +++ b/pkg/executors/golang/compile/compile.go @@ -117,7 +117,7 @@ func compile(ctx context.Context, ui *ui.UI, actions *discover.ActionsDiscovered var binarypath string - if err := codegen.PatchGoMod(actions.ParentDir, shuttlelocaldir, ui); err != nil { + if err := codegen.NewPatcher().Patch(ctx, actions.ParentDir, shuttlelocaldir); err != nil { return "", fmt.Errorf("failed to patch generated go.mod: %w", err) } From 7afc4c4e6d1eb828b22dd06b5e4daae998a0a2a7 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 7 Dec 2023 17:16:06 +0100 Subject: [PATCH 11/17] feat: add shuttle Signed-off-by: Kasper J. Hermansen --- .../patch/replace_existing/.shuttle/actions/tmp/go.mod | 9 +++++++++ .../patch/root_module/.shuttle/actions/tmp/go.mod | 7 +++++++ .../patch/root_workspace/.shuttle/actions/tmp/go.mod | 9 +++++++++ 3 files changed, 25 insertions(+) create mode 100644 pkg/executors/golang/codegen/testdata/patch/replace_existing/.shuttle/actions/tmp/go.mod create mode 100644 pkg/executors/golang/codegen/testdata/patch/root_module/.shuttle/actions/tmp/go.mod create mode 100644 pkg/executors/golang/codegen/testdata/patch/root_workspace/.shuttle/actions/tmp/go.mod diff --git a/pkg/executors/golang/codegen/testdata/patch/replace_existing/.shuttle/actions/tmp/go.mod b/pkg/executors/golang/codegen/testdata/patch/replace_existing/.shuttle/actions/tmp/go.mod new file mode 100644 index 0000000..35779f4 --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/replace_existing/.shuttle/actions/tmp/go.mod @@ -0,0 +1,9 @@ +module actions + +require ( + replace_existing v0.0.0 +) + +go 1.21.4 + +replace replace_existing => ../bogus diff --git a/pkg/executors/golang/codegen/testdata/patch/root_module/.shuttle/actions/tmp/go.mod b/pkg/executors/golang/codegen/testdata/patch/root_module/.shuttle/actions/tmp/go.mod new file mode 100644 index 0000000..c63a4d8 --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/root_module/.shuttle/actions/tmp/go.mod @@ -0,0 +1,7 @@ +module actions + +require ( + root_module +) + +go 1.21.4 diff --git a/pkg/executors/golang/codegen/testdata/patch/root_workspace/.shuttle/actions/tmp/go.mod b/pkg/executors/golang/codegen/testdata/patch/root_workspace/.shuttle/actions/tmp/go.mod new file mode 100644 index 0000000..b215f6e --- /dev/null +++ b/pkg/executors/golang/codegen/testdata/patch/root_workspace/.shuttle/actions/tmp/go.mod @@ -0,0 +1,9 @@ +module actions + +require ( + root_workspace v0.0.0 + subpackage v0.0.0 + othersubpackage v0.0.0 +) + +go 1.21.4 From 25c00114d2d9014f33da153babf8e98b4512185b Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 7 Dec 2023 17:24:38 +0100 Subject: [PATCH 12/17] feat: refactor go mod Signed-off-by: Kasper J. Hermansen --- .../golang/codegen/patch_gomodule.go | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/pkg/executors/golang/codegen/patch_gomodule.go b/pkg/executors/golang/codegen/patch_gomodule.go index b65254b..d3f8853 100644 --- a/pkg/executors/golang/codegen/patch_gomodule.go +++ b/pkg/executors/golang/codegen/patch_gomodule.go @@ -16,11 +16,15 @@ func newGoModuleFinder() *goModuleFinder { } func (s *goModuleFinder) Find(ctx context.Context, rootDir string) (packages map[string]string, ok bool, err error) { - if !s.rootModExists(rootDir) { + contents, err := s.getGoModFile(rootDir) + if err != nil { + return nil, true, err + } + if contents == nil { return nil, false, nil } - moduleName, modulePath, err := s.getRootModule(rootDir) + moduleName, modulePath, err := s.getModuleFromModFile(contents) if err != nil { return nil, true, fmt.Errorf("failed to parse go.mod in root of project: %w", err) } @@ -31,19 +35,32 @@ func (s *goModuleFinder) Find(ctx context.Context, rootDir string) (packages map return packages, true, nil } -func (g *goModuleFinder) getRootModule(rootDir string) (moduleName string, modulePath string, err error) { +func (g *goModuleFinder) getGoModFile(rootDir string) (contents []string, err error) { + goMod := path.Join(rootDir, "go.mod") + if _, err := os.Stat(goMod); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + + return nil, err + } + modFile, err := os.ReadFile(path.Join(rootDir, "go.mod")) if err != nil { - return "", "", err + return nil, err } - modFileContent := string(modFile) - lines := strings.Split(modFileContent, "\n") + lines := strings.Split(string(modFile), "\n") + if len(lines) == 0 { - return "", "", errors.New("go mod is empty") + return nil, errors.New("go mod is empty") } - for _, line := range lines { + return lines, nil +} + +func (g *goModuleFinder) getModuleFromModFile(contents []string) (moduleName string, modulePath string, err error) { + for _, line := range contents { modFileLine := strings.TrimSpace(line) if strings.HasPrefix(modFileLine, "module") { sections := strings.Split(modFileLine, " ") @@ -59,12 +76,3 @@ func (g *goModuleFinder) getRootModule(rootDir string) (moduleName string, modul return "", "", errors.New("failed to find a valid go.mod file") } - -func (g *goModuleFinder) rootModExists(rootDir string) bool { - goMod := path.Join(rootDir, "go.mod") - if _, err := os.Stat(goMod); errors.Is(err, os.ErrNotExist) { - return false - } - - return true -} From 7db120ed0573d9a899f8df80847aa834d51fca80 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 7 Dec 2023 17:26:23 +0100 Subject: [PATCH 13/17] refactor: patch gomodule Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch_gomodule.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/executors/golang/codegen/patch_gomodule.go b/pkg/executors/golang/codegen/patch_gomodule.go index d3f8853..f125794 100644 --- a/pkg/executors/golang/codegen/patch_gomodule.go +++ b/pkg/executors/golang/codegen/patch_gomodule.go @@ -16,11 +16,11 @@ func newGoModuleFinder() *goModuleFinder { } func (s *goModuleFinder) Find(ctx context.Context, rootDir string) (packages map[string]string, ok bool, err error) { - contents, err := s.getGoModFile(rootDir) + contents, ok, err := s.getGoModFile(rootDir) if err != nil { return nil, true, err } - if contents == nil { + if !ok { return nil, false, nil } @@ -35,28 +35,28 @@ func (s *goModuleFinder) Find(ctx context.Context, rootDir string) (packages map return packages, true, nil } -func (g *goModuleFinder) getGoModFile(rootDir string) (contents []string, err error) { +func (g *goModuleFinder) getGoModFile(rootDir string) (contents []string, ok bool, err error) { goMod := path.Join(rootDir, "go.mod") if _, err := os.Stat(goMod); err != nil { if errors.Is(err, os.ErrNotExist) { - return nil, nil + return nil, false, nil } - return nil, err + return nil, true, err } modFile, err := os.ReadFile(path.Join(rootDir, "go.mod")) if err != nil { - return nil, err + return nil, true, err } lines := strings.Split(string(modFile), "\n") if len(lines) == 0 { - return nil, errors.New("go mod is empty") + return nil, true, errors.New("go mod is empty") } - return lines, nil + return lines, true, nil } func (g *goModuleFinder) getModuleFromModFile(contents []string) (moduleName string, modulePath string, err error) { From 0441cbb72b83f14bfaf4303dcf369b341878ead5 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Fri, 8 Dec 2023 09:51:32 +0100 Subject: [PATCH 14/17] feat: with split workspace patcher Signed-off-by: Kasper J. Hermansen --- .../golang/codegen/patch_gomodule.go | 12 +- pkg/executors/golang/codegen/patch_patcher.go | 157 +++++++++++------- .../golang/codegen/patch_workspace.go | 11 ++ 3 files changed, 114 insertions(+), 66 deletions(-) diff --git a/pkg/executors/golang/codegen/patch_gomodule.go b/pkg/executors/golang/codegen/patch_gomodule.go index f125794..b312ef5 100644 --- a/pkg/executors/golang/codegen/patch_gomodule.go +++ b/pkg/executors/golang/codegen/patch_gomodule.go @@ -24,13 +24,13 @@ func (s *goModuleFinder) Find(ctx context.Context, rootDir string) (packages map return nil, false, nil } - moduleName, modulePath, err := s.getModuleFromModFile(contents) + moduleName, err := s.getModuleFromModFile(contents) if err != nil { return nil, true, fmt.Errorf("failed to parse go.mod in root of project: %w", err) } packages = make(map[string]string, 0) - packages[moduleName] = modulePath + packages[moduleName] = "" return packages, true, nil } @@ -59,20 +59,20 @@ func (g *goModuleFinder) getGoModFile(rootDir string) (contents []string, ok boo return lines, true, nil } -func (g *goModuleFinder) getModuleFromModFile(contents []string) (moduleName string, modulePath string, err error) { +func (g *goModuleFinder) getModuleFromModFile(contents []string) (moduleName string, err error) { for _, line := range contents { modFileLine := strings.TrimSpace(line) if strings.HasPrefix(modFileLine, "module") { sections := strings.Split(modFileLine, " ") if len(sections) < 2 { - return "", "", fmt.Errorf("invalid module line: %s", modFileLine) + return "", fmt.Errorf("invalid module line: %s", modFileLine) } moduleName := sections[1] - return moduleName, ".", nil + return moduleName, nil } } - return "", "", errors.New("failed to find a valid go.mod file") + return "", errors.New("failed to find a valid go.mod file") } diff --git a/pkg/executors/golang/codegen/patch_patcher.go b/pkg/executors/golang/codegen/patch_patcher.go index 891ae14..48ebe1b 100644 --- a/pkg/executors/golang/codegen/patch_patcher.go +++ b/pkg/executors/golang/codegen/patch_patcher.go @@ -1,8 +1,8 @@ package codegen import ( - "bytes" "fmt" + "io/fs" "os" "path" "strings" @@ -10,26 +10,6 @@ import ( "golang.org/x/exp/slices" ) -type module struct { - name string - path string -} - -func modulesFromMap(packages map[string]string) []module { - modules := make([]module, 0, len(packages)) - for moduleName, modulePath := range packages { - modules = append(modules, module{ - name: moduleName, - path: modulePath, - }) - } - slices.SortFunc(modules, func(a, b module) int { - return strings.Compare(a.name, b.name) - }) - - return modules -} - type goModPatcher struct { writeFileFunc writeFileFunc } @@ -39,62 +19,119 @@ func newGoModPatcher(writeFileFunc writeFileFunc) *goModPatcher { } func (p *goModPatcher) patch(rootDir string, shuttleLocalDir string, packages map[string]string) error { - actionsModFilePath := path.Join(shuttleLocalDir, "tmp/go.mod") - relativeActionsModFilePath := strings.TrimPrefix(path.Join(strings.TrimPrefix(shuttleLocalDir, rootDir), "tmp/go.mod"), "/") - - segmentsToRoot := strings.Count(relativeActionsModFilePath, "/") - actionsModFileContents, err := os.ReadFile(actionsModFilePath) + actionsModFile, err := p.readActionsMod(shuttleLocalDir) if err != nil { return err } - actionsModFilePermissions, err := os.Stat(actionsModFilePath) - if err != nil { - return err + + for _, module := range modulesFromMap(packages) { + if !actionsModFile.containsModule(module.name) { + continue + } + + actionsModFile.replaceModulePath(rootDir, module) } - actionsModFile := string(actionsModFileContents) - actionsModFileLines := strings.Split(actionsModFile, "\n") + return actionsModFile.commit() +} + +func (g *goModPatcher) readActionsMod(shuttleLocalDir string) (*actionsModFile, error) { + path := path.Join(shuttleLocalDir, "tmp/go.mod") - actionsModFileContainsModule := func(moduleName string) bool { - return strings.Contains(actionsModFile, moduleName) + content, err := os.ReadFile(path) + if err != nil { + return nil, err + } + info, err := os.Stat(path) + if err != nil { + return nil, err } - for _, module := range modulesFromMap(packages) { - if !actionsModFileContainsModule(module.name) { - continue - } + lines := strings.Split(string(content), "\n") - relativeToActionsModulePath := path.Join(strings.Repeat("../", segmentsToRoot), module.path) + return &actionsModFile{ + info: info, + content: lines, + path: path, - foundReplace := false - for i, line := range actionsModFileLines { - lineTrim := strings.TrimSpace(line) + writeFileFunc: g.writeFileFunc, + }, nil +} - if strings.Contains(lineTrim, fmt.Sprintf("replace %s", module.name)) { - actionsModFileLines[i] = fmt.Sprintf("replace %s => %s", module.name, relativeToActionsModulePath) - foundReplace = true - break - } - } +type actionsModFile struct { + info fs.FileInfo + content []string + path string - if !foundReplace { - actionsModFileLines = append( - actionsModFileLines, - fmt.Sprintf("\nreplace %s => %s", module.name, relativeToActionsModulePath), - ) + writeFileFunc writeFileFunc +} - } +func (a *actionsModFile) containsModule(moduleName string) bool { + return slices.ContainsFunc(a.content, func(s string) bool { + return strings.Contains(s, moduleName) + }) +} + +func (a *actionsModFile) replaceModulePath(rootDir string, module module) { + relativeToActionsModulePath := path.Join(strings.Repeat("../", a.segmentsTo(rootDir)), module.path) - if err != nil { - return err + foundReplace := false + for i, line := range a.content { + lineTrim := strings.TrimSpace(line) + + if strings.Contains(lineTrim, fmt.Sprintf("replace %s", module.name)) { + a.content[i] = fmt.Sprintf("replace %s => %s", module.name, relativeToActionsModulePath) + foundReplace = true + break } + } - actionsFileWriter := bytes.NewBufferString(strings.Join(actionsModFileLines, "\n")) - err = p.writeFileFunc(actionsModFilePath, actionsFileWriter.Bytes(), actionsModFilePermissions.Mode()) - if err != nil { - return err + if !foundReplace { + a.content = append( + a.content, + fmt.Sprintf("\nreplace %s => %s", module.name, relativeToActionsModulePath), + ) + + } +} + +func (a *actionsModFile) segmentsTo(dirPath string) int { + relativeActionsModFilePath := strings.TrimPrefix( + strings.TrimPrefix( + a.path, + dirPath, + ), + "/", + ) + + return strings.Count(relativeActionsModFilePath, "/") +} + +func (a *actionsModFile) commit() error { + return a.writeFileFunc( + a.path, + []byte(strings.Join(a.content, "\n")), + a.info.Mode(), + ) +} + +type module struct { + name string + path string +} + +func modulesFromMap(packages map[string]string) []module { + modules := make([]module, 0, len(packages)) + for moduleName, modulePath := range packages { + modules = append(modules, module{ + name: moduleName, + path: modulePath, + }) } + slices.SortFunc(modules, func(a, b module) int { + return strings.Compare(a.name, b.name) + }) - return nil + return modules } diff --git a/pkg/executors/golang/codegen/patch_workspace.go b/pkg/executors/golang/codegen/patch_workspace.go index 294842e..bb0de16 100644 --- a/pkg/executors/golang/codegen/patch_workspace.go +++ b/pkg/executors/golang/codegen/patch_workspace.go @@ -99,6 +99,17 @@ func (w *workspaceFinder) getWorkspaceModule(rootDir string, absoluteModulePath modulePath = strings.TrimPrefix(absoluteModulePath, rootDir) return moduleName, modulePath, nil + } else if strings.HasPrefix(modFileLine, "use") && strings.Contains(modFileLine, ".") { + sections := strings.Split(modFileLine, " ") + if len(sections) == 2 { + return "", "", fmt.Errorf("invalid module line: %s", modFileLine) + } + + moduleName := sections[1] + modulePath = strings.TrimPrefix(absoluteModulePath, rootDir) + + return moduleName, modulePath, nil + } } From bbb714cb954bd327bcfb137dad4af20233b98d2f Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Fri, 8 Dec 2023 09:54:45 +0100 Subject: [PATCH 15/17] refactor: into modfile and module Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/mod_file.go | 68 +++++++++++++++ pkg/executors/golang/codegen/module.go | 27 ++++++ pkg/executors/golang/codegen/patch_patcher.go | 82 ------------------- 3 files changed, 95 insertions(+), 82 deletions(-) create mode 100644 pkg/executors/golang/codegen/mod_file.go create mode 100644 pkg/executors/golang/codegen/module.go diff --git a/pkg/executors/golang/codegen/mod_file.go b/pkg/executors/golang/codegen/mod_file.go new file mode 100644 index 0000000..ba905e7 --- /dev/null +++ b/pkg/executors/golang/codegen/mod_file.go @@ -0,0 +1,68 @@ +package codegen + +import ( + "fmt" + "io/fs" + "path" + "strings" + + "golang.org/x/exp/slices" +) + +type actionsModFile struct { + info fs.FileInfo + content []string + path string + + writeFileFunc writeFileFunc +} + +func (a *actionsModFile) containsModule(moduleName string) bool { + return slices.ContainsFunc(a.content, func(s string) bool { + return strings.Contains(s, moduleName) + }) +} + +func (a *actionsModFile) replaceModulePath(rootDir string, module module) { + relativeToActionsModulePath := path.Join(strings.Repeat("../", a.segmentsTo(rootDir)), module.path) + + foundReplace := false + for i, line := range a.content { + lineTrim := strings.TrimSpace(line) + + if strings.Contains(lineTrim, fmt.Sprintf("replace %s", module.name)) { + a.content[i] = fmt.Sprintf("replace %s => %s", module.name, relativeToActionsModulePath) + foundReplace = true + break + } + + } + + if !foundReplace { + a.content = append( + a.content, + fmt.Sprintf("\nreplace %s => %s", module.name, relativeToActionsModulePath), + ) + + } +} + +func (a *actionsModFile) segmentsTo(dirPath string) int { + relativeActionsModFilePath := strings.TrimPrefix( + strings.TrimPrefix( + a.path, + dirPath, + ), + "/", + ) + + return strings.Count(relativeActionsModFilePath, "/") +} + +func (a *actionsModFile) commit() error { + return a.writeFileFunc( + a.path, + []byte(strings.Join(a.content, "\n")), + a.info.Mode(), + ) +} diff --git a/pkg/executors/golang/codegen/module.go b/pkg/executors/golang/codegen/module.go new file mode 100644 index 0000000..2650905 --- /dev/null +++ b/pkg/executors/golang/codegen/module.go @@ -0,0 +1,27 @@ +package codegen + +import ( + "strings" + + "golang.org/x/exp/slices" +) + +type module struct { + name string + path string +} + +func modulesFromMap(packages map[string]string) []module { + modules := make([]module, 0, len(packages)) + for moduleName, modulePath := range packages { + modules = append(modules, module{ + name: moduleName, + path: modulePath, + }) + } + slices.SortFunc(modules, func(a, b module) int { + return strings.Compare(a.name, b.name) + }) + + return modules +} diff --git a/pkg/executors/golang/codegen/patch_patcher.go b/pkg/executors/golang/codegen/patch_patcher.go index 48ebe1b..1f40b89 100644 --- a/pkg/executors/golang/codegen/patch_patcher.go +++ b/pkg/executors/golang/codegen/patch_patcher.go @@ -1,13 +1,9 @@ package codegen import ( - "fmt" - "io/fs" "os" "path" "strings" - - "golang.org/x/exp/slices" ) type goModPatcher struct { @@ -57,81 +53,3 @@ func (g *goModPatcher) readActionsMod(shuttleLocalDir string) (*actionsModFile, writeFileFunc: g.writeFileFunc, }, nil } - -type actionsModFile struct { - info fs.FileInfo - content []string - path string - - writeFileFunc writeFileFunc -} - -func (a *actionsModFile) containsModule(moduleName string) bool { - return slices.ContainsFunc(a.content, func(s string) bool { - return strings.Contains(s, moduleName) - }) -} - -func (a *actionsModFile) replaceModulePath(rootDir string, module module) { - relativeToActionsModulePath := path.Join(strings.Repeat("../", a.segmentsTo(rootDir)), module.path) - - foundReplace := false - for i, line := range a.content { - lineTrim := strings.TrimSpace(line) - - if strings.Contains(lineTrim, fmt.Sprintf("replace %s", module.name)) { - a.content[i] = fmt.Sprintf("replace %s => %s", module.name, relativeToActionsModulePath) - foundReplace = true - break - } - - } - - if !foundReplace { - a.content = append( - a.content, - fmt.Sprintf("\nreplace %s => %s", module.name, relativeToActionsModulePath), - ) - - } -} - -func (a *actionsModFile) segmentsTo(dirPath string) int { - relativeActionsModFilePath := strings.TrimPrefix( - strings.TrimPrefix( - a.path, - dirPath, - ), - "/", - ) - - return strings.Count(relativeActionsModFilePath, "/") -} - -func (a *actionsModFile) commit() error { - return a.writeFileFunc( - a.path, - []byte(strings.Join(a.content, "\n")), - a.info.Mode(), - ) -} - -type module struct { - name string - path string -} - -func modulesFromMap(packages map[string]string) []module { - modules := make([]module, 0, len(packages)) - for moduleName, modulePath := range packages { - modules = append(modules, module{ - name: moduleName, - path: modulePath, - }) - } - slices.SortFunc(modules, func(a, b module) int { - return strings.Compare(a.name, b.name) - }) - - return modules -} From 234b807ffd20aeb1cac4f8966575d83009c105ee Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Fri, 8 Dec 2023 12:08:27 +0100 Subject: [PATCH 16/17] feat: add mod file test Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/mod_file_test.go | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 pkg/executors/golang/codegen/mod_file_test.go diff --git a/pkg/executors/golang/codegen/mod_file_test.go b/pkg/executors/golang/codegen/mod_file_test.go new file mode 100644 index 0000000..250d6d1 --- /dev/null +++ b/pkg/executors/golang/codegen/mod_file_test.go @@ -0,0 +1,247 @@ +package codegen + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestContainsModule(t *testing.T) { + t.Parallel() + + sut := actionsModFile{ + content: []string{ + "someModuleName", + "someOtherModuleName", + " someSpacedModule", + "package somePackaged", + "module someModModule", + "require someRequireModule", + }, + } + + t.Run("contains a module name", func(t *testing.T) { + actual := sut.containsModule("someModuleName") + + assert.True(t, actual) + }) + + t.Run("does not contain module", func(t *testing.T) { + actual := sut.containsModule("someNonExistingModule") + + assert.False(t, actual) + }) + + t.Run("spaced matches", func(t *testing.T) { + actual := sut.containsModule("someSpacedModule") + + assert.True(t, actual) + }) + + t.Run("packaged matches", func(t *testing.T) { + actual := sut.containsModule("somePackaged") + + assert.True(t, actual) + }) + + t.Run("modded module matches", func(t *testing.T) { + actual := sut.containsModule("someModModule") + + assert.True(t, actual) + }) + + t.Run("required module matches", func(t *testing.T) { + actual := sut.containsModule("someRequireModule") + + assert.True(t, actual) + }) + + t.Run("case sensitive doesn't match", func(t *testing.T) { + actual := sut.containsModule("SOMEMODULENAME") + + assert.False(t, actual) + }) +} + +func TestReplaceModulePath(t *testing.T) { + t.Parallel() + + createSut := func() actionsModFile { + + modFileContent := `module actions + +require ( + root_workspace v0.0.0 + subpackage v0.0.0 + othersubpackage v0.0.0 +) + +go 1.21.4 + +replace othersubpackage => ../../../../othersubpackage` + + return actionsModFile{ + path: "/some-path/some-other-path", + content: strings.Split(modFileContent, "\n"), + } + + } + + t.Run("module matches, not replaced already", func(t *testing.T) { + sut := createSut() + + expected := `module actions + +require ( + root_workspace v0.0.0 + subpackage v0.0.0 + othersubpackage v0.0.0 +) + +go 1.21.4 + +replace othersubpackage => ../../../../othersubpackage + +replace subpackage => ../subpackage` + + sut.replaceModulePath("some-other-path/newpath", module{ + name: "subpackage", + path: "subpackage", + }) + + assert.Equal(t, expected, strings.Join(sut.content, "\n")) + }) + + t.Run("module matches, not replaced already, deeper nesting", func(t *testing.T) { + sut := createSut() + + expected := `module actions + +require ( + root_workspace v0.0.0 + subpackage v0.0.0 + othersubpackage v0.0.0 +) + +go 1.21.4 + +replace othersubpackage => ../../../../othersubpackage + +replace subpackage => subpackage` + + sut.replaceModulePath("/some-path", module{ + name: "subpackage", + path: "subpackage", + }) + + assert.Equal(t, expected, strings.Join(sut.content, "\n")) + }) + + t.Run("module matches, already replaced already", func(t *testing.T) { + sut := createSut() + + expected := `module actions + +require ( + root_workspace v0.0.0 + subpackage v0.0.0 + othersubpackage v0.0.0 +) + +go 1.21.4 + +replace othersubpackage => ../othersubpackage` + + sut.replaceModulePath("some-other-path/newpath", module{ + name: "othersubpackage", + path: "othersubpackage", + }) + + assert.Equal(t, expected, strings.Join(sut.content, "\n")) + }) + + t.Run("module matches, already replaced already deeper nesting", func(t *testing.T) { + sut := createSut() + + expected := `module actions + +require ( + root_workspace v0.0.0 + subpackage v0.0.0 + othersubpackage v0.0.0 +) + +go 1.21.4 + +replace othersubpackage => othersubpackage` + + sut.replaceModulePath("/some-path", module{ + name: "othersubpackage", + path: "othersubpackage", + }) + + assert.Equal(t, expected, strings.Join(sut.content, "\n")) + }) +} + +func TestSegmentsTo(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + path string + rootDir string + expected int + }{ + { + name: "empty", + path: "", + rootDir: "", + expected: 0, + }, + { + name: "current dir", + path: "/some-dir/", + rootDir: "/some-dir/", + expected: 0, + }, + { + name: "one level", + path: "/some-dir/some-other-dir/", + rootDir: "/some-dir", + expected: 1, + }, + { + name: "2 level", + path: "/some-dir/some-other-dir/some-third-dir/", + rootDir: "/some-dir", + expected: 2, + }, + { + name: "1 level", + path: "/some-dir/some-other-dir/some-third-dir/", + rootDir: "/some-dir/some-other-dir", + expected: 1, + }, + { + name: "without trailing", + path: "/some-dir/some-third-dir", + rootDir: "/some-dir", + expected: 0, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + sut := actionsModFile{ + path: testCase.path, + } + + actual := sut.segmentsTo(testCase.rootDir) + + assert.Equal(t, testCase.expected, actual) + }) + } + +} From 9ddd5ca17b394ff7aae6a9a4a79db144c424b491 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Mon, 11 Dec 2023 09:35:40 +0100 Subject: [PATCH 17/17] chore: fix typo Signed-off-by: Kasper J. Hermansen --- pkg/executors/golang/codegen/patch.go | 2 +- pkg/executors/golang/codegen/patch_finder.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/executors/golang/codegen/patch.go b/pkg/executors/golang/codegen/patch.go index b477a79..d4c8720 100644 --- a/pkg/executors/golang/codegen/patch.go +++ b/pkg/executors/golang/codegen/patch.go @@ -15,7 +15,7 @@ type Patcher struct { func NewPatcher() *Patcher { return &Patcher{ - patchFinder: newChainedwPatchFinder( + patchFinder: newChainedPatchFinder( newWorkspaceFinder(), newGoModuleFinder(), newDefaultFinder(), diff --git a/pkg/executors/golang/codegen/patch_finder.go b/pkg/executors/golang/codegen/patch_finder.go index 379a7bf..16b5317 100644 --- a/pkg/executors/golang/codegen/patch_finder.go +++ b/pkg/executors/golang/codegen/patch_finder.go @@ -15,7 +15,7 @@ type chainedPackageFinder struct { finders []packageFinder } -func newChainedwPatchFinder(finders ...packageFinder) *chainedPackageFinder { +func newChainedPatchFinder(finders ...packageFinder) *chainedPackageFinder { return &chainedPackageFinder{ finders: finders, }