From a0dd8d53534d4d671a72a46ca33c468c8497a5b8 Mon Sep 17 00:00:00 2001 From: nitishfy Date: Sat, 7 Dec 2024 18:05:52 +0530 Subject: [PATCH 1/3] add file remove promotion step Signed-off-by: nitishfy add symlink test case Signed-off-by: nitishfy fix failing test case Signed-off-by: nitishfy convert funcs to methods Signed-off-by: nitishfy add test caes Signed-off-by: nitishfy add ui code Signed-off-by: nitishfy handle error Signed-off-by: nitishfy add docs Signed-off-by: nitishfy remove dot Signed-off-by: nitishfy --- docs/docs/35-references/10-promotion-steps.md | 47 ++++++ internal/directives/file_deleter.go | 151 ++++++++++++++++++ internal/directives/file_deleter_test.go | 151 ++++++++++++++++++ .../directives/schemas/delete-config.json | 14 ++ internal/directives/zz_config_types.go | 5 + .../registry/use-discover-registries.ts | 6 + ui/src/gen/directives/delete-config.json | 13 ++ 7 files changed, 387 insertions(+) create mode 100644 internal/directives/file_deleter.go create mode 100644 internal/directives/file_deleter_test.go create mode 100644 internal/directives/schemas/delete-config.json create mode 100644 ui/src/gen/directives/delete-config.json diff --git a/docs/docs/35-references/10-promotion-steps.md b/docs/docs/35-references/10-promotion-steps.md index 383147522..87228b1c4 100644 --- a/docs/docs/35-references/10-promotion-steps.md +++ b/docs/docs/35-references/10-promotion-steps.md @@ -300,6 +300,53 @@ steps: # Render manifests to ./out, commit, push, etc... ``` +### `delete` + +`delete` deletes the files or the directory, + +#### `delete` Configuration + +| Name | Type | Required | Description | +|-----------|------|----------|------------------------------------------| +| `path` | `string` | Y | Path to the file or directory to delete. | + +#### `delete` Example + +The most common usage of this step is to clean up unwanted files +or directories before proceeding to subsequent steps in the promotion +process. For instance, you might use the delete step to remove +intermediate configurations or artifacts generated in earlier steps. + +Consider a scenario where a Stage combines content from two working trees +to render Stage-specific manifests. If temporary files are generated during +this process, you can use the delete step to clean them up: + +```yaml +vars: +- name: gitRepo + value: https://github.com/example/repo.git +steps: +- uses: git-clone + config: + repoURL: ${{ vars.gitRepo }} + checkout: + - commit: ${{ commitFrom(vars.gitRepo, warehouse("base")).ID }} + path: ./src + - commit: ${{ commitFrom(vars.gitRepo, warehouse(ctx.stage + "-overlay")).ID }} + path: ./overlay + - branch: stage/${{ ctx.stage }} + create: true + path: ./out +- uses: copy + config: + inPath: ./overlay/stages/${{ ctx.stage }}/kustomization.yaml + outPath: ./src/stages/${{ ctx.stage }}/kustomization.yaml +- uses: delete + config: + path: ./overlay/stages/${{ ctx.stage }}/temp-file.yaml +# Render manifests to ./out, commit, push, etc... +``` + ### `kustomize-set-image` `kustomize-set-image` updates the `kustomization.yaml` file in a specified diff --git a/internal/directives/file_deleter.go b/internal/directives/file_deleter.go new file mode 100644 index 000000000..2d041dd3e --- /dev/null +++ b/internal/directives/file_deleter.go @@ -0,0 +1,151 @@ +package directives + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/xeipuuv/gojsonschema" + + kargoapi "github.com/akuity/kargo/api/v1alpha1" +) + +func init() { + builtins.RegisterPromotionStepRunner(newFileDeleter(), nil) +} + +// fileDeleter is an implementation of the PromotionStepRunner interface that +// deletes a file or directory. +type fileDeleter struct { + schemaLoader gojsonschema.JSONLoader +} + +// newFileDeleter returns an implementation of the PromotionStepRunner interface +// that deletes a file or directory. +func newFileDeleter() PromotionStepRunner { + r := &fileDeleter{} + r.schemaLoader = getConfigSchemaLoader(r.Name()) + return r +} + +// Name implements the PromotionStepRunner interface +func (f *fileDeleter) Name() string { + return "delete" +} + +// RunPromotionStep implements the PromotionStepRunner interface. +func (f *fileDeleter) RunPromotionStep( + ctx context.Context, + stepCtx *PromotionStepContext, +) (PromotionStepResult, error) { + // Validate the configuration against the JSON Schema. + if err := validate(f.schemaLoader, gojsonschema.NewGoLoader(stepCtx.Config), f.Name()); err != nil { + return PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, err + } + + // Convert the configuration into a typed object. + cfg, err := ConfigToStruct[DeleteConfig](stepCtx.Config) + if err != nil { + return PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, + fmt.Errorf("could not convert config into %s config: %w", f.Name(), err) + } + + return f.runPromotionStep(ctx, stepCtx, cfg) +} + +func (f *fileDeleter) runPromotionStep( + _ context.Context, + stepCtx *PromotionStepContext, + cfg DeleteConfig, +) (PromotionStepResult, error) { + absPath, err := f.resolveAbsPath(stepCtx.WorkDir, cfg.Path) + if err != nil { + return PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, + fmt.Errorf("could not secure join path %q: %w", cfg.Path, err) + } + + symlink, err := f.isSymlink(absPath) + if err != nil { + return PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, err + } + + if symlink { + err = os.Remove(absPath) + if err != nil { + return PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, err + } + } else { + // Secure join the paths to prevent path traversal. + pathToDelete, err := securejoin.SecureJoin(stepCtx.WorkDir, cfg.Path) + if err != nil { + return PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, + fmt.Errorf("could not secure join path %q: %w", cfg.Path, err) + } + + if err = removePath(pathToDelete); err != nil { + return PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, + fmt.Errorf("failed to delete %q: %w", cfg.Path, sanitizePathError(err, stepCtx.WorkDir)) + } + } + + return PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, nil +} + +func (f *fileDeleter) isSymlink(path string) (bool, error) { + fi, err := os.Lstat(path) + if err != nil { + // if file doesn't exist, it's not a symlink + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return fi.Mode()&os.ModeSymlink != 0, nil +} + +// resolveAbsPath resolves the absolute path from the workDir base path +func (f *fileDeleter) resolveAbsPath(workDir string, path string) (string, error) { + absBase, err := filepath.Abs(workDir) + if err != nil { + return "", err + } + + fullPath := filepath.Join(workDir, path) + absPath, err := filepath.Abs(fullPath) + if err != nil { + return "", err + } + + // Get the relative path from base to the requested path + // If the requested path tries to escape, this will return + // an error or a path starting with "../" + relPath, err := filepath.Rel(absBase, absPath) + if err != nil { + return "", err + } + + // Check if path attempts to escape + if strings.HasPrefix(relPath, "..") { + return "", errors.New("path attempts to traverse outside the working directory") + } + + return absPath, nil +} + +func removePath(path string) error { + fi, err := os.Lstat(path) + if err != nil { + return err + } + + if fi.IsDir() { + return os.RemoveAll(path) + } + + return os.Remove(path) +} diff --git a/internal/directives/file_deleter_test.go b/internal/directives/file_deleter_test.go new file mode 100644 index 000000000..473ce7cc2 --- /dev/null +++ b/internal/directives/file_deleter_test.go @@ -0,0 +1,151 @@ +package directives + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + kargoapi "github.com/akuity/kargo/api/v1alpha1" +) + +func Test_fileDeleter_runPromotionStep(t *testing.T) { + tests := []struct { + name string + setupFiles func(*testing.T) string + cfg DeleteConfig + assertions func(*testing.T, string, PromotionStepResult, error) + }{ + { + name: "succeeds deleting file", + setupFiles: func(t *testing.T) string { + tmpDir := t.TempDir() + + path := filepath.Join(tmpDir, "input.txt") + require.NoError(t, os.WriteFile(path, []byte("test content"), 0o600)) + + return tmpDir + }, + cfg: DeleteConfig{ + Path: "input.txt", + }, + assertions: func(t *testing.T, _ string, result PromotionStepResult, err error) { + assert.NoError(t, err) + assert.Equal(t, PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, result) + + _, statError := os.Stat("input.txt") + assert.True(t, os.IsNotExist(statError)) + }, + }, + { + name: "succeeds deleting directory", + setupFiles: func(t *testing.T) string { + tmpDir := t.TempDir() + dirPath := filepath.Join(tmpDir, "dirToDelete") + require.NoError(t, os.Mkdir(dirPath, 0o700)) + return tmpDir + }, + cfg: DeleteConfig{ + Path: "dirToDelete", + }, + assertions: func(t *testing.T, workDir string, result PromotionStepResult, err error) { + assert.NoError(t, err) + assert.Equal(t, PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, result) + + _, statErr := os.Stat(filepath.Join(workDir, "dirToDelete")) + assert.True(t, os.IsNotExist(statErr)) + }, + }, + { + name: "fails for non-existent path", + setupFiles: func(t *testing.T) string { + return t.TempDir() + }, + cfg: DeleteConfig{ + Path: "nonExistentFile.txt", + }, + assertions: func(t *testing.T, _ string, result PromotionStepResult, err error) { + assert.Error(t, err) + assert.Equal(t, PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, result) + }, + }, + { + name: "removes symlink only", + setupFiles: func(t *testing.T) string { + tmpDir := t.TempDir() + + inDir := filepath.Join(tmpDir, "input") + require.NoError(t, os.Mkdir(inDir, 0o755)) + + filePath := filepath.Join(inDir, "input.txt") + require.NoError(t, os.WriteFile(filePath, []byte("test content"), 0o600)) + + symlinkPath := filepath.Join(inDir, "symlink.txt") + require.NoError(t, os.Symlink("input.txt", symlinkPath)) + + return tmpDir + }, + cfg: DeleteConfig{ + Path: "input/symlink.txt", + }, + assertions: func(t *testing.T, workDir string, result PromotionStepResult, err error) { + assert.NoError(t, err) + require.Equal(t, PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, result) + + _, statErr := os.Stat(filepath.Join(workDir, "input", "input.txt")) + assert.NoError(t, statErr) + + _, statErr = os.Lstat(filepath.Join(workDir, "input", "symlink.txt")) + assert.Error(t, statErr) + assert.True(t, os.IsNotExist(statErr)) + }, + }, + { + name: "removes a file within a symlink", + setupFiles: func(t *testing.T) string { + tmpDir := t.TempDir() + + inDir := filepath.Join(tmpDir, "bar") + require.NoError(t, os.Mkdir(inDir, 0o755)) + + filePath := filepath.Join(inDir, "file.txt") + require.NoError(t, os.WriteFile(filePath, []byte("test content"), 0o600)) + + symlinkPath := filepath.Join(tmpDir, "foo") + require.NoError(t, os.Symlink(inDir, symlinkPath)) + + return tmpDir + }, + cfg: DeleteConfig{ + Path: "foo/", + }, + assertions: func(t *testing.T, workDir string, result PromotionStepResult, err error) { + assert.NoError(t, err) + require.Equal(t, PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, result) + + _, statErr := os.Stat(filepath.Join(workDir, "foo", "file.txt")) + assert.Error(t, statErr) + assert.True(t, os.IsNotExist(statErr)) + + _, statErr = os.Stat(filepath.Join(workDir, "bar", "file.txt")) + assert.NoError(t, statErr) + }, + }, + } + runner := &fileDeleter{} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workDir := tt.setupFiles(t) + result, err := runner.runPromotionStep( + context.Background(), + &PromotionStepContext{WorkDir: workDir}, + tt.cfg, + ) + tt.assertions(t, workDir, result, err) + }) + } +} diff --git a/internal/directives/schemas/delete-config.json b/internal/directives/schemas/delete-config.json new file mode 100644 index 000000000..31a79358f --- /dev/null +++ b/internal/directives/schemas/delete-config.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "DeleteConfig", + "type": "object", + "additionalProperties": false, + "required": ["path"], + "properties": { + "path": { + "type": "string", + "description": "Path is the path to the file or directory to delete.", + "minLength": 1 + } + } +} diff --git a/internal/directives/zz_config_types.go b/internal/directives/zz_config_types.go index 6a9f964de..c200d7d3b 100644 --- a/internal/directives/zz_config_types.go +++ b/internal/directives/zz_config_types.go @@ -117,6 +117,11 @@ type CopyConfig struct { OutPath string `json:"outPath"` } +type DeleteConfig struct { + // Path is the path to the file or directory to delete. + Path string `json:"path"` +} + type GitClearConfig struct { // Path to a working directory of a local repository from which to remove all files, // excluding the .git/ directory. diff --git a/ui/src/features/promotion-directives/registry/use-discover-registries.ts b/ui/src/features/promotion-directives/registry/use-discover-registries.ts index 36e3351be..70572c5f1 100644 --- a/ui/src/features/promotion-directives/registry/use-discover-registries.ts +++ b/ui/src/features/promotion-directives/registry/use-discover-registries.ts @@ -29,6 +29,7 @@ import { JSONSchema7 } from 'json-schema'; // IMPORTANT(Marvin9): this must be replaced with proper discovery mechanism import argocdUpdateConfig from '@ui/gen/directives/argocd-update-config.json'; import copyConfig from '@ui/gen/directives/copy-config.json'; +import deleteConfig from '@ui/gen/directives/delete-config.json'; import gitOverwriteConfig from '@ui/gen/directives/git-clear-config.json'; import gitCloneConfig from '@ui/gen/directives/git-clone-config.json'; import gitCommitConfig from '@ui/gen/directives/git-commit-config.json'; @@ -61,6 +62,11 @@ export const useDiscoverPromotionDirectivesRegistries = (): PromotionDirectivesR unstable_icons: [faCopy], config: copyConfig as JSONSchema7 }, + { + identifier: 'delete', + unstable_icons: [], + config: deleteConfig as JSONSchema7 + }, { identifier: 'git-clone', unstable_icons: [faCodeBranch, faClone], diff --git a/ui/src/gen/directives/delete-config.json b/ui/src/gen/directives/delete-config.json new file mode 100644 index 000000000..21df07078 --- /dev/null +++ b/ui/src/gen/directives/delete-config.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "DeleteConfig", + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "description": "Path is the path to the file or directory to delete.", + "minLength": 1 + } + } +} \ No newline at end of file From 425b82a2048046a6446bd94c61e36c28d0bcd97b Mon Sep 17 00:00:00 2001 From: Nitish Kumar Date: Thu, 19 Dec 2024 22:18:39 +0530 Subject: [PATCH 2/3] Update docs/docs/35-references/10-promotion-steps.md Co-authored-by: Hidde Beydals --- docs/docs/35-references/10-promotion-steps.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/35-references/10-promotion-steps.md b/docs/docs/35-references/10-promotion-steps.md index 87228b1c4..369090445 100644 --- a/docs/docs/35-references/10-promotion-steps.md +++ b/docs/docs/35-references/10-promotion-steps.md @@ -302,7 +302,7 @@ steps: ### `delete` -`delete` deletes the files or the directory, +`delete` deletes a file or directory. #### `delete` Configuration From df9127cd388f6b98679840e1c0f92f9bc475b494 Mon Sep 17 00:00:00 2001 From: Kent Rancourt Date: Thu, 19 Dec 2024 20:06:45 -0500 Subject: [PATCH 3/3] Update docs/docs/35-references/10-promotion-steps.md Signed-off-by: Kent Rancourt --- docs/docs/35-references/10-promotion-steps.md | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/docs/docs/35-references/10-promotion-steps.md b/docs/docs/35-references/10-promotion-steps.md index 369090445..33901e42e 100644 --- a/docs/docs/35-references/10-promotion-steps.md +++ b/docs/docs/35-references/10-promotion-steps.md @@ -312,14 +312,8 @@ steps: #### `delete` Example -The most common usage of this step is to clean up unwanted files -or directories before proceeding to subsequent steps in the promotion -process. For instance, you might use the delete step to remove -intermediate configurations or artifacts generated in earlier steps. - -Consider a scenario where a Stage combines content from two working trees -to render Stage-specific manifests. If temporary files are generated during -this process, you can use the delete step to clean them up: +One, common usage of this step is to remove intermediate files +produced by the promotion process prior to a `git-commit` step: ```yaml vars: @@ -330,21 +324,20 @@ steps: config: repoURL: ${{ vars.gitRepo }} checkout: - - commit: ${{ commitFrom(vars.gitRepo, warehouse("base")).ID }} + - commit: ${{ commitFrom(vars.gitRepo) }} path: ./src - - commit: ${{ commitFrom(vars.gitRepo, warehouse(ctx.stage + "-overlay")).ID }} - path: ./overlay - branch: stage/${{ ctx.stage }} create: true path: ./out -- uses: copy - config: - inPath: ./overlay/stages/${{ ctx.stage }}/kustomization.yaml - outPath: ./src/stages/${{ ctx.stage }}/kustomization.yaml + +# Steps that produce intermediate files in ./out... + - uses: delete config: - path: ./overlay/stages/${{ ctx.stage }}/temp-file.yaml -# Render manifests to ./out, commit, push, etc... + path: ./out/unwanted/file +- uses: git-commit + config: + path: ./out ``` ### `kustomize-set-image`