From eb125fe3dcd8674376b9c332605afc58460e13b9 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sun, 17 Nov 2024 17:50:14 +0900 Subject: [PATCH 1/6] Implementing `secrets:` section --- book.go | 1 + debugger_test.go | 41 ++++++++ dump.go | 6 +- dump_test.go | 119 +++++++++++++++++++++- exec.go | 5 +- exec_test.go | 89 ++++++++++++++++ go.mod | 2 +- go.sum | 4 +- operator.go | 8 +- operator_test.go | 2 +- option.go | 19 ++++ store.go | 28 ++++- testdata/book/with_secrets.yml | 19 ++++ testdata/with_secrets.yml.debugger.golden | 30 ++++++ 14 files changed, 359 insertions(+), 14 deletions(-) create mode 100644 testdata/book/with_secrets.yml create mode 100644 testdata/with_secrets.yml.debugger.golden diff --git a/book.go b/book.go index 2e49cc98..e2fec03a 100644 --- a/book.go +++ b/book.go @@ -591,6 +591,7 @@ func (bk *book) merge(loaded *book) error { for k, v := range loaded.vars { bk.vars[k] = v } + bk.secrets = append(bk.secrets, loaded.secrets...) bk.runnerErrs = loaded.runnerErrs bk.rawSteps = loaded.rawSteps bk.hostRules = loaded.hostRules diff --git a/debugger_test.go b/debugger_test.go index 0c99eff3..8e427bd7 100644 --- a/debugger_test.go +++ b/debugger_test.go @@ -114,3 +114,44 @@ func TestDebuggerWithStderr(t *testing.T) { }) } } + +func TestDebuggerWithSecrets(t *testing.T) { + noColor(t) + tests := []struct { + book string + }{ + {"testdata/book/with_secrets.yml"}, + } + ctx := context.Background() + for _, tt := range tests { + t.Run(tt.book, func(t *testing.T) { + out := new(bytes.Buffer) + hs := testutil.HTTPServer(t) + opts := []Option{ + Book(tt.book), + HTTPRunner("req", hs.URL, hs.Client()), + Stderr(out), + Scopes(ScopeAllowRunExec), + } + o, err := New(opts...) + if err != nil { + t.Fatal(err) + } + if err := o.Run(ctx); err != nil { + t.Error(err) + } + + got := out.String() + + f := fmt.Sprintf("%s.debugger", filepath.Base(tt.book)) + if os.Getenv("UPDATE_GOLDEN") != "" { + golden.Update(t, "testdata", f, got) + return + } + + if diff := golden.Diff(t, "testdata", f, got); diff != "" { + t.Error(diff) + } + }) + } +} diff --git a/dump.go b/dump.go index a172fe26..ee7d8829 100644 --- a/dump.go +++ b/dump.go @@ -9,6 +9,7 @@ import ( "reflect" "github.com/goccy/go-json" + "github.com/k1LoW/donegroup" ) const dumpRunnerKey = "dump" @@ -53,7 +54,10 @@ func (rnr *dumpRunner) Run(ctx context.Context, s *step, first bool) error { if err != nil { return err } - out = f + donegroup.Cleanup(ctx, func() error { + return f.Close() + }) + out = o.maskRule.NewWriter(f) default: return fmt.Errorf("invalid dump out: %v", pp) } diff --git a/dump_test.go b/dump_test.go index 4b33518f..f11568a9 100644 --- a/dump_test.go +++ b/dump_test.go @@ -7,6 +7,8 @@ import ( "os" "path/filepath" "testing" + + "github.com/k1LoW/maskedio" ) func TestDumpRunnerRun(t *testing.T) { @@ -147,7 +149,7 @@ func TestDumpRunnerRun(t *testing.T) { } buf := new(bytes.Buffer) o.store = &tt.store - o.stdout = buf + o.stdout = maskedio.NewWriter(buf) o.useMap = tt.store.useMap o.steps = tt.steps d := newDumpRunner() @@ -406,3 +408,118 @@ func TestDumpRunnerRunWithExpandOut(t *testing.T) { }) } } + +func TestDumpRunnerRunWithSecrets(t *testing.T) { + tests := []struct { + store store + expr string + disableNL bool + steps []*step + secrets []string + want string + }{ + { + store{ + steps: []map[string]any{}, + vars: map[string]any{ + "key": "value", + }, + }, + "vars.key", + false, + nil, + []string{"vars.key"}, + `***** +`, + }, + { + store{ + steps: []map[string]any{}, + vars: map[string]any{ + "key": "value", + }, + }, + "vars", + false, + nil, + []string{"vars.key"}, + `{ + "key": "*****" +} +`, + }, + { + store{ + steps: []map[string]any{ + { + "key": "value", + }, + }, + vars: map[string]any{}, + }, + "steps", + false, + nil, + []string{"steps[0].key"}, + `[ + { + "key": "*****" + } +] +`, + }, + { + store{ + steps: []map[string]any{}, + stepMap: map[string]map[string]any{ + "stepkey": {"key": "value"}, + }, + vars: map[string]any{}, + useMap: true, + stepMapKeys: []string{"stepkey", "stepnext"}, + }, + "steps", + false, + []*step{ + {key: "stepkey"}, + {key: "stepnext"}, + }, + []string{"steps.stepkey.key"}, + `{ + "stepkey": { + "key": "*****" + } +} +`, + }, + } + ctx := context.Background() + for i, tt := range tests { + t.Run(fmt.Sprintf("%d.%s", i, tt.expr), func(t *testing.T) { + buf := new(bytes.Buffer) + o, err := New(Stdout(buf)) + if err != nil { + t.Fatal(err) + } + tt.store.secrets = tt.secrets + tt.store.mr = o.store.mr + tt.store.setMaskKeywords(tt.store.toMap()) + o.store = &tt.store + o.useMap = tt.store.useMap + o.steps = tt.steps + d := newDumpRunner() + s := newStep(0, "stepKey", o, nil) + s.dumpRequest = &dumpRequest{ + expr: tt.expr, + disableTrailingNewline: tt.disableNL, + } + if err := d.Run(ctx, s, true); err != nil { + t.Fatal(err) + } + got := buf.String() + if got != tt.want { + t.Errorf("got\n%#v\nwant\n%#v", got, tt.want) + } + }) + } +} diff --git a/exec.go b/exec.go index 061404ee..78402222 100644 --- a/exec.go +++ b/exec.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "os" "strings" "github.com/cli/safeexec" @@ -84,8 +83,8 @@ func (rnr *execRunner) run(ctx context.Context, c *execCommand, s *step) error { o.capturers.captureExecStdin(c.stdin) } if c.liveOutput { - cmd.Stdout = io.MultiWriter(stdout, os.Stdout) - cmd.Stderr = io.MultiWriter(stderr, os.Stderr) + cmd.Stdout = io.MultiWriter(stdout, o.maskRule.NewWriter(o.stdout)) + cmd.Stderr = io.MultiWriter(stderr, o.maskRule.NewWriter(o.stderr)) } else { cmd.Stdout = stdout cmd.Stderr = stderr diff --git a/exec_test.go b/exec_test.go index c102eaf5..2c296e8b 100644 --- a/exec_test.go +++ b/exec_test.go @@ -1,6 +1,7 @@ package runn import ( + "bytes" "context" "fmt" "strings" @@ -107,3 +108,91 @@ func TestExecShell(t *testing.T) { }) } } + +func TestExecRunWithSecrets(t *testing.T) { + if err := setScopes(ScopeAllowRunExec); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := setScopes(ScopeDenyRunExec); err != nil { + t.Fatal(err) + } + }) + tests := []struct { + vars map[string]any + secrets []string + command string + liveOutput bool + want map[string]any + wantStdout string + wantStderr string + }{ + { + map[string]any{"message": "hello"}, + []string{"vars.message"}, + "echo hello!!", + false, + map[string]any{ + "stdout": "hello!!\n", + "stderr": "", + "exit_code": 0, + }, + "", + "", + }, + { + map[string]any{"message": "hello"}, + []string{"vars.message"}, + "echo hello!!", + true, + map[string]any{ + "stdout": "hello!!\n", + "stderr": "", + "exit_code": 0, + }, + "*****!!\n", + "", + }, + } + for _, tt := range tests { + t.Run(tt.command, func(t *testing.T) { + ctx, cancel := donegroup.WithCancel(context.Background()) + t.Cleanup(cancel) + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + opts := []Option{ + Stdout(stdout), + Stderr(stderr), + Secret(tt.secrets...), + } + for k, v := range tt.vars { + opts = append(opts, Var(k, v)) + } + + o, err := New(opts...) + if err != nil { + t.Fatal(err) + } + r := newExecRunner() + s := newStep(0, "stepKey", o, nil) + c := &execCommand{command: tt.command, liveOutput: tt.liveOutput} + if err := r.run(ctx, c, s); err != nil { + t.Error(err) + return + } + got := o.store.steps[0] + if diff := cmp.Diff(got, tt.want, nil); diff != "" { + t.Error(diff) + } + + gotStdout := stdout.String() + if gotStdout != tt.wantStdout { + t.Errorf("got %s, want %s", gotStdout, tt.wantStdout) + } + gotStderr := stderr.String() + if gotStderr != tt.wantStderr { + t.Errorf("got %s, want %s", gotStderr, tt.wantStderr) + } + }) + } +} diff --git a/go.mod b/go.mod index d802a2cb..084e04d6 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/k1LoW/grpcstub v0.24.1 github.com/k1LoW/grpcurlreq v0.2.1 github.com/k1LoW/httpstub v0.17.0 - github.com/k1LoW/maskedio v0.3.0 + github.com/k1LoW/maskedio v0.4.0 github.com/k1LoW/protoresolv v0.1.1 github.com/k1LoW/repin v0.3.4 github.com/k1LoW/sshc/v4 v4.2.0 diff --git a/go.sum b/go.sum index d20ea7eb..11e86cbd 100644 --- a/go.sum +++ b/go.sum @@ -1028,8 +1028,8 @@ github.com/k1LoW/grpcurlreq v0.2.1 h1:vV5XPcudceX5+DvnEvyDY11E/MkJ01RmYoHO3/8aNj github.com/k1LoW/grpcurlreq v0.2.1/go.mod h1:fYHtOo6/5ans9ub1CDRIQQfvo4NyTGwPud+FH/Rnx90= github.com/k1LoW/httpstub v0.17.0 h1:VOeyfSriYbHQL1wGk9sfqa8TDpP6Zti/MPT3xcYDtF0= github.com/k1LoW/httpstub v0.17.0/go.mod h1:r9Cu3m/G+ILM8jjPebA/P4dXbIH8s8jnPVrhFETAFYA= -github.com/k1LoW/maskedio v0.3.0 h1:MTtPgMC47v+VhX2i8c/XLMtAlH7E5RFC+oSjh8f22cA= -github.com/k1LoW/maskedio v0.3.0/go.mod h1:ugf3OzV7imakceGtmkOY13oJsDynL2YVXRcn9VUZEiE= +github.com/k1LoW/maskedio v0.4.0 h1:YmPBXo+NOOzucZx8K4k1i63YUz7nwbG1Hya8PUQnRiA= +github.com/k1LoW/maskedio v0.4.0/go.mod h1:ugf3OzV7imakceGtmkOY13oJsDynL2YVXRcn9VUZEiE= github.com/k1LoW/protoresolv v0.1.1 h1:3Q5Sl4bOGyXy9etMZJraF60XqPQBqodFMfGoqBEBwYE= github.com/k1LoW/protoresolv v0.1.1/go.mod h1:eVklYuDOPjSNzmbHZiNmXwmXX4OHqscY0MduCGQbSK0= github.com/k1LoW/repin v0.3.4 h1:xcNuBBc/ISHUNBzjXNTCux4OYZND5ZMiyz4SrRtpDhg= diff --git a/operator.go b/operator.go index 2ab88b11..224a37a5 100644 --- a/operator.go +++ b/operator.go @@ -72,8 +72,8 @@ type operator struct { ifCond string skipTest bool skipped bool - stdout io.Writer - stderr io.Writer + stdout *maskedio.Writer + stderr *maskedio.Writer newOnly bool // Skip some errors for `runn list` bookPath string numberOfSteps int // Number of steps for `runn list` @@ -457,8 +457,8 @@ func New(opts ...Option) (*operator, error) { included: bk.included, ifCond: bk.ifCond, skipTest: bk.skipTest, - stdout: bk.stdout, - stderr: bk.stderr, + stdout: st.maskRule().NewWriter(bk.stdout), + stderr: st.maskRule().NewWriter(bk.stderr), newOnly: bk.loadOnly, bookPath: bk.path, beforeFuncs: bk.beforeFuncs, diff --git a/operator_test.go b/operator_test.go index c7871e18..07f3ef8c 100644 --- a/operator_test.go +++ b/operator_test.go @@ -970,7 +970,7 @@ func TestShard(t *testing.T) { cmp.AllowUnexported(allow...), cmpopts.IgnoreUnexported(ignore...), cmpopts.IgnoreFields(stopw.Span{}, "ID"), - cmpopts.IgnoreFields(operator{}, "id", "concurrency", "mu", "dbg", "needs", "nm", "maskRule"), + cmpopts.IgnoreFields(operator{}, "id", "concurrency", "mu", "dbg", "needs", "nm", "maskRule", "stdout", "stderr"), cmpopts.IgnoreFields(cdpRunner{}, "ctx", "cancel", "opts", "mu", "operatorID"), cmpopts.IgnoreFields(sshRunner{}, "client", "sess", "stdin", "stdout", "stderr", "operatorID"), cmpopts.IgnoreFields(grpcRunner{}, "mu", "operatorID"), diff --git a/option.go b/option.go index 39b0fadd..598fcd18 100644 --- a/option.go +++ b/option.go @@ -699,6 +699,25 @@ func Var(k any, v any) Option { } } +// Secret - Set secret var names to be masked. +func Secret(secrets ...string) Option { + return func(bk *book) error { + if bk == nil { + return ErrNilBook + } + for _, secret := range secrets { + if strings.HasPrefix(secret, storeRootKeyCurrent+".") { + return fmt.Errorf("secrets: does not support 'current.': %s", secret) + } + if strings.HasPrefix(secret, storeRootKeyPrevious+".") { + return fmt.Errorf("secrets: does not support 'previous.': %s", secret) + } + } + bk.secrets = append(bk.secrets, secrets...) + return nil + } +} + // Func - Set function to runner. func Func(k string, v any) Option { return func(bk *book) error { diff --git a/store.go b/store.go index e3d77380..abf53fc0 100644 --- a/store.go +++ b/store.go @@ -8,6 +8,7 @@ import ( "time" "github.com/k1LoW/maskedio" + "github.com/spf13/cast" ) const ( @@ -73,7 +74,7 @@ type store struct { } func newStore(vars, funcs map[string]any, secrets []string, useMap bool, stepMapKeys []string) *store { - return &store{ + s := &store{ steps: []map[string]any{}, stepMap: map[string]map[string]any{}, stepMapKeys: stepMapKeys, @@ -87,6 +88,9 @@ func newStore(vars, funcs map[string]any, secrets []string, useMap bool, stepMap secrets: secrets, mr: maskedio.NewRule(), } + s.setMaskKeywords(s.toMap()) + + return s } func (s *store) record(v map[string]any) { @@ -250,6 +254,8 @@ func (s *store) toMap() map[string]any { runnm[storeRunnKeyRunNIndex] = s.runNIndex store[storeRootKeyRunn] = runnm + s.setMaskKeywords(store) + return store } @@ -293,6 +299,8 @@ func (s *store) toMapForIncludeRunner() map[string]any { store[storeRootKeyRunn] = runnm + s.setMaskKeywords(store) + return store } @@ -342,9 +350,27 @@ func (s *store) toMapForDbg() map[string]any { runnm[storeRunnKeyRunNIndex] = s.runNIndex store[storeRootKeyRunn] = runnm + s.setMaskKeywords(store) + return store } +func (s *store) setMaskKeywords(store map[string]any) { + for _, key := range s.secrets { + v, err := Eval(key, store) + if err != nil { + continue + } + + switch vv := v.(type) { + case map[string]any: + case []any: + default: + s.maskRule().SetKeyword(cast.ToString(vv)) + } + } +} + func (s *store) clearSteps() { s.steps = []map[string]any{} s.stepMap = map[string]map[string]any{} diff --git a/testdata/book/with_secrets.yml b/testdata/book/with_secrets.yml new file mode 100644 index 00000000..d0e47ac5 --- /dev/null +++ b/testdata/book/with_secrets.yml @@ -0,0 +1,19 @@ +desc: With secrets +debug: true +vars: + message: hello +secrets: + - vars.message +steps: + - + desc: 'Print "hello world!!"' + exec: + command: echo hello world!! + - + desc: 'Print "hello world!!" again' + exec: + command: cat + stdin: '{{ steps[0].stdout }}' + - + desc: 'Check result of previous command contains "hello"' + test: 'steps[1].stdout contains "hello"' diff --git a/testdata/with_secrets.yml.debugger.golden b/testdata/with_secrets.yml.debugger.golden new file mode 100644 index 00000000..34fa73ea --- /dev/null +++ b/testdata/with_secrets.yml.debugger.golden @@ -0,0 +1,30 @@ +Run "Print \"***** world!!\"" on "With secrets".steps[0] +-----START COMMAND----- +echo ***** world!! +-----END COMMAND----- +-----START STDOUT----- +***** world!! + +-----END STDOUT----- +-----START STDERR----- + +-----END STDERR----- + +Run "Print \"***** world!!\" again" on "With secrets".steps[1] +-----START COMMAND----- +cat +-----END COMMAND----- +-----START STDIN----- +***** world!! + +-----END STDIN----- +-----START STDOUT----- +***** world!! + +-----END STDOUT----- +-----START STDERR----- + +-----END STDERR----- + +Run "Check result of previous command contains \"*****\"" on "With secrets".steps[2] +Run "test" on "With secrets".steps[2] From 618ebd25d06e2558961bdcc71335574f593d3f5c Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sun, 17 Nov 2024 18:16:02 +0900 Subject: [PATCH 2/6] Fix lint warn --- dump.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dump.go b/dump.go index ee7d8829..240f436a 100644 --- a/dump.go +++ b/dump.go @@ -54,9 +54,11 @@ func (rnr *dumpRunner) Run(ctx context.Context, s *step, first bool) error { if err != nil { return err } - donegroup.Cleanup(ctx, func() error { + if err := donegroup.Cleanup(ctx, func() error { return f.Close() - }) + }); err != nil { + return err + } out = o.maskRule.NewWriter(f) default: return fmt.Errorf("invalid dump out: %v", pp) From 31ed89aff0b12136551d4ac1d1a218c5fbaee7ad Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sun, 17 Nov 2024 19:20:55 +0900 Subject: [PATCH 3/6] Fix --- dump_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/dump_test.go b/dump_test.go index f11568a9..1e4f7ba3 100644 --- a/dump_test.go +++ b/dump_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "github.com/k1LoW/donegroup" "github.com/k1LoW/maskedio" ) @@ -140,9 +141,10 @@ func TestDumpRunnerRun(t *testing.T) { `hello`, }, } - ctx := context.Background() for i, tt := range tests { t.Run(fmt.Sprintf("%d.%s", i, tt.expr), func(t *testing.T) { + ctx, cancel := donegroup.WithCancel(context.Background()) + t.Cleanup(cancel) o, err := New() if err != nil { t.Fatal(err) @@ -298,9 +300,11 @@ func TestDumpRunnerRunWithOut(t *testing.T) { `hello`, }, } - ctx := context.Background() + for i, tt := range tests { t.Run(fmt.Sprintf("%d.%s with out", i, tt.expr), func(t *testing.T) { + ctx, cancel := donegroup.WithCancel(context.Background()) + t.Cleanup(cancel) p := filepath.Join(t.TempDir(), "tmp") o, err := New() if err != nil { @@ -385,9 +389,11 @@ func TestDumpRunnerRunWithExpandOut(t *testing.T) { filepath.Join(tmp, "value3.ext"), }, } - ctx := context.Background() + for _, tt := range tests { t.Run(tt.out, func(t *testing.T) { + ctx, cancel := donegroup.WithCancel(context.Background()) + t.Cleanup(cancel) o, err := New() if err != nil { t.Fatal(err) @@ -493,9 +499,11 @@ func TestDumpRunnerRunWithSecrets(t *testing.T) { `, }, } - ctx := context.Background() + for i, tt := range tests { t.Run(fmt.Sprintf("%d.%s", i, tt.expr), func(t *testing.T) { + ctx, cancel := donegroup.WithCancel(context.Background()) + t.Cleanup(cancel) buf := new(bytes.Buffer) o, err := New(Stdout(buf)) if err != nil { From 9436ce5aea420cf71926907ede6a4a9c3641dc5b Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sun, 17 Nov 2024 20:48:01 +0900 Subject: [PATCH 4/6] Add disableMaskingSecrets option for Dump runner --- dump.go | 13 +++++++++++-- dump_test.go | 38 ++++++++++++++++++++++++++------------ go.mod | 2 +- go.sum | 4 ++-- operator.go | 5 +++++ 5 files changed, 45 insertions(+), 17 deletions(-) diff --git a/dump.go b/dump.go index 240f436a..cae113a5 100644 --- a/dump.go +++ b/dump.go @@ -20,6 +20,7 @@ type dumpRequest struct { expr string out string disableTrailingNewline bool + disableMaskingSecrets bool } func newDumpRunner() *dumpRunner { @@ -39,7 +40,11 @@ func (rnr *dumpRunner) Run(ctx context.Context, s *step, first bool) error { store[storeRootKeyCurrent] = o.store.latest() } if r.out == "" { - out = o.stdout + if r.disableMaskingSecrets { + out = o.stdout.Unwrap() + } else { + out = o.stdout + } } else { p, err := EvalExpand(r.out, store) if err != nil { @@ -59,7 +64,11 @@ func (rnr *dumpRunner) Run(ctx context.Context, s *step, first bool) error { }); err != nil { return err } - out = o.maskRule.NewWriter(f) + if r.disableMaskingSecrets { + out = f + } else { + out = o.maskRule.NewWriter(f) + } default: return fmt.Errorf("invalid dump out: %v", pp) } diff --git a/dump_test.go b/dump_test.go index 1e4f7ba3..1eb32d63 100644 --- a/dump_test.go +++ b/dump_test.go @@ -417,12 +417,12 @@ func TestDumpRunnerRunWithExpandOut(t *testing.T) { func TestDumpRunnerRunWithSecrets(t *testing.T) { tests := []struct { - store store - expr string - disableNL bool - steps []*step - secrets []string - want string + store store + expr string + steps []*step + secrets []string + disableMaskingSecrets bool + want string }{ { store{ @@ -432,9 +432,9 @@ func TestDumpRunnerRunWithSecrets(t *testing.T) { }, }, "vars.key", - false, nil, []string{"vars.key"}, + false, `***** `, }, @@ -446,9 +446,9 @@ func TestDumpRunnerRunWithSecrets(t *testing.T) { }, }, "vars", - false, nil, []string{"vars.key"}, + false, `{ "key": "*****" } @@ -464,9 +464,9 @@ func TestDumpRunnerRunWithSecrets(t *testing.T) { vars: map[string]any{}, }, "steps", - false, nil, []string{"steps[0].key"}, + false, `[ { "key": "*****" @@ -485,17 +485,31 @@ func TestDumpRunnerRunWithSecrets(t *testing.T) { stepMapKeys: []string{"stepkey", "stepnext"}, }, "steps", - false, []*step{ {key: "stepkey"}, {key: "stepnext"}, }, []string{"steps.stepkey.key"}, + false, `{ "stepkey": { "key": "*****" } } +`, + }, + { + store{ + steps: []map[string]any{}, + vars: map[string]any{ + "key": "value", + }, + }, + "vars.key", + nil, + []string{"vars.key"}, + true, + `value `, }, } @@ -518,8 +532,8 @@ func TestDumpRunnerRunWithSecrets(t *testing.T) { d := newDumpRunner() s := newStep(0, "stepKey", o, nil) s.dumpRequest = &dumpRequest{ - expr: tt.expr, - disableTrailingNewline: tt.disableNL, + expr: tt.expr, + disableMaskingSecrets: tt.disableMaskingSecrets, } if err := d.Run(ctx, s, true); err != nil { t.Fatal(err) diff --git a/go.mod b/go.mod index 084e04d6..e24d3964 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/k1LoW/grpcstub v0.24.1 github.com/k1LoW/grpcurlreq v0.2.1 github.com/k1LoW/httpstub v0.17.0 - github.com/k1LoW/maskedio v0.4.0 + github.com/k1LoW/maskedio v0.4.2 github.com/k1LoW/protoresolv v0.1.1 github.com/k1LoW/repin v0.3.4 github.com/k1LoW/sshc/v4 v4.2.0 diff --git a/go.sum b/go.sum index 11e86cbd..61a2441b 100644 --- a/go.sum +++ b/go.sum @@ -1028,8 +1028,8 @@ github.com/k1LoW/grpcurlreq v0.2.1 h1:vV5XPcudceX5+DvnEvyDY11E/MkJ01RmYoHO3/8aNj github.com/k1LoW/grpcurlreq v0.2.1/go.mod h1:fYHtOo6/5ans9ub1CDRIQQfvo4NyTGwPud+FH/Rnx90= github.com/k1LoW/httpstub v0.17.0 h1:VOeyfSriYbHQL1wGk9sfqa8TDpP6Zti/MPT3xcYDtF0= github.com/k1LoW/httpstub v0.17.0/go.mod h1:r9Cu3m/G+ILM8jjPebA/P4dXbIH8s8jnPVrhFETAFYA= -github.com/k1LoW/maskedio v0.4.0 h1:YmPBXo+NOOzucZx8K4k1i63YUz7nwbG1Hya8PUQnRiA= -github.com/k1LoW/maskedio v0.4.0/go.mod h1:ugf3OzV7imakceGtmkOY13oJsDynL2YVXRcn9VUZEiE= +github.com/k1LoW/maskedio v0.4.2 h1:8s1qIWEJHvi8oEXq7fl72SabWTDK634OcxhGlEnv/GY= +github.com/k1LoW/maskedio v0.4.2/go.mod h1:ugf3OzV7imakceGtmkOY13oJsDynL2YVXRcn9VUZEiE= github.com/k1LoW/protoresolv v0.1.1 h1:3Q5Sl4bOGyXy9etMZJraF60XqPQBqodFMfGoqBEBwYE= github.com/k1LoW/protoresolv v0.1.1/go.mod h1:eVklYuDOPjSNzmbHZiNmXwmXX4OHqscY0MduCGQbSK0= github.com/k1LoW/repin v0.3.4 h1:xcNuBBc/ISHUNBzjXNTCux4OYZND5ZMiyz4SrRtpDhg= diff --git a/operator.go b/operator.go index 224a37a5..cacb0d18 100644 --- a/operator.go +++ b/operator.go @@ -722,10 +722,15 @@ func (op *operator) appendStep(idx int, key string, s map[string]any) error { if !ok { disableNL = false } + disableMask, ok := vv["disableMaskingSecrets"] + if !ok { + disableMask = false + } step.dumpRequest = &dumpRequest{ expr: cast.ToString(expr), out: cast.ToString(out), disableTrailingNewline: cast.ToBool(disableNL), + disableMaskingSecrets: cast.ToBool(disableMask), } default: return fmt.Errorf("invalid dump request: %v", vv) From 569598f1f8071c117c146573a16d57cd532af893 Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sun, 17 Nov 2024 21:25:20 +0900 Subject: [PATCH 5/6] Support `current.` --- dump_test.go | 24 ++++++++++++++++++++++++ option.go | 3 --- store.go | 5 +++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/dump_test.go b/dump_test.go index 1eb32d63..8fa994aa 100644 --- a/dump_test.go +++ b/dump_test.go @@ -510,6 +510,30 @@ func TestDumpRunnerRunWithSecrets(t *testing.T) { []string{"vars.key"}, true, `value +`, + }, + { + store{ + steps: []map[string]any{}, + stepMap: map[string]map[string]any{ + "stepkey": {"key": "value"}, + }, + vars: map[string]any{}, + useMap: true, + stepMapKeys: []string{"stepkey", "stepnext"}, + }, + "steps", + []*step{ + {key: "stepkey"}, + {key: "stepnext"}, + }, + []string{"current.key"}, + false, + `{ + "stepkey": { + "key": "*****" + } +} `, }, } diff --git a/option.go b/option.go index 598fcd18..b6df3f16 100644 --- a/option.go +++ b/option.go @@ -706,9 +706,6 @@ func Secret(secrets ...string) Option { return ErrNilBook } for _, secret := range secrets { - if strings.HasPrefix(secret, storeRootKeyCurrent+".") { - return fmt.Errorf("secrets: does not support 'current.': %s", secret) - } if strings.HasPrefix(secret, storeRootKeyPrevious+".") { return fmt.Errorf("secrets: does not support 'previous.': %s", secret) } diff --git a/store.go b/store.go index abf53fc0..d7a21146 100644 --- a/store.go +++ b/store.go @@ -356,6 +356,11 @@ func (s *store) toMapForDbg() map[string]any { } func (s *store) setMaskKeywords(store map[string]any) { + store[storeRootKeyCurrent] = s.latest() + defer func() { + delete(store, storeRootKeyCurrent) + }() + for _, key := range s.secrets { v, err := Eval(key, store) if err != nil { From 3862cadd6046f75688f5f55dec2904aaaa54e66f Mon Sep 17 00:00:00 2001 From: k1LoW Date: Sun, 17 Nov 2024 21:35:00 +0900 Subject: [PATCH 6/6] Update README --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 66398a11..f164dbec 100644 --- a/README.md +++ b/README.md @@ -446,6 +446,17 @@ vars: In the example, each variable can be used in `{{ vars.username }}` or `{{ vars.token }}` in `steps:`. +### `secrets:` + +List of secret var names to be masked. + +``` yaml +secrets: + - vars.secret_token + - binded_password + - current.res.message.token +``` + ### `debug:` Enable debug output for runn. @@ -1673,7 +1684,8 @@ or dump: expr: steps[4].rows out: path/to/dump.out - disableTrailingNewline: true + disableTrailingNewline: true # disable trailing newline. default is false + disableMaskingSecrets: true # disable masking secrets. default is false ``` The `dump` runner can run in the same steps as the other runners.