Merge pull request #1076 from k1LoW/secrets

Implement `secrets:` section

README.md

@@ -446,6 +446,17 @@ vars:
 
 In the example, each variable can be used in `{{ vars.username }}` or `{{ vars.token }}` in `steps:`.
 
+### `secrets:`
+
+List of secret var names to be masked.
+
+``` yaml
+secrets:
+  - vars.secret_token
+  - binded_password
+  - current.res.message.token
+```
+
 ### `debug:`
 
 Enable debug output for runn.
@@ -1673,7 +1684,8 @@ or
   dump:
     expr: steps[4].rows
     out: path/to/dump.out
-    disableTrailingNewline: true
+    disableTrailingNewline: true # disable trailing newline. default is false
+    disableMaskingSecrets: true  # disable masking secrets. default is false
 ```
 
 The `dump` runner can run in the same steps as the other runners.

book.go

@@ -591,6 +591,7 @@ func (bk *book) merge(loaded *book) error {
 	for k, v := range loaded.vars {
 		bk.vars[k] = v
 	}
+	bk.secrets = append(bk.secrets, loaded.secrets...)
 	bk.runnerErrs = loaded.runnerErrs
 	bk.rawSteps = loaded.rawSteps
 	bk.hostRules = loaded.hostRules

debugger_test.go

@@ -114,3 +114,44 @@ func TestDebuggerWithStderr(t *testing.T) {
 		})
 	}
 }
+
+func TestDebuggerWithSecrets(t *testing.T) {
+	noColor(t)
+	tests := []struct {
+		book string
+	}{
+		{"testdata/book/with_secrets.yml"},
+	}
+	ctx := context.Background()
+	for _, tt := range tests {
+		t.Run(tt.book, func(t *testing.T) {
+			out := new(bytes.Buffer)
+			hs := testutil.HTTPServer(t)
+			opts := []Option{
+				Book(tt.book),
+				HTTPRunner("req", hs.URL, hs.Client()),
+				Stderr(out),
+				Scopes(ScopeAllowRunExec),
+			}
+			o, err := New(opts...)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := o.Run(ctx); err != nil {
+				t.Error(err)
+			}
+
+			got := out.String()
+
+			f := fmt.Sprintf("%s.debugger", filepath.Base(tt.book))
+			if os.Getenv("UPDATE_GOLDEN") != "" {
+				golden.Update(t, "testdata", f, got)
+				return
+			}
+
+			if diff := golden.Diff(t, "testdata", f, got); diff != "" {
+				t.Error(diff)
+			}
+		})
+	}
+}

dump.go

@@ -9,6 +9,7 @@ import (
 	"reflect"
 
 	"github.com/goccy/go-json"
+	"github.com/k1LoW/donegroup"
 )
 
 const dumpRunnerKey = "dump"
@@ -19,6 +20,7 @@ type dumpRequest struct {
 	expr                   string
 	out                    string
 	disableTrailingNewline bool
+	disableMaskingSecrets  bool
 }
 
 func newDumpRunner() *dumpRunner {
@@ -38,7 +40,11 @@ func (rnr *dumpRunner) Run(ctx context.Context, s *step, first bool) error {
 		store[storeRootKeyCurrent] = o.store.latest()
 	}
 	if r.out == "" {
-		out = o.stdout
+		if r.disableMaskingSecrets {
+			out = o.stdout.Unwrap()
+		} else {
+			out = o.stdout
+		}
 	} else {
 		p, err := EvalExpand(r.out, store)
 		if err != nil {
@@ -53,7 +59,16 @@ func (rnr *dumpRunner) Run(ctx context.Context, s *step, first bool) error {
 			if err != nil {
 				return err
 			}
-			out = f
+			if err := donegroup.Cleanup(ctx, func() error {
+				return f.Close()
+			}); err != nil {
+				return err
+			}
+			if r.disableMaskingSecrets {
+				out = f
+			} else {
+				out = o.maskRule.NewWriter(f)
+			}
 		default:
 			return fmt.Errorf("invalid dump out: %v", pp)
 		}

dump_test.go

@@ -7,6 +7,9 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/k1LoW/donegroup"
+	"github.com/k1LoW/maskedio"
 )
 
 func TestDumpRunnerRun(t *testing.T) {
@@ -138,16 +141,17 @@ func TestDumpRunnerRun(t *testing.T) {
 			`hello`,
 		},
 	}
-	ctx := context.Background()
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d.%s", i, tt.expr), func(t *testing.T) {
+			ctx, cancel := donegroup.WithCancel(context.Background())
+			t.Cleanup(cancel)
 			o, err := New()
 			if err != nil {
 				t.Fatal(err)
 			}
 			buf := new(bytes.Buffer)
 			o.store = &tt.store
-			o.stdout = buf
+			o.stdout = maskedio.NewWriter(buf)
 			o.useMap = tt.store.useMap
 			o.steps = tt.steps
 			d := newDumpRunner()
@@ -296,9 +300,11 @@ func TestDumpRunnerRunWithOut(t *testing.T) {
 			`hello`,
 		},
 	}
-	ctx := context.Background()
+
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d.%s with out", i, tt.expr), func(t *testing.T) {
+			ctx, cancel := donegroup.WithCancel(context.Background())
+			t.Cleanup(cancel)
 			p := filepath.Join(t.TempDir(), "tmp")
 			o, err := New()
 			if err != nil {
@@ -383,9 +389,11 @@ func TestDumpRunnerRunWithExpandOut(t *testing.T) {
 			filepath.Join(tmp, "value3.ext"),
 		},
 	}
-	ctx := context.Background()
+
 	for _, tt := range tests {
 		t.Run(tt.out, func(t *testing.T) {
+			ctx, cancel := donegroup.WithCancel(context.Background())
+			t.Cleanup(cancel)
 			o, err := New()
 			if err != nil {
 				t.Fatal(err)
@@ -406,3 +414,158 @@ func TestDumpRunnerRunWithExpandOut(t *testing.T) {
 		})
 	}
 }
+
+func TestDumpRunnerRunWithSecrets(t *testing.T) {
+	tests := []struct {
+		store                 store
+		expr                  string
+		steps                 []*step
+		secrets               []string
+		disableMaskingSecrets bool
+		want                  string
+	}{
+		{
+			store{
+				steps: []map[string]any{},
+				vars: map[string]any{
+					"key": "value",
+				},
+			},
+			"vars.key",
+			nil,
+			[]string{"vars.key"},
+			false,
+			`*****
+`,
+		},
+		{
+			store{
+				steps: []map[string]any{},
+				vars: map[string]any{
+					"key": "value",
+				},
+			},
+			"vars",
+			nil,
+			[]string{"vars.key"},
+			false,
+			`{
+  "key": "*****"
+}
+`,
+		},
+		{
+			store{
+				steps: []map[string]any{
+					{
+						"key": "value",
+					},
+				},
+				vars: map[string]any{},
+			},
+			"steps",
+			nil,
+			[]string{"steps[0].key"},
+			false,
+			`[
+  {
+    "key": "*****"
+  }
+]
+`,
+		},
+		{
+			store{
+				steps: []map[string]any{},
+				stepMap: map[string]map[string]any{
+					"stepkey": {"key": "value"},
+				},
+				vars:        map[string]any{},
+				useMap:      true,
+				stepMapKeys: []string{"stepkey", "stepnext"},
+			},
+			"steps",
+			[]*step{
+				{key: "stepkey"},
+				{key: "stepnext"},
+			},
+			[]string{"steps.stepkey.key"},
+			false,
+			`{
+  "stepkey": {
+    "key": "*****"
+  }
+}
+`,
+		},
+		{
+			store{
+				steps: []map[string]any{},
+				vars: map[string]any{
+					"key": "value",
+				},
+			},
+			"vars.key",
+			nil,
+			[]string{"vars.key"},
+			true,
+			`value
+`,
+		},
+		{
+			store{
+				steps: []map[string]any{},
+				stepMap: map[string]map[string]any{
+					"stepkey": {"key": "value"},
+				},
+				vars:        map[string]any{},
+				useMap:      true,
+				stepMapKeys: []string{"stepkey", "stepnext"},
+			},
+			"steps",
+			[]*step{
+				{key: "stepkey"},
+				{key: "stepnext"},
+			},
+			[]string{"current.key"},
+			false,
+			`{
+  "stepkey": {
+    "key": "*****"
+  }
+}
+`,
+		},
+	}
+
+	for i, tt := range tests {
+		t.Run(fmt.Sprintf("%d.%s", i, tt.expr), func(t *testing.T) {
+			ctx, cancel := donegroup.WithCancel(context.Background())
+			t.Cleanup(cancel)
+			buf := new(bytes.Buffer)
+			o, err := New(Stdout(buf))
+			if err != nil {
+				t.Fatal(err)
+			}
+			tt.store.secrets = tt.secrets
+			tt.store.mr = o.store.mr
+			tt.store.setMaskKeywords(tt.store.toMap())
+			o.store = &tt.store
+			o.useMap = tt.store.useMap
+			o.steps = tt.steps
+			d := newDumpRunner()
+			s := newStep(0, "stepKey", o, nil)
+			s.dumpRequest = &dumpRequest{
+				expr:                  tt.expr,
+				disableMaskingSecrets: tt.disableMaskingSecrets,
+			}
+			if err := d.Run(ctx, s, true); err != nil {
+				t.Fatal(err)
+			}
+			got := buf.String()
+			if got != tt.want {
+				t.Errorf("got\n%#v\nwant\n%#v", got, tt.want)
+			}
+		})
+	}
+}

exec.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"strings"
 
 	"github.com/cli/safeexec"
@@ -84,8 +83,8 @@ func (rnr *execRunner) run(ctx context.Context, c *execCommand, s *step) error {
 		o.capturers.captureExecStdin(c.stdin)
 	}
 	if c.liveOutput {
-		cmd.Stdout = io.MultiWriter(stdout, os.Stdout)
-		cmd.Stderr = io.MultiWriter(stderr, os.Stderr)
+		cmd.Stdout = io.MultiWriter(stdout, o.maskRule.NewWriter(o.stdout))
+		cmd.Stderr = io.MultiWriter(stderr, o.maskRule.NewWriter(o.stderr))
 	} else {
 		cmd.Stdout = stdout
 		cmd.Stderr = stderr