states: store ephemeral output values in memory (#35676)

Ephemeral root output values must be kept in the in-memory state representation, but not written to the state file. To achieve this, we store ephemeral root outputs separately from non-ephemeral root outputs, so Terraform can access them during a single plan or apply phase.

Ephemeral root outputs always have a value of null in the state file. This means that the "terraform output" command, that reads the state file, reports null values for these outputs. Consumers of 'terraform output -json' should use the presence of '"ephemeral": true' in such output to interpret the value correctly.

internal/backend/backendrun/operation.go

@@ -237,6 +237,12 @@ type RunningOperation struct {
 	// this state is managed by the backend. This should only be read
 	// after the operation completes to avoid read/write races.
 	State *states.State
+
+	// EphemeralOutputValues is populated only after an Apply operation
+	// completes, and contains the value for each ephemeral output in the root
+	// module.
+	// Ephemeral output values are not stored in the state file.
+	EphemeralOutputValues map[string]*states.OutputValue
 }
 
 // OperationResult describes the result status of an operation.

internal/backend/local/backend_apply.go

@@ -376,6 +376,8 @@ func (b *Local) opApply(
 		return
 	}
 
+	runningOp.EphemeralOutputValues = applyState.EphemeralRootOutputValues
+
 	// Store the final state
 	runningOp.State = applyState
 	err := statemgr.WriteAndPersist(opState, applyState, schemas)

internal/backend/local/backend_local_test.go

@@ -266,6 +266,10 @@ func (s *stateStorageThatFailsRefresh) GetRootOutputValues(ctx context.Context)
 	return nil, fmt.Errorf("unimplemented")
 }
 
+func (s *stateStorageThatFailsRefresh) GetEphemeralRootOutputValues(ctx context.Context) (map[string]*states.OutputValue, error) {
+	return nil, fmt.Errorf("unimplemented")
+}
+
 func (s *stateStorageThatFailsRefresh) WriteState(*states.State) error {
 	return fmt.Errorf("unimplemented")
 }

internal/cloud/state.go

@@ -598,6 +598,11 @@ func (s *State) GetRootOutputValues(ctx context.Context) (map[string]*states.Out
 	return result, nil
 }
 
+func (s *State) GetEphemeralRootOutputValues(ctx context.Context) (map[string]*states.OutputValue, error) {
+	// NOTE Ephemeral output values are not yet supported by the cloud backend.
+	return nil, nil
+}
+
 func clamp(val, min, max int64) int64 {
 	if val < min {
 		return min

internal/command/jsonformat/state.go

@@ -85,7 +85,6 @@ func (state State) renderHumanStateModule(renderer Renderer, module jsonstate.Mo
 }
 
 func (state State) renderHumanStateOutputs(renderer Renderer, opts computed.RenderHumanOpts) {
-
 	if len(state.RootModuleOutputs) > 0 {
 		renderer.Streams.Printf("\n\nOutputs:\n\n")
 

internal/command/jsonstate/state.go

@@ -221,6 +221,12 @@ func MarshalOutputs(outputs map[string]*states.OutputValue) (map[string]Output,
 
 	ret := make(map[string]Output)
 	for k, v := range outputs {
+
+		if v.Ephemeral {
+			// should never happen
+			panic(fmt.Sprintf("Ephemeral output value %s passed to state.MarshalOutputs. This is a bug in Terraform - please report it.", k))
+		}
+
 		ty := v.Value.Type()
 		ov, err := ctyjson.Marshal(v.Value, ty)
 		if err != nil {

internal/command/output.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"strings"
 
+	"maps"
+
 	"github.com/hashicorp/terraform/internal/command/arguments"
 	"github.com/hashicorp/terraform/internal/command/views"
 	"github.com/hashicorp/terraform/internal/states"
@@ -89,12 +91,17 @@ func (c *OutputCommand) Outputs(statePath string) (map[string]*states.OutputValu
 		return nil, diags
 	}
 
-	output, err := stateStore.GetRootOutputValues(ctx)
+	outputs, err := stateStore.GetRootOutputValues(ctx)
+	if err != nil {
+		return nil, diags.Append(err)
+	}
+	ephemeralOutputs, err := stateStore.GetEphemeralRootOutputValues(ctx)
 	if err != nil {
 		return nil, diags.Append(err)
 	}
+	maps.Copy(outputs, ephemeralOutputs)
 
-	return output, diags
+	return outputs, diags
 }
 
 func (c *OutputCommand) Help() string {

internal/command/output_test.go

@@ -80,7 +80,43 @@ func TestOutput_json(t *testing.T) {
 	}
 
 	actual := strings.TrimSpace(output.Stdout())
-	expected := "{\n  \"foo\": {\n    \"sensitive\": false,\n    \"type\": \"string\",\n    \"value\": \"bar\"\n  }\n}"
+	expected := "{\n  \"foo\": {\n    \"ephemeral\": false,\n    \"sensitive\": false,\n    \"type\": \"string\",\n    \"value\": \"bar\"\n  }\n}"
+	if actual != expected {
+		t.Fatalf("wrong output\ngot:  %#v\nwant: %#v", actual, expected)
+	}
+}
+
+func TestOutput_jsonEphemeral(t *testing.T) {
+	originalState := states.BuildState(func(s *states.SyncState) {
+		s.SetEphemeralOutputValue(
+			addrs.OutputValue{Name: "foo"}.Absolute(addrs.RootModuleInstance),
+			cty.StringVal("bar"),
+			false,
+		)
+	})
+
+	statePath := testStateFile(t, originalState)
+
+	view, done := testView(t)
+	c := &OutputCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			View:             view,
+		},
+	}
+
+	args := []string{
+		"-state", statePath,
+		"-json",
+	}
+	code := c.Run(args)
+	output := done(t)
+	if code != 0 {
+		t.Fatalf("bad: \n%s", output.Stderr())
+	}
+
+	actual := strings.TrimSpace(output.Stdout())
+	expected := "{\n  \"foo\": {\n    \"ephemeral\": true,\n    \"sensitive\": false,\n    \"type\": \"string\",\n    \"value\": null\n  }\n}"
 	if actual != expected {
 		t.Fatalf("wrong output\ngot:  %#v\nwant: %#v", actual, expected)
 	}

internal/command/refresh_test.go

@@ -559,7 +559,7 @@ func TestRefresh_backup(t *testing.T) {
 	statePath := testStateFile(t, state)
 
 	// Output path
-	outf, err := ioutil.TempFile(td, "tf")
+	outf, err := os.CreateTemp(td, "tf")
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
@@ -574,7 +574,7 @@ func TestRefresh_backup(t *testing.T) {
 	}
 
 	// Backup path
-	backupf, err := ioutil.TempFile(td, "tf")
+	backupf, err := os.CreateTemp(td, "tf")
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}

internal/command/views/output.go

@@ -217,7 +217,11 @@ func (v *OutputJSON) Output(name string, outputs map[string]*states.OutputValue)
 	// show in the single value case. We must now maintain that behavior
 	// for compatibility, so this is an emulation of the JSON
 	// serialization of outputs used in state format version 3.
+	//
+	// Note that when running the output command, the value of an ephemeral
+	// output is always nil and its type is always cty.DynamicPseudoType.
 	type OutputMeta struct {
+		Ephemeral bool            `json:"ephemeral"`
 		Sensitive bool            `json:"sensitive"`
 		Type      json.RawMessage `json:"type"`
 		Value     json.RawMessage `json:"value"`
@@ -236,6 +240,7 @@ func (v *OutputJSON) Output(name string, outputs map[string]*states.OutputValue)
 			return diags
 		}
 		outputMetas[n] = OutputMeta{
+			Ephemeral: os.Ephemeral,
 			Sensitive: os.Sensitive,
 			Type:      json.RawMessage(jsonType),
 			Value:     json.RawMessage(jsonVal),

internal/command/views/output_test.go

@@ -149,6 +149,7 @@ foo = <sensitive>
 			arguments.ViewJSON,
 			`{
   "bar": {
+    "ephemeral": false,
     "sensitive": false,
     "type": [
       "list",
@@ -161,6 +162,7 @@ foo = <sensitive>
     ]
   },
   "baz": {
+    "ephemeral": false,
     "sensitive": false,
     "type": [
       "object",
@@ -175,6 +177,7 @@ foo = <sensitive>
     }
   },
   "foo": {
+    "ephemeral": false,
     "sensitive": true,
     "type": "string",
     "value": "secret"

internal/states/output_value.go

@@ -16,4 +16,5 @@ type OutputValue struct {
 	Addr      addrs.AbsOutputValue
 	Value     cty.Value
 	Sensitive bool
+	Ephemeral bool
 }

internal/states/remote/state.go

@@ -73,6 +73,19 @@ func (s *State) GetRootOutputValues(ctx context.Context) (map[string]*states.Out
 	return state.RootOutputValues, nil
 }
 
+func (s *State) GetEphemeralRootOutputValues(ctx context.Context) (map[string]*states.OutputValue, error) {
+	if err := s.RefreshState(); err != nil {
+		return nil, fmt.Errorf("Failed to load state: %s", err)
+	}
+
+	state := s.State()
+	if state == nil {
+		state = states.NewState()
+	}
+
+	return state.EphemeralRootOutputValues, nil
+}
+
 // StateForMigration is part of our implementation of statemgr.Migrator.
 func (s *State) StateForMigration() *statefile.File {
 	s.mu.Lock()

internal/states/state.go

@@ -29,14 +29,21 @@ type State struct {
 	// an implementation detail and must not be used by outside callers.
 	Modules map[string]*Module
 
-	// OutputValues contains the state for each output value defined in the
-	// root module.
+	// RootOutputValues contains the state for each non-ephemeral output value
+	// defined in the root module.
 	//
 	// Output values in other modules don't persist anywhere between runs,
 	// so Terraform Core tracks those only internally and does not expose
 	// them in any artifacts that survive between runs.
 	RootOutputValues map[string]*OutputValue
 
+	// EphemeralRootOutputValues contains the state for each ephemeral output
+	// value defined in the root module.
+	//
+	// Ephemeral outputs are treated separately from non-ephemeral outputs, to
+	// ensure that their values are never written to the state file.
+	EphemeralRootOutputValues map[string]*OutputValue
+
 	// CheckResults contains a snapshot of the statuses of checks at the
 	// end of the most recent update to the state. Callers might compare
 	// checks between runs to see if e.g. a previously-failing check has
@@ -56,8 +63,9 @@ func NewState() *State {
 	modules := map[string]*Module{}
 	modules[addrs.RootModuleInstance.String()] = NewModule(addrs.RootModuleInstance)
 	return &State{
-		Modules:          modules,
-		RootOutputValues: make(map[string]*OutputValue),
+		Modules:                   modules,
+		RootOutputValues:          make(map[string]*OutputValue),
+		EphemeralRootOutputValues: make(map[string]*OutputValue),
 	}
 }
 
@@ -77,7 +85,7 @@ func (s *State) Empty() bool {
 	if s == nil {
 		return true
 	}
-	if len(s.RootOutputValues) != 0 {
+	if len(s.RootOutputValues) != 0 || len(s.EphemeralRootOutputValues) != 0 {
 		return false
 	}
 	for _, ms := range s.Modules {
@@ -301,9 +309,9 @@ func (s *State) OutputValue(addr addrs.AbsOutputValue) *OutputValue {
 // SetOutputValue updates the value stored for the given output value if and
 // only if it's a root module output value.
 //
-// All other output values will just be silently ignored, because we don't
-// store those here anymore. (They live in a namedvals.State object hidden
-// in the internals of Terraform Core.)
+// All child module output values will just be silently ignored, because we
+// don't store those here any more. (They live in a namedvals.State object
+// hidden in the internals of Terraform Core.)
 func (s *State) SetOutputValue(addr addrs.AbsOutputValue, value cty.Value, sensitive bool) {
 	if !addr.Module.IsRoot() {
 		return
@@ -323,6 +331,41 @@ func (s *State) RemoveOutputValue(addr addrs.AbsOutputValue) {
 	delete(s.RootOutputValues, addr.OutputValue.Name)
 }
 
+// EphemeralOutputValue returns the state for the output value with the given
+// address, or nil if no such ephemeral output value is tracked in the state.
+//
+// Only root module output values are tracked in the state, so this always
+// returns nil for output values in any other module.
+func (s *State) EphemeralOutputValue(addr addrs.AbsOutputValue) *OutputValue {
+	if !addr.Module.IsRoot() {
+		return nil
+	}
+	return s.EphemeralRootOutputValues[addr.OutputValue.Name]
+}
+
+// SetEphemeralOutputValue updates the value stored for the given ephemeral
+// output value if and only if it's a root module output value.
+func (s *State) SetEphemeralOutputValue(addr addrs.AbsOutputValue, value cty.Value, sensitive bool) {
+	if !addr.Module.IsRoot() {
+		return
+	}
+	s.EphemeralRootOutputValues[addr.OutputValue.Name] = &OutputValue{
+		Addr:      addr,
+		Value:     value,
+		Sensitive: sensitive,
+		Ephemeral: true,
+	}
+}
+
+// RemoveOutputValue removes the record of a previously-stored ephemeral output
+// value.
+func (s *State) RemoveEphemeralOutputValue(addr addrs.AbsOutputValue) {
+	if !addr.Module.IsRoot() {
+		return
+	}
+	delete(s.EphemeralRootOutputValues, addr.OutputValue.Name)
+}
+
 // ProviderAddrs returns a list of all of the provider configuration addresses
 // referenced throughout the receiving state.
 //

internal/states/state_deepcopy.go

@@ -35,10 +35,15 @@ func (s *State) DeepCopy() *State {
 	for k, v := range s.RootOutputValues {
 		outputValues[k] = v.DeepCopy()
 	}
+	ephemeralOutputValues := make(map[string]*OutputValue, len(s.EphemeralRootOutputValues))
+	for k, v := range s.EphemeralRootOutputValues {
+		ephemeralOutputValues[k] = v.DeepCopy()
+	}
 	return &State{
-		Modules:          modules,
-		RootOutputValues: outputValues,
-		CheckResults:     s.CheckResults.DeepCopy(),
+		Modules:                   modules,
+		RootOutputValues:          outputValues,
+		EphemeralRootOutputValues: ephemeralOutputValues,
+		CheckResults:              s.CheckResults.DeepCopy(),
 	}
 }
 
@@ -228,5 +233,6 @@ func (os *OutputValue) DeepCopy() *OutputValue {
 		Addr:      os.Addr,
 		Value:     os.Value,
 		Sensitive: os.Sensitive,
+		Ephemeral: os.Ephemeral,
 	}
 }