From 21cbab249d2fadef30964df8f6181391d5cc24ac Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 22 Aug 2015 16:24:16 -0700 Subject: [PATCH 1/4] Allow resources to set an initial state from config. Some resources don't really have a true "create" step: either they are just reading a resource or they are executing an "upsert"-like operation on a particular unique name. This new mechanism allows such resources to participate in the "refresh" step without first going through an "apply", thus allowing them to see if a resource of the same name already exists and produce a diff in terms of that existing resource. This mechanism is only appropriate for situations where the Read function is able to completely update the state based on an existing resource. If a resource has write-only attributes that only get set during Create then this mechanism won't work well for them. --- command/refresh.go | 28 ++++++---------- helper/schema/provider.go | 12 +++++++ helper/schema/resource.go | 56 ++++++++++++++++++++++++++++++++ helper/schema/schema.go | 9 ++++++ rpc/resource_provider.go | 41 ++++++++++++++++++++++++ terraform/eval_state.go | 57 +++++++++++++++++++++++++++++++++ terraform/resource_provider.go | 5 +++ terraform/transform_resource.go | 16 +++++++++ 8 files changed, 206 insertions(+), 18 deletions(-) diff --git a/command/refresh.go b/command/refresh.go index ee3cd7007102..e292f7517e3f 100644 --- a/command/refresh.go +++ b/command/refresh.go @@ -48,31 +48,23 @@ func (c *RefreshCommand) Run(args []string) int { return 1 } - // Verify that the state path exists. The "ContextArg" function below + // Verify that the state path can be read. The "ContextArg" function below // will actually do this, but we want to provide a richer error message // if possible. + // It is okay to refresh without a state since the configuration may + // contain logical resources or resources that do not explicitly need to + // be created first. if !state.State().IsRemote() { if _, err := os.Stat(c.Meta.statePath); err != nil { - if os.IsNotExist(err) { + if !os.IsNotExist(err) { c.Ui.Error(fmt.Sprintf( - "The Terraform state file for your infrastructure does not\n"+ - "exist. The 'refresh' command only works and only makes sense\n"+ - "when there is existing state that Terraform is managing. Please\n"+ - "double-check the value given below and try again. If you\n"+ - "haven't created infrastructure with Terraform yet, use the\n"+ - "'terraform apply' command.\n\n"+ - "Path: %s", - c.Meta.statePath)) - return 1 - } - - c.Ui.Error(fmt.Sprintf( - "There was an error reading the Terraform state that is needed\n"+ + "There was an error reading the Terraform state that is needed\n"+ "for refreshing. The path and error are shown below.\n\n"+ "Path: %s\n\nError: %s", - c.Meta.statePath, - err)) - return 1 + c.Meta.statePath, + err)) + return 1 + } } } diff --git a/helper/schema/provider.go b/helper/schema/provider.go index aa35d32d09fe..5b67860930d2 100644 --- a/helper/schema/provider.go +++ b/helper/schema/provider.go @@ -175,6 +175,18 @@ func (p *Provider) Diff( return r.Diff(s, c) } +// InitialState implementation of terraform.ResourceProvider interface. +func (p *Provider) InitialState( + info *terraform.InstanceInfo, + c *terraform.ResourceConfig) (*terraform.InstanceState, error) { + r, ok := p.ResourcesMap[info.Type] + if !ok { + return nil, fmt.Errorf("unknown resource type: %s", info.Type) + } + + return r.InitialState(c, p.meta) +} + // Refresh implementation of terraform.ResourceProvider interface. func (p *Provider) Refresh( info *terraform.InstanceInfo, diff --git a/helper/schema/resource.go b/helper/schema/resource.go index 571fe18a6038..12664b94c8e7 100644 --- a/helper/schema/resource.go +++ b/helper/schema/resource.go @@ -50,6 +50,30 @@ type Resource struct { // needs to make any remote API calls. MigrateState StateMigrateFunc + // SetInitialState is an optional function to pre-set the state for a + // new resource instance. It is provided a ResourceData for the + // configuration and it may mutate that ResourceData; if an Id is set + // when this function returns, the resource data will be written into + // the state so that it can be considered when refreshing and planning. + // + // The function must not make any network requests. It may only + // read the configuration and set any initial state that is needed + // to retrieve the *true* remote state once "Read" is called. + // + // Use this, for example, if the resource configuration includes a + // unique id that might already be in use in the remote system. By + // pre-setting the Id, Terraform can check if there is an existing + // resource to update, and in that case use the "Update" function + // rather than "Create" when "terraform apply" is run. + // + // You don't need to use this if the id for your resource is a computed + // value issued by the target system when the resource is first created. + // + // The "Read" function may subsequently set the Id back to "" to indicate + // that the resource doesn't exist after all, in which case + // "terraform apply" will try to create it as normal. + SetInitialState InitialStateFunc + // The functions below are the CRUD operations for this resource. // // The only optional operation is Update. If Update is not implemented, @@ -95,6 +119,9 @@ type DeleteFunc func(*ResourceData, interface{}) error // See Resource documentation. type ExistsFunc func(*ResourceData, interface{}) (bool, error) +// See Resource documentation. +type InitialStateFunc func(*ResourceData, interface{}) error + // See Resource documentation. type StateMigrateFunc func( int, *terraform.InstanceState, interface{}) (*terraform.InstanceState, error) @@ -167,6 +194,35 @@ func (r *Resource) Validate(c *terraform.ResourceConfig) ([]string, []error) { return schemaMap(r.Schema).Validate(c) } +// InitialState uses the configuration to potentially determine an initial +// state for a new instance. +func (r *Resource) InitialState( + c *terraform.ResourceConfig, + meta interface{}) (*terraform.InstanceState, error) { + + if r.SetInitialState == nil { + return nil, nil + } + + data, err := schemaMap(r.Schema).ConfigData(c) + if err != nil { + return nil, err + } + + err = r.SetInitialState(data, meta) + if err != nil { + return nil, err + } + + // Resource implementation will set the id in order to signal that it wants + // to prime the initial state with whatever's in the resource data. + if data.Id() != "" { + return data.State(), nil + } else { + return nil, nil + } +} + // Refresh refreshes the state of the resource. func (r *Resource) Refresh( s *terraform.InstanceState, diff --git a/helper/schema/schema.go b/helper/schema/schema.go index ec0c8dd51f27..63dfed2cff6d 100644 --- a/helper/schema/schema.go +++ b/helper/schema/schema.go @@ -254,6 +254,15 @@ func (m schemaMap) Data( }, nil } +// ConfigData returns a ResourceData for the given config. +func (m schemaMap) ConfigData( + c *terraform.ResourceConfig) (*ResourceData, error) { + return &ResourceData{ + schema: m, + config: c, + }, nil +} + // Diff returns the diff for a resource given the schema map, // state, and configuration. func (m schemaMap) Diff( diff --git a/rpc/resource_provider.go b/rpc/resource_provider.go index 3fe6927de8b2..55b35116ef48 100644 --- a/rpc/resource_provider.go +++ b/rpc/resource_provider.go @@ -142,6 +142,26 @@ func (p *ResourceProvider) Diff( return resp.Diff, err } +func (p *ResourceProvider) InitialState( + info *terraform.InstanceInfo, + c *terraform.ResourceConfig) (*terraform.InstanceState, error) { + + var resp ResourceProviderInitialStateResponse + args := &ResourceProviderInitialStateArgs{ + Info: info, + Config: c, + } + err := p.Client.Call(p.Name+".InitialState", args, &resp) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + return resp.State, nil +} + func (p *ResourceProvider) Refresh( info *terraform.InstanceInfo, s *terraform.InstanceState) (*terraform.InstanceState, error) { @@ -221,6 +241,16 @@ type ResourceProviderDiffResponse struct { Error *BasicError } +type ResourceProviderInitialStateArgs struct { + Info *terraform.InstanceInfo + Config *terraform.ResourceConfig +} + +type ResourceProviderInitialStateResponse struct { + State *terraform.InstanceState + Error *BasicError +} + type ResourceProviderRefreshArgs struct { Info *terraform.InstanceInfo State *terraform.InstanceState @@ -339,6 +369,17 @@ func (s *ResourceProviderServer) Diff( return nil } +func (s *ResourceProviderServer) InitialState( + args *ResourceProviderInitialStateArgs, + result *ResourceProviderInitialStateResponse) error { + state, err := s.Provider.InitialState(args.Info, args.Config) + *result = ResourceProviderInitialStateResponse{ + State: state, + Error: NewBasicError(err), + } + return nil +} + func (s *ResourceProviderServer) Refresh( args *ResourceProviderRefreshArgs, result *ResourceProviderRefreshResponse) error { diff --git a/terraform/eval_state.go b/terraform/eval_state.go index 3a3e9efd6e76..5191d7b3c5d1 100644 --- a/terraform/eval_state.go +++ b/terraform/eval_state.go @@ -374,3 +374,60 @@ func (n *EvalUndeposeState) Eval(ctx EvalContext) (interface{}, error) { return nil, nil } + +// EvalSetInitialId is an EvalNode implementation that will try to prime a +// resource's state with an id from the configuration if it doesn't already +// have an id set. +// +// If executed before EvalRefresh, this allows a resource to be considered as +// already-existing and skip the "create" step if a resource with the +// configured id already exists, or if the resource is a read-only +// data-collection resource. +type EvalSetInitialState struct { + Name string + ResourceType string + Info *InstanceInfo + Provider *ResourceProvider + ProviderName string + Config **ResourceConfig + State **InstanceState + Dependencies []string + Output **InstanceState +} + +// TODO: test +func (n *EvalSetInitialState) Eval(ctx EvalContext) (interface{}, error) { + provider := *n.Provider + state := *n.State + config := *n.Config + + // If we already have a state then we have nothing to do; this applies + // only to new resources that haven't yet been initialized. + if state != nil { + return nil, nil + } + + // Ask the provider for a configured state. + newState, err := provider.InitialState(n.Info, config) + if err != nil { + return nil, fmt.Errorf("%s: %s", n.Info.Id, err.Error()) + } + + if newState != nil { + newState, err = writeInstanceToState(ctx, n.Name, n.ResourceType, n.ProviderName, n.Dependencies, + func(rs *ResourceState) error { + rs.Primary = newState + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("%s: %s", n.Info.Id, err.Error()) + } + } + + if n.Output != nil { + *n.Output = newState + } + + return nil, nil +} diff --git a/terraform/resource_provider.go b/terraform/resource_provider.go index ea23b031d3a4..f870a523a5e8 100644 --- a/terraform/resource_provider.go +++ b/terraform/resource_provider.go @@ -66,6 +66,11 @@ type ResourceProvider interface { *InstanceState, *ResourceConfig) (*InstanceDiff, error) + // InitialState produces an initial state for a new resource instance + // using its configuration alone. It may return a nil *InstanceState + // if the state cannot be determined until its first Apply. + InitialState(*InstanceInfo, *ResourceConfig) (*InstanceState, error) + // Refresh refreshes a resource and updates all of its attributes // with the latest information. Refresh(*InstanceInfo, *InstanceState) (*InstanceState, error) diff --git a/terraform/transform_resource.go b/terraform/transform_resource.go index 0b56721b07a7..86c6612b716f 100644 --- a/terraform/transform_resource.go +++ b/terraform/transform_resource.go @@ -263,10 +263,26 @@ func (n *graphNodeExpandedResource) EvalTree() EvalNode { Ops: []walkOperation{walkRefresh}, Node: &EvalSequence{ Nodes: []EvalNode{ + &EvalInterpolate{ + Config: n.Resource.RawConfig.Copy(), + Resource: resource, + Output: &resourceConfig, + }, &EvalGetProvider{ Name: n.ProvidedBy()[0], Output: &provider, }, + &EvalSetInitialState{ + Name: n.stateId(), + Info: info, + ResourceType: n.Resource.Type, + Config: &resourceConfig, + Provider: &provider, + ProviderName: n.Resource.Provider, + Dependencies: n.StateDependencies(), + State: &state, + Output: &state, + }, &EvalReadState{ Name: n.stateId(), Output: &state, From 84c9a93d5aa1d38b3c4de2f3ec906031b559e7cf Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 22 Aug 2015 16:28:04 -0700 Subject: [PATCH 2/4] Make terraform_remote_state resources available on initial plan. This resource exists only to read data, so there's no reason not to treat it as existing immediately, so that its data can be used in provider configurations. --- builtin/providers/terraform/resource_state.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/builtin/providers/terraform/resource_state.go b/builtin/providers/terraform/resource_state.go index fb0e85ee2c7e..0b5d027613bf 100644 --- a/builtin/providers/terraform/resource_state.go +++ b/builtin/providers/terraform/resource_state.go @@ -10,10 +10,18 @@ import ( func resourceRemoteState() *schema.Resource { return &schema.Resource{ - Create: resourceRemoteStateCreate, + Create: resourceRemoteStateRead, Read: resourceRemoteStateRead, Delete: resourceRemoteStateDelete, + SetInitialState: func(d *schema.ResourceData, meta interface {}) error { + // Just need to set the id to *something* non-empty, and then + // we'll get an opportunity to fill the initial state for real + // when the "Read" function is called. + d.SetId(time.Now().UTC().String()) + return nil + }, + Schema: map[string]*schema.Schema{ "backend": &schema.Schema{ Type: schema.TypeString, @@ -35,10 +43,6 @@ func resourceRemoteState() *schema.Resource { } } -func resourceRemoteStateCreate(d *schema.ResourceData, meta interface{}) error { - return resourceRemoteStateRead(d, meta) -} - func resourceRemoteStateRead(d *schema.ResourceData, meta interface{}) error { backend := d.Get("backend").(string) config := make(map[string]string) From b7c4328de83e48c14a3fe42a9ed96ec91f1decf8 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 22 Aug 2015 16:31:39 -0700 Subject: [PATCH 3/4] aws_s3_bucket to recognize when a bucket already exists. S3 buckets have globally-unique names, so an existing bucket may already exist with the same name. If it's owned by the same account then we can treat it as an update of that existing resource in diffs, giving a more honest account of what we're going to do when we apply. If it's owned by a different account then we'll still fail, but it's still better because we'll fail during the planning stage rather than the apply stage, and thus we won't blow up leaving the state half-applied. --- builtin/providers/aws/resource_aws_s3_bucket.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/builtin/providers/aws/resource_aws_s3_bucket.go b/builtin/providers/aws/resource_aws_s3_bucket.go index 0e07ee733308..ccce0511d7e7 100644 --- a/builtin/providers/aws/resource_aws_s3_bucket.go +++ b/builtin/providers/aws/resource_aws_s3_bucket.go @@ -19,6 +19,11 @@ func resourceAwsS3Bucket() *schema.Resource { Update: resourceAwsS3BucketUpdate, Delete: resourceAwsS3BucketDelete, + SetInitialState: func(d *schema.ResourceData, meta interface {}) error { + d.SetId(d.Get("bucket").(string)) + return nil + }, + Schema: map[string]*schema.Schema{ "bucket": &schema.Schema{ Type: schema.TypeString, From 244940d5fdf71440c63dc104f0382898c9c42476 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 22 Aug 2015 17:35:05 -0700 Subject: [PATCH 4/4] Detect same-named resources when instantiating IAM objects. IAM objects must have unique names. Now if a user declares an object with a name that is already in use, Terraform will notice this and diff the configuration against the existing object. --- builtin/providers/aws/resource_aws_iam_group.go | 5 +++++ builtin/providers/aws/resource_aws_iam_role.go | 8 ++++++++ builtin/providers/aws/resource_aws_iam_user.go | 5 +++++ builtin/providers/aws/resource_aws_key_pair.go | 16 ++++++++++++++++ 4 files changed, 34 insertions(+) diff --git a/builtin/providers/aws/resource_aws_iam_group.go b/builtin/providers/aws/resource_aws_iam_group.go index 45defaaaf9cb..d2a16ae0979a 100644 --- a/builtin/providers/aws/resource_aws_iam_group.go +++ b/builtin/providers/aws/resource_aws_iam_group.go @@ -18,6 +18,11 @@ func resourceAwsIamGroup() *schema.Resource { //Update: resourceAwsIamGroupUpdate, Delete: resourceAwsIamGroupDelete, + SetInitialState: func(d *schema.ResourceData, meta interface {}) error { + d.SetId(d.Get("name").(string)) + return nil + }, + Schema: map[string]*schema.Schema{ "arn": &schema.Schema{ Type: schema.TypeString, diff --git a/builtin/providers/aws/resource_aws_iam_role.go b/builtin/providers/aws/resource_aws_iam_role.go index 833dc3626b17..9bffb3c39610 100644 --- a/builtin/providers/aws/resource_aws_iam_role.go +++ b/builtin/providers/aws/resource_aws_iam_role.go @@ -19,6 +19,11 @@ func resourceAwsIamRole() *schema.Resource { //Update: resourceAwsIamRoleUpdate, Delete: resourceAwsIamRoleDelete, + SetInitialState: func(d *schema.ResourceData, meta interface {}) error { + d.SetId(d.Get("name").(string)) + return nil + }, + Schema: map[string]*schema.Schema{ "arn": &schema.Schema{ Type: schema.TypeString, @@ -110,6 +115,9 @@ func resourceAwsIamRoleReadResult(d *schema.ResourceData, role *iam.Role) error if err := d.Set("unique_id", role.RoleId); err != nil { return err } + if err := d.Set("assume_role_policy", role.AssumeRolePolicyDocument); err != nil { + return err + } return nil } diff --git a/builtin/providers/aws/resource_aws_iam_user.go b/builtin/providers/aws/resource_aws_iam_user.go index c0ef4b8a462d..76f3459369a6 100644 --- a/builtin/providers/aws/resource_aws_iam_user.go +++ b/builtin/providers/aws/resource_aws_iam_user.go @@ -19,6 +19,11 @@ func resourceAwsIamUser() *schema.Resource { //Update: resourceAwsIamUserUpdate, Delete: resourceAwsIamUserDelete, + SetInitialState: func(d *schema.ResourceData, meta interface {}) error { + d.SetId(d.Get("name").(string)) + return nil + }, + Schema: map[string]*schema.Schema{ "arn": &schema.Schema{ Type: schema.TypeString, diff --git a/builtin/providers/aws/resource_aws_key_pair.go b/builtin/providers/aws/resource_aws_key_pair.go index e747fbfc5055..e4b90a550dff 100644 --- a/builtin/providers/aws/resource_aws_key_pair.go +++ b/builtin/providers/aws/resource_aws_key_pair.go @@ -18,6 +18,22 @@ func resourceAwsKeyPair() *schema.Resource { Update: nil, Delete: resourceAwsKeyPairDelete, + SetInitialState: func(d *schema.ResourceData, meta interface {}) error { + // If the configuration is forcing a particular key name + // then we'll prime our initial state with it so that + // we can determine if we're going to collide with an + // existing key. + keyName := d.Get("key_name").(string) + if keyName != "" { + d.SetId(keyName) + + // Don't set the public_key since we don't actually know + // what the remote state says, and Read can't fetch it. + d.Set("public_key", "") + } + return nil + }, + Schema: map[string]*schema.Schema{ "key_name": &schema.Schema{ Type: schema.TypeString,