From 64ff0e3dadd4b1e654d095518485591a0c0056d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Budai?= Date: Tue, 26 Nov 2024 12:53:52 +0100 Subject: [PATCH] awscloud: add very verbose logging to createFleet creation We still see this error sometimes: Unable to start secure instance: Unable to create fleet: InsufficientInstanceCapacity: There is no Spot capacity available that matches your request This is awkward because the message mentions that there is no spot capacity, even though the current code should retry on InsufficientInstanceCapacity. I also confirmed this by searching for the retries log messages: there are none in the logs. We need a bigger hammer. Let's log everything that happens in the createFleet method in order to have better understanding why the retry logic isn't triggered. We should probably move most of the newly added logs to the debug level, but let's delay that until we have more insight into what's happening. --- internal/cloud/awscloud/secure-instance.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/internal/cloud/awscloud/secure-instance.go b/internal/cloud/awscloud/secure-instance.go index d23d927aad..6866942292 100644 --- a/internal/cloud/awscloud/secure-instance.go +++ b/internal/cloud/awscloud/secure-instance.go @@ -3,6 +3,7 @@ package awscloud import ( "context" "encoding/base64" + "encoding/json" "errors" "fmt" "slices" @@ -592,6 +593,7 @@ func (a *AWS) deleteSGIfExists(si *SecureInstance) error { } func (a *AWS) createFleet(input *ec2.CreateFleetInput) (*ec2.CreateFleetOutput, error) { + logCreateFleetInput(input) createFleetOutput, err := a.ec2.CreateFleet(context.Background(), input) if err != nil { return createFleetOutput, fmt.Errorf("Unable to create spot fleet: %w", err) @@ -602,20 +604,26 @@ func (a *AWS) createFleet(input *ec2.CreateFleetInput) (*ec2.CreateFleetOutput, logrus.Warnf("Received errors (%s) from CreateFleet, retrying CreateFleet with OnDemand instance", strings.Join(fleetErrs, "; ")) input.SpotOptions = nil input.TargetCapacitySpecification.DefaultTargetCapacityType = ec2types.DefaultTargetCapacityTypeOnDemand + logCreateFleetInput(input) createFleetOutput, err = a.ec2.CreateFleet(context.Background(), input) if err != nil { return createFleetOutput, fmt.Errorf("Unable to create on demand fleet: %w", err) } + } else { + logrus.Infof("Won't retry CreateFleet with OnDemand instance, retry: %v, errors: %s", retry, strings.Join(fleetErrs, "; ")) } retry, fleetErrs = doCreateFleetRetry(createFleetOutput) if len(fleetErrs) > 0 && retry { logrus.Warnf("Received errors (%s) from CreateFleet with OnDemand instance option, retrying across availability zones", strings.Join(fleetErrs, "; ")) input.LaunchTemplateConfigs[0].Overrides = nil + logCreateFleetInput(input) createFleetOutput, err = a.ec2.CreateFleet(context.Background(), input) if err != nil { return createFleetOutput, fmt.Errorf("Unable to create on demand fleet across AZs: %w", err) } + } else { + logrus.Infof("Won't retry CreateFleet across AZs, retry: %v, errors: %s", retry, strings.Join(fleetErrs, "; ")) } if len(createFleetOutput.Errors) > 0 { @@ -650,15 +658,26 @@ func doCreateFleetRetry(cfOutput *ec2.CreateFleetOutput) (bool, []string) { logrus.Infof("Checking to retry fleet create on error %s (msg: %s)", *err.ErrorCode, *err.ErrorMessage) if slices.Contains(retryCodes, *err.ErrorCode) { retry = true + logrus.Infof("doCreateFleetRetry: setting retry to true") } msg = append(msg, fmt.Sprintf("%s: %s", *err.ErrorCode, *err.ErrorMessage)) } // Do not retry in case an instance already exists, in that case just fail and let the worker terminate the SI if len(cfOutput.Instances) > 0 && len(cfOutput.Instances[0].InstanceIds) > 0 { + logrus.Infof("doCreateFleetRetry: cancelling retry, instance already exists: %s", cfOutput.Instances[0].InstanceIds) retry = false msg = append(msg, fmt.Sprintf("Already launched instance (%s), aborting create fleet", cfOutput.Instances[0].InstanceIds)) } + logrus.Infof("doCreateFleetRetry: returning retry: %v, msg: %v", retry, msg) return retry, msg } + +func logCreateFleetInput(input *ec2.CreateFleetInput) { + if inputJSON, err := json.Marshal(input); err != nil { + logrus.Warnf("Unable to marshal input for logging: %v", input) + } else { + logrus.Infof("Creating fleet with input: %s", inputJSON) + } +}