increased client RetryMaxAttempts to 5. added api listing cache

cmd/gofireprox/main.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 )
 
 func main() {
@@ -20,6 +21,8 @@ func main() {
 	command := flag.String("command", "", "Commands: list, create, delete, update")
 	apiID := flag.String("api_id", "", "API ID")
 	proxyURL := flag.String("url", "", "URL end-point")
+	verbose := flag.Bool("verbose", false, "toggles verbosity to reduce API requests that fetch additional verbose data")
+	apiCacheDuration := flag.Duration("cache-duration", 60*time.Second, "sets api list cache duration in seconds. Ex. -cache-duration 120s for 120 seconds or 2m for 2 minutes")
 	flag.Parse()
 
 	fpOptions := &gofireprox.FireProxOptions{
@@ -31,6 +34,8 @@ func main() {
 		Command:         *command,
 		APIID:           *apiID,
 		URL:             *proxyURL,
+		Verbose:         *verbose,
+		CacheDuration:   *apiCacheDuration,
 	}
 
 	fp, err := gofireprox.NewFireProx(fpOptions)

go.mod

@@ -3,21 +3,21 @@ module github.com/mr-pmillz/gofireprox
 go 1.21
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.23.1
-	github.com/aws/aws-sdk-go-v2/config v1.25.4
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.3
-	github.com/aws/aws-sdk-go-v2/service/apigateway v1.20.3
+	github.com/aws/aws-sdk-go-v2 v1.24.0
+	github.com/aws/aws-sdk-go-v2/config v1.26.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.13
+	github.com/aws/aws-sdk-go-v2/service/apigateway v1.21.6
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect
-	github.com/aws/smithy-go v1.17.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
+	github.com/aws/smithy-go v1.19.0 // indirect
 )

go.sum

@@ -1,30 +1,30 @@
-github.com/aws/aws-sdk-go-v2 v1.23.1 h1:qXaFsOOMA+HsZtX8WoCa+gJnbyW7qyFFBlPqvTSzbaI=
-github.com/aws/aws-sdk-go-v2 v1.23.1/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA=
-github.com/aws/aws-sdk-go-v2/config v1.25.4 h1:r+X1x8QI6FEPdJDWCNBDZHyAcyFwSjHN8q8uuus+Axs=
-github.com/aws/aws-sdk-go-v2/config v1.25.4/go.mod h1:8GTjImECskr7D88P/Nn9uM4M4rLY9i77hLJZgkZEWV8=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.3 h1:8PeI2krzzjDJ5etmgaMiD1JswsrLrWvKKu/uBUtNy1g=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.3/go.mod h1:Kdh/okh+//vQ/AjEt81CjvkTo64+/zIE4OewP7RpfXk=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 h1:KehRNiVzIfAcj6gw98zotVbb/K67taJE0fkfgM6vzqU=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5/go.mod h1:VhnExhw6uXy9QzetvpXDolo1/hjhx4u9qukBGkuUwjs=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 h1:LAm3Ycm9HJfbSCd5I+wqC2S9Ej7FPrgr5CQoOljJZcE=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4/go.mod h1:xEhvbJcyUf/31yfGSQBe01fukXwXJ0gxDp7rLfymWE0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 h1:4GV0kKZzUxiWxSVpn/9gwR0g21NF1Jsyduzo9rHgC/Q=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4/go.mod h1:dYvTNAggxDZy6y1AF7YDwXsPuHFy/VNEpEI/2dWK9IU=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
-github.com/aws/aws-sdk-go-v2/service/apigateway v1.20.3 h1:87+aU2h8WFYb5g70IbVZ++i2gb7jITRChktpGzjj3As=
-github.com/aws/aws-sdk-go-v2/service/apigateway v1.20.3/go.mod h1:6sB8lAVH3vfP5U70/wiSyAW+1Geh73EED/VAgeVSxb0=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 h1:rdovz3rEu0vZKbzoMYPTehp0E8veoE9AyfzqCr5Eeao=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4/go.mod h1:aYCGNjyUCUelhofxlZyj63srdxWUSsBSGg5l6MCuXuE=
-github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 h1:CdsSOGlFF3Pn+koXOIpTtvX7st0IuGsZ8kJqcWMlX54=
-github.com/aws/aws-sdk-go-v2/service/sso v1.17.3/go.mod h1:oA6VjNsLll2eVuUoF2D+CMyORgNzPEW/3PyUdq6WQjI=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 h1:cbRqFTVnJV+KRpwFl76GJdIZJKKCdTPnjUZ7uWh3pIU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1/go.mod h1:hHL974p5auvXlZPIjJTblXJpbkfK4klBczlsEaMCGVY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 h1:yEvZ4neOQ/KpUqyR+X0ycUTW/kVRNR4nDZ38wStHGAA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.25.4/go.mod h1:feTnm2Tk/pJxdX+eooEsxvlvTWBvDm6CasRZ+JOs2IY=
-github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI=
-github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
+github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
+github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
+github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/service/apigateway v1.21.6 h1:ePPaOVn92r5n8Neecdpy93hDmR0PBH6H6b7VQCE5vKE=
+github.com/aws/aws-sdk-go-v2/service/apigateway v1.21.6/go.mod h1:P/zwE9uiC6eK/kL3CS60lxTTVC2zAvaS4iW31io41V4=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 h1:HJeiuZ2fldpd0WqngyMR6KW7ofkXNLyOaHwEIGm39Cs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.6/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=

gofireprox.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -18,6 +19,7 @@ import (
 type FireProx struct {
 	Options *FireProxOptions
 	Client  *apigateway.Client
+	Cache   APICache
 }
 
 type FireProxOptions struct {
@@ -29,6 +31,18 @@ type FireProxOptions struct {
 	Command         string
 	APIID           string
 	URL             string
+	Verbose         bool
+	CacheDuration   time.Duration
+}
+
+type RegionCache struct {
+	APIs      []types.RestApi
+	Timestamp time.Time
+}
+
+type APICache struct {
+	Data  map[string]RegionCache
+	Mutex sync.Mutex
 }
 
 // List of all AWS regions as of 2023-05-10 except governmental regions.
@@ -87,7 +101,10 @@ func NewFireProx(opts *FireProxOptions) (*FireProx, error) {
 		log.Fatal(err)
 	}
 
-	client := apigateway.NewFromConfig(cfg)
+	client := apigateway.NewFromConfig(cfg, func(options *apigateway.Options) {
+		// Increase the default retry attempts from 3 to 5 as 3 seems to hit request quotas quite often.
+		options.RetryMaxAttempts = 5
+	})
 	fp := &FireProx{
 		Options: &FireProxOptions{
 			AccessKey:       opts.AccessKey,
@@ -98,6 +115,8 @@ func NewFireProx(opts *FireProxOptions) (*FireProx, error) {
 			Command:         opts.Command,
 			APIID:           opts.APIID,
 			URL:             opts.URL,
+			Verbose:         opts.Verbose,
+			CacheDuration:   opts.CacheDuration * time.Second,
 		},
 		Client: client,
 	}
@@ -110,6 +129,7 @@ func (fp *FireProx) Cleanup() {
 	items, err := fp.ListAPIs()
 	if err != nil {
 		log.Println("Error listing APIs, make sure your aws config/account is properly configured with the appropriate permissions.")
+		log.Println(err.Error())
 	}
 	time.Sleep(5 * time.Second)
 	for _, item := range items {
@@ -323,53 +343,108 @@ func (fp *FireProx) storeAPI(apiID, name, createdAT, targetURL, proxyURL string)
 	fmt.Printf("[%v] (%s) %s %s => %s\n", createdAT, apiID, name, proxyURL, targetURL)
 }
 
+// InvalidateCache clears the cached data for a specific region
+func (fp *FireProx) InvalidateCache(region string) {
+	fp.Cache.Mutex.Lock()
+	defer fp.Cache.Mutex.Unlock()
+	delete(fp.Cache.Data, region) // Remove the cache for the specified region
+}
+
 // ListAPIs ...
 func (fp *FireProx) ListAPIs() ([]types.RestApi, error) {
+	region := fp.Options.Region // Assuming region is part of options
+	fp.Cache.Mutex.Lock()
+	defer fp.Cache.Mutex.Unlock()
+
+	// Initialize region cache if not exists
+	if fp.Cache.Data == nil {
+		fp.Cache.Data = make(map[string]RegionCache)
+	}
+
+	if regionCache, exists := fp.Cache.Data[region]; exists && time.Since(regionCache.Timestamp) < fp.Options.CacheDuration {
+		return regionCache.APIs, nil // Return cached data for the region
+	}
+
 	input := &apigateway.GetRestApisInput{}
 
 	resp, err := fp.Client.GetRestApis(context.TODO(), input)
 	if err != nil {
 		return nil, err
 	}
 
-	apiIDs := make([]string, len(resp.Items))
-	for i, item := range resp.Items {
-		proxyURL, err := fp.getIntegration(aws.ToString(item.Id))
-		if err != nil {
-			return nil, err
+	if fp.Options.Verbose || fp.Options.Command == "list" {
+		apiIDs := make([]string, len(resp.Items))
+		for i, item := range resp.Items {
+			proxyURL, err := fp.getIntegration(aws.ToString(item.Id))
+			if err != nil {
+				return nil, err
+			}
+			proxyURL = strings.ReplaceAll(proxyURL, "{proxy}", "")
+			proxiedURL := fmt.Sprintf("https://%s.execute-api.%s.amazonaws.com/fireprox/", aws.ToString(item.Id), fp.Options.Region)
+			fmt.Printf("[%s] (%s) %s: %s => %s\n", item.CreatedDate.String(), aws.ToString(item.Id), aws.ToString(item.Name), proxiedURL, proxyURL)
+			apiIDs[i] = *item.Id
 		}
-		proxyURL = strings.ReplaceAll(proxyURL, "{proxy}", "")
-		proxiedURL := fmt.Sprintf("https://%s.execute-api.%s.amazonaws.com/fireprox/", aws.ToString(item.Id), fp.Options.Region)
-		fmt.Printf("[%s] (%s) %s: %s => %s\n", item.CreatedDate.String(), aws.ToString(item.Id), aws.ToString(item.Name), proxiedURL, proxyURL)
-		apiIDs[i] = *item.Id
+	}
+
+	// Update the cache for the region
+	fp.Cache.Data[region] = RegionCache{
+		APIs:      resp.Items,
+		Timestamp: time.Now(),
 	}
 
 	return resp.Items, nil
 }
 
-// DeleteAPI ...
-func (fp *FireProx) DeleteAPI(apiID string) bool {
-	items, err := fp.ListAPIs()
-	if err != nil {
-		log.Println("Error listing APIs, make sure your aws config/account is properly configured with the appropriate permissions.")
-		return false
+// IfExistsDeleteAPI ...
+func (fp *FireProx) IfExistsDeleteAPI(apiID string) bool {
+	var items []types.RestApi
+	if regionCache, exists := fp.Cache.Data[fp.Options.Region]; exists && time.Since(regionCache.Timestamp) < fp.Options.CacheDuration {
+		items = regionCache.APIs
+	} else {
+		apis, err := fp.ListAPIs()
+		if err != nil {
+			log.Println("Error listing APIs, make sure your aws config/account is properly configured with the appropriate permissions.")
+			log.Println(err.Error())
+			return false
+		}
+		items = apis
 	}
+
 	time.Sleep(5 * time.Second)
 	for _, item := range items {
 		if apiID == *item.Id {
 			input := &apigateway.DeleteRestApiInput{
 				RestApiId: item.Id,
 			}
-			_, err = fp.Client.DeleteRestApi(context.TODO(), input)
+			_, err := fp.Client.DeleteRestApi(context.TODO(), input)
 			if err != nil {
-				log.Println("[ERROR] Failed to delete API:", aws.ToString(item.Id))
+				log.Printf("[ERROR] Failed to delete API: %+v\nError: %+v\n", aws.ToString(item.Id), err.Error())
+				fp.InvalidateCache(fp.Options.Region) // Invalidate cache for the current region
+				return false
 			}
+			fp.InvalidateCache(fp.Options.Region) // Invalidate cache for the current region
 			return true
 		}
 	}
 	return false
 }
 
+// DeleteAPI directly deletes the apiID without confirming the apiID's existence first
+// this helps to reduce api calls that can quickly exceed the quota
+// it returns true if successful, false if err
+func (fp *FireProx) DeleteAPI(apiID string) bool {
+	input := &apigateway.DeleteRestApiInput{
+		RestApiId: aws.String(apiID),
+	}
+	_, err := fp.Client.DeleteRestApi(context.TODO(), input)
+	if err != nil {
+		log.Printf("[ERROR] Failed to delete API: %+v\nError: %+v\n", apiID, err.Error())
+		return false
+	}
+
+	return true
+}
+
 // CreateAPI ...
 func (fp *FireProx) CreateAPI() (string, string, error) {
 	fmt.Printf("Creating => %s...\n", fp.Options.URL)