From 49e2d52bda54d352f53c806fea7e04e8e7dc245c Mon Sep 17 00:00:00 2001 From: Yevgeny Pats Date: Fri, 5 Feb 2021 16:51:46 +0200 Subject: [PATCH] Migrate AWS provider to standalone repository as a plugin Previous history is at https://github.com/cloudquery/cloudquery --- .github/workflows/release.yml | 24 + .github/workflows/test.yml | 34 ++ .gitignore | 13 + .goreleaser.yaml | 33 ++ Makefile | 31 + autoscaling/client.go | 41 ++ autoscaling/launch_configurations.go | 179 ++++++ cloudtrail/client.go | 41 ++ cloudtrail/event_selectors.go | 35 ++ cloudtrail/trails.go | 158 +++++ cloudwatch/alarms.go | 188 ++++++ cloudwatch/client.go | 41 ++ cloudwatchlogs/client.go | 41 ++ cloudwatchlogs/metric_filters.go | 105 ++++ config.go | 52 ++ directconnect/client.go | 41 ++ directconnect/gateways.go | 72 +++ ec2/byoip_cidr.go | 66 +++ ec2/client.go | 65 +++ ec2/customer_gateways.go | 102 ++++ ec2/flow_logs.go | 122 ++++ ec2/images.go | 239 ++++++++ ec2/instances.go | 646 ++++++++++++++++++++ ec2/internet_gateways.go | 134 +++++ ec2/nat_gateways.go | 168 ++++++ ec2/network_acls.go | 205 +++++++ ec2/route_tabls.go | 242 ++++++++ ec2/security_groups.go | 279 +++++++++ ec2/subnets.go | 165 ++++++ ec2/vpc_peering_connections.go | 272 +++++++++ ec2/vpcs.go | 203 +++++++ ecr/client.go | 41 ++ ecr/images.go | 219 +++++++ ecs/client.go | 41 ++ ecs/clusters.go | 238 ++++++++ ecs/services.go | 361 ++++++++++++ efs/client.go | 41 ++ efs/filesystems.go | 136 +++++ elasticbeanstalk/client.go | 41 ++ elasticbeanstalk/enironments.go | 191 ++++++ elbv2/client.go | 43 ++ elbv2/load_balancers.go | 168 ++++++ elbv2/target_groups.go | 118 ++++ emr/client.go | 41 ++ emr/clusters.go | 94 +++ fsx/backups.go | 126 ++++ fsx/client.go | 41 ++ go.mod | 12 + go.sum | 845 +++++++++++++++++++++++++++ iam/attached_policies.go | 31 + iam/client.go | 49 ++ iam/groups.go | 114 ++++ iam/password_policies.go | 69 +++ iam/policies.go | 141 +++++ iam/roles.go | 188 ++++++ iam/users.go | 343 +++++++++++ iam/virtual_mfa_devices.go | 63 ++ kms/client.go | 41 ++ kms/keys.go | 170 ++++++ organizations/accounts.go | 76 +++ organizations/client.go | 40 ++ provider.go | 339 +++++++++++ rds/certificates.go | 78 +++ rds/client.go | 45 ++ rds/clusters.go | 356 +++++++++++ rds/subnet_groups.go | 114 ++++ redshift/client.go | 43 ++ redshift/clusters.go | 509 ++++++++++++++++ redshift/subnet_groups.go | 162 +++++ resource/client.go | 5 + s3/buckets.go | 297 ++++++++++ s3/client.go | 45 ++ sns/client.go | 41 ++ sns/subscriptions.go | 72 +++ 74 files changed, 10265 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 .goreleaser.yaml create mode 100644 Makefile create mode 100644 autoscaling/client.go create mode 100644 autoscaling/launch_configurations.go create mode 100644 cloudtrail/client.go create mode 100644 cloudtrail/event_selectors.go create mode 100644 cloudtrail/trails.go create mode 100644 cloudwatch/alarms.go create mode 100644 cloudwatch/client.go create mode 100644 cloudwatchlogs/client.go create mode 100644 cloudwatchlogs/metric_filters.go create mode 100644 config.go create mode 100644 directconnect/client.go create mode 100644 directconnect/gateways.go create mode 100644 ec2/byoip_cidr.go create mode 100644 ec2/client.go create mode 100644 ec2/customer_gateways.go create mode 100644 ec2/flow_logs.go create mode 100644 ec2/images.go create mode 100644 ec2/instances.go create mode 100644 ec2/internet_gateways.go create mode 100644 ec2/nat_gateways.go create mode 100644 ec2/network_acls.go create mode 100644 ec2/route_tabls.go create mode 100644 ec2/security_groups.go create mode 100644 ec2/subnets.go create mode 100644 ec2/vpc_peering_connections.go create mode 100644 ec2/vpcs.go create mode 100644 ecr/client.go create mode 100644 ecr/images.go create mode 100644 ecs/client.go create mode 100644 ecs/clusters.go create mode 100644 ecs/services.go create mode 100644 efs/client.go create mode 100644 efs/filesystems.go create mode 100644 elasticbeanstalk/client.go create mode 100644 elasticbeanstalk/enironments.go create mode 100644 elbv2/client.go create mode 100644 elbv2/load_balancers.go create mode 100644 elbv2/target_groups.go create mode 100644 emr/client.go create mode 100644 emr/clusters.go create mode 100644 fsx/backups.go create mode 100644 fsx/client.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 iam/attached_policies.go create mode 100644 iam/client.go create mode 100644 iam/groups.go create mode 100644 iam/password_policies.go create mode 100644 iam/policies.go create mode 100644 iam/roles.go create mode 100644 iam/users.go create mode 100644 iam/virtual_mfa_devices.go create mode 100644 kms/client.go create mode 100644 kms/keys.go create mode 100644 organizations/accounts.go create mode 100644 organizations/client.go create mode 100644 provider.go create mode 100644 rds/certificates.go create mode 100644 rds/client.go create mode 100644 rds/clusters.go create mode 100644 rds/subnet_groups.go create mode 100644 redshift/client.go create mode 100644 redshift/clusters.go create mode 100644 redshift/subnet_groups.go create mode 100644 resource/client.go create mode 100644 s3/buckets.go create mode 100644 s3/client.go create mode 100644 sns/client.go create mode 100644 sns/subscriptions.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..726c6b0e5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,24 @@ +name: release +on: + push: + tags: + - '*' +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: checkout code + uses: actions/checkout@v2 + with: + submodules: 'true' + - run: git fetch --prune --unshallow + - name: setup dependencies + uses: actions/setup-go@v2 + - name: release dry run + run: make release-dry-run + - name: setup release environment + run: |- + echo "${{secrets.GORELEASER_GITHUB_TOKEN}}" | docker login ghcr.io --username $GITHUB_ACTOR --password-stdin + echo 'GITHUB_TOKEN=${{secrets.GORELEASER_GITHUB_TOKEN}}' > .release-env + - name: release publish + run: make release diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..27aeab276 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +name: test + +on: + push: + branches: + - main + pull_request: + branches: [ main ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.14 + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + go get -v -t -d ./... + + - name: Build + run: go build -v . + + - name: Test + run: go test -v ./... --tags=integration + diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..d8e072a14 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +.DS_store +crash.log +.idea +.vscode +cloudquery +cloudquery.db +cloudquery.zip +/config.yml +dist/ +vendor/ +bin/ +.release-env +cq-provider-aws diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 000000000..fb453ad44 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,33 @@ +before: + hooks: + - go mod download + - go generate ./... +builds: + - flags: + - -buildmode=exe + env: + - CGO_ENABLED=1 + - GO111MODULE=on + - CC=$PWD/gencc.sh + - CXX=$PWD/gencpp.sh + main: ./provider.go + ldflags: + - -s -w -X github.com/cloudquery/cloudquery/cmd.Version={{.Version}} -X github.com/cloudquery/cloudquery/cmd.Commit={{.Commit}} -X github.com/cloudquery/cloudquery/cmd.Date={{.Date}} + goos: + - windows + - linux + - darwin + goarch: + - amd64 +archives: + - + name_template: "{{ .Binary }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" + format: binary +checksum: + name_template: 'checksums.txt' +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..95dec474a --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +PACKAGE_NAME := github.com/troian/golang-cross-example +GOLANG_CROSS_VERSION ?= v1.15.2 + +.PHONY: release-dry-run +release-dry-run: + @docker run \ + --privileged \ + -e CGO_ENABLED=1 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v `pwd`:/go/src/$(PACKAGE_NAME) \ + -w /go/src/$(PACKAGE_NAME) \ + troian/golang-cross:${GOLANG_CROSS_VERSION} \ + --rm-dist --skip-validate --skip-publish + +.PHONY: release +release: + @if [ ! -f ".release-env" ]; then \ + echo "\033[91m.release-env is required for release\033[0m";\ + exit 1;\ + fi + docker run \ + --rm \ + --privileged \ + -e CGO_ENABLED=1 \ + --env-file .release-env \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ~/.docker:/root/.docker \ + -v `pwd`:/go/src/$(PACKAGE_NAME) \ + -w /go/src/$(PACKAGE_NAME) \ + troian/golang-cross:${GOLANG_CROSS_VERSION} \ + release --rm-dist diff --git a/autoscaling/client.go b/autoscaling/client.go new file mode 100644 index 000000000..b7e381a2b --- /dev/null +++ b/autoscaling/client.go @@ -0,0 +1,41 @@ +package autoscaling + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *autoscaling.AutoScaling +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: autoscaling.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "launch_configurations": + return c.launchConfigurations(config) + default: + return fmt.Errorf("unsupported resource autoscaling.%s", resource) + } +} diff --git a/autoscaling/launch_configurations.go b/autoscaling/launch_configurations.go new file mode 100644 index 000000000..ba5d0a692 --- /dev/null +++ b/autoscaling/launch_configurations.go @@ -0,0 +1,179 @@ +package autoscaling + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/cloudquery/cloudquery/providers/common" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type LaunchConfiguration struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + + // AWS account id + AccountID string + + Region string + AssociatePublicIpAddress *bool + BlockDeviceMappings []*LaunchConfigurationBlockDeviceMapping `gorm:"constraint:OnDelete:CASCADE;"` + ClassicLinkVPCId *string + ClassicLinkVPCSecurityGroups *string + CreatedTime *time.Time + EbsOptimized *bool + IamInstanceProfile *string + ImageId *string + InstanceMonitoringEnabled *bool + InstanceType *string + KernelId *string + KeyName *string + + LaunchConfigurationARN *string `neo:"unique"` + LaunchConfigurationName *string + + MetadataHttpEndpoint *string + MetadataHttpPutResponseHopLimit *int64 + MetadataHttpTokens *string + + PlacementTenancy *string + RamdiskId *string + SecurityGroups *string + SpotPrice *string + UserData *string +} + +func (LaunchConfiguration) TableName() string { + return "aws_autoscaling_launch_configurations" +} + +type LaunchConfigurationBlockDeviceMapping struct { + ID uint `gorm:"primarykey"` + LaunchConfigurationID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + // The device name exposed to the EC2 instance (for example, /dev/sdh or xvdh). + // For more information, see Device Naming on Linux Instances (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html) + // in the Amazon EC2 User Guide for Linux Instances. + DeviceName *string + + EbsDeleteOnTermination *bool + EbsEncrypted *bool + EbsIops *int64 + EbsSnapshotId *string + EbsVolumeSize *int64 + EbsVolumeType *string + + // If NoDevice is true for the root device, instances might fail the EC2 health + // check. In that case, Amazon EC2 Auto Scaling launches replacement instances. + NoDevice *bool + + // The name of the virtual device (for example, ephemeral0). + VirtualName *string +} + +func (LaunchConfigurationBlockDeviceMapping) TableName() string { + return "aws_autoscaling_launch_configuration_block_device_mapping" +} + +func (c *Client) transformLaunchConfigurationBlockDeviceMapping(value *autoscaling.BlockDeviceMapping) *LaunchConfigurationBlockDeviceMapping { + res := LaunchConfigurationBlockDeviceMapping{ + AccountID: c.accountID, + Region: c.region, + DeviceName: value.DeviceName, + NoDevice: value.NoDevice, + VirtualName: value.VirtualName, + } + if value.Ebs != nil { + res.EbsDeleteOnTermination = value.Ebs.DeleteOnTermination + res.EbsEncrypted = value.Ebs.Encrypted + res.EbsIops = value.Ebs.Iops + res.EbsSnapshotId = value.Ebs.SnapshotId + res.EbsVolumeSize = value.Ebs.VolumeSize + res.EbsVolumeType = value.Ebs.VolumeType + } + return &res +} + +func (c *Client) transformLaunchConfigurationBlockDeviceMappings(values []*autoscaling.BlockDeviceMapping) []*LaunchConfigurationBlockDeviceMapping { + var tValues []*LaunchConfigurationBlockDeviceMapping + for _, v := range values { + tValues = append(tValues, c.transformLaunchConfigurationBlockDeviceMapping(v)) + } + return tValues +} + +func (c *Client) transformLaunchConfiguration(value *autoscaling.LaunchConfiguration) *LaunchConfiguration { + res := LaunchConfiguration{ + Region: c.region, + AccountID: c.accountID, + AssociatePublicIpAddress: value.AssociatePublicIpAddress, + BlockDeviceMappings: c.transformLaunchConfigurationBlockDeviceMappings(value.BlockDeviceMappings), + ClassicLinkVPCId: value.ClassicLinkVPCId, + ClassicLinkVPCSecurityGroups: common.StringListToString(value.ClassicLinkVPCSecurityGroups), + CreatedTime: value.CreatedTime, + EbsOptimized: value.EbsOptimized, + IamInstanceProfile: value.IamInstanceProfile, + ImageId: value.ImageId, + InstanceType: value.InstanceType, + KernelId: value.KernelId, + KeyName: value.KeyName, + LaunchConfigurationARN: value.LaunchConfigurationARN, + LaunchConfigurationName: value.LaunchConfigurationName, + PlacementTenancy: value.PlacementTenancy, + RamdiskId: value.RamdiskId, + SecurityGroups: common.StringListToString(value.SecurityGroups), + SpotPrice: value.SpotPrice, + UserData: value.UserData, + } + + if value.MetadataOptions != nil { + res.MetadataHttpEndpoint = value.MetadataOptions.HttpEndpoint + res.MetadataHttpPutResponseHopLimit = value.MetadataOptions.HttpPutResponseHopLimit + res.MetadataHttpTokens = value.MetadataOptions.HttpTokens + } + + if value.InstanceMonitoring != nil { + res.InstanceMonitoringEnabled = value.InstanceMonitoring.Enabled + } + + return &res +} + +func (c *Client) transformLaunchConfigurations(values []*autoscaling.LaunchConfiguration) []*LaunchConfiguration { + var tValues []*LaunchConfiguration + for _, v := range values { + tValues = append(tValues, c.transformLaunchConfiguration(v)) + } + return tValues +} + +var LaunchConfigurationTables = []interface{}{ + &LaunchConfiguration{}, + &LaunchConfigurationBlockDeviceMapping{}, +} + +func (c *Client) launchConfigurations(gConfig interface{}) error { + var config autoscaling.DescribeLaunchConfigurationsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(LaunchConfigurationTables...) + for { + output, err := c.svc.DescribeLaunchConfigurations(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformLaunchConfigurations(output.LaunchConfigurations)) + c.log.Info("Fetched resources", zap.String("resource", "auto_scaling.launch_configurations"), zap.Int("count", len(output.LaunchConfigurations))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/cloudtrail/client.go b/cloudtrail/client.go new file mode 100644 index 000000000..fff4fa0ae --- /dev/null +++ b/cloudtrail/client.go @@ -0,0 +1,41 @@ +package cloudtrail + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudtrail" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *cloudtrail.CloudTrail +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: cloudtrail.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "trails": + return c.trails(config) + default: + return fmt.Errorf("unsupported resource autoscaling.%s", resource) + } +} diff --git a/cloudtrail/event_selectors.go b/cloudtrail/event_selectors.go new file mode 100644 index 000000000..c8a15ff19 --- /dev/null +++ b/cloudtrail/event_selectors.go @@ -0,0 +1,35 @@ +package cloudtrail + +import ( + "github.com/aws/aws-sdk-go/service/cloudtrail" +) + + +type EventSelector struct { + ID uint `gorm:"primarykey"` + TrailID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + IncludeManagementEvents *bool + ReadWriteType *string +} + +func (EventSelector) TableName() string { + return "aws_cloudtrail_trail_event_selectors" +} + + +func (c *Client) transformEventSelectors(values []*cloudtrail.EventSelector) []*EventSelector { + var tValues []*EventSelector + for _, value := range values { + tValue := EventSelector { + AccountID: c.accountID, + Region: c.region, + IncludeManagementEvents: value.IncludeManagementEvents, + ReadWriteType: value.ReadWriteType, + } + tValues = append(tValues, &tValue) + } + return tValues +} + diff --git a/cloudtrail/trails.go b/cloudtrail/trails.go new file mode 100644 index 000000000..cec2b9cfa --- /dev/null +++ b/cloudtrail/trails.go @@ -0,0 +1,158 @@ +package cloudtrail + +import ( + "github.com/aws/aws-sdk-go/service/cloudtrail" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "regexp" + "time" +) + +type Trail struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + CloudWatchLogsLogGroupArn *string + CloudWatchLogsLogGroupName *string + CloudWatchLogsRoleArn *string + HasCustomEventSelectors *bool + HasInsightSelectors *bool + HomeRegion *string + IncludeGlobalServiceEvents *bool + IsMultiRegionTrail *bool + IsOrganizationTrail *bool + KmsKeyId *string + LogFileValidationEnabled *bool + Name *string + S3BucketName *string + S3KeyPrefix *string + SnsTopicARN *string + SnsTopicName *string + TrailARN *string `neo:"unique"` + EventSelectors []*EventSelector `gorm:"constraint:OnDelete:CASCADE;"` + IsLogging *bool + + // Status + LatestCloudWatchLogsDeliveryError *string + LatestCloudWatchLogsDeliveryTime *time.Time + LatestDeliveryAttemptSucceeded *string + LatestDeliveryAttemptTime *string + LatestDeliveryError *string + LatestDeliveryTime *time.Time + LatestDigestDeliveryError *string + LatestDigestDeliveryTime *time.Time + LatestNotificationAttemptSucceeded *string + LatestNotificationAttemptTime *string + LatestNotificationError *string + LatestNotificationTime *time.Time + StartLoggingTime *time.Time + StopLoggingTime *time.Time + TimeLoggingStarted *string + TimeLoggingStopped *string +} + +func (Trail) TableName() string { + return "aws_cloudtrail_trails" +} + +//log-group:([a-zA-Z0-9/]+): +var groupNameRegex = regexp.MustCompile("arn:aws:logs:[a-z0-9-]+:[0-9]+:log-group:([a-zA-Z0-9-/]+):") + +func (c *Client) transformTrails(values []*cloudtrail.Trail) ([]*Trail, error) { + var tValues []*Trail + for _, value := range values { + groupName := "" + if value.CloudWatchLogsLogGroupArn != nil { + matches := groupNameRegex.FindStringSubmatch(*value.CloudWatchLogsLogGroupArn) + if len(matches) < 2 { + c.log.Warn("CloudWatchLogsLogGroupARN doesn't fit standard regex", zap.String("arn", *value.CloudWatchLogsLogGroupArn)) + } else { + groupName = matches[1] + } + } else { + c.log.Info("CloudWatchLogsLogGroupARN is empty") + } + + statusOutput, err := c.svc.GetTrailStatus(&cloudtrail.GetTrailStatusInput{Name: value.TrailARN}) + if err != nil { + return nil, err + } + res := Trail{ + Region: c.region, + AccountID: c.accountID, + CloudWatchLogsLogGroupArn: value.CloudWatchLogsLogGroupArn, + CloudWatchLogsLogGroupName: &groupName, + CloudWatchLogsRoleArn: value.CloudWatchLogsRoleArn, + HasCustomEventSelectors: value.HasCustomEventSelectors, + HasInsightSelectors: value.HasInsightSelectors, + HomeRegion: value.HomeRegion, + IncludeGlobalServiceEvents: value.IncludeGlobalServiceEvents, + IsMultiRegionTrail: value.IsMultiRegionTrail, + IsOrganizationTrail: value.IsOrganizationTrail, + KmsKeyId: value.KmsKeyId, + LogFileValidationEnabled: value.LogFileValidationEnabled, + Name: value.Name, + S3BucketName: value.S3BucketName, + S3KeyPrefix: value.S3KeyPrefix, + SnsTopicARN: value.SnsTopicARN, + SnsTopicName: value.SnsTopicName, + TrailARN: value.TrailARN, + IsLogging: statusOutput.IsLogging, + LatestCloudWatchLogsDeliveryError: statusOutput.LatestCloudWatchLogsDeliveryError, + LatestCloudWatchLogsDeliveryTime: statusOutput.LatestCloudWatchLogsDeliveryTime, + LatestDeliveryAttemptSucceeded: statusOutput.LatestDeliveryAttemptSucceeded, + LatestDeliveryAttemptTime: statusOutput.LatestDeliveryAttemptTime, + LatestDeliveryError: statusOutput.LatestDeliveryError, + LatestDeliveryTime: statusOutput.LatestDeliveryTime, + LatestDigestDeliveryError: statusOutput.LatestDigestDeliveryError, + LatestDigestDeliveryTime: statusOutput.LatestDigestDeliveryTime, + LatestNotificationAttemptSucceeded: statusOutput.LatestNotificationAttemptSucceeded, + LatestNotificationAttemptTime: statusOutput.LatestNotificationAttemptTime, + LatestNotificationError: statusOutput.LatestNotificationError, + LatestNotificationTime: statusOutput.LatestNotificationTime, + StartLoggingTime: statusOutput.StartLoggingTime, + StopLoggingTime: statusOutput.StopLoggingTime, + TimeLoggingStarted: statusOutput.TimeLoggingStarted, + TimeLoggingStopped: statusOutput.TimeLoggingStopped, + } + + output, err := c.svc.GetEventSelectors(&cloudtrail.GetEventSelectorsInput{TrailName: value.TrailARN}) + if err != nil { + return nil, err + } + res.EventSelectors = c.transformEventSelectors(output.EventSelectors) + + tValues = append(tValues, &res) + } + + return tValues, nil +} + +var TrailTables = []interface{}{ + &Trail{}, + + &EventSelector{}, +} + +func (c *Client) trails(gConfig interface{}) error { + var config cloudtrail.DescribeTrailsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + + output, err := c.svc.DescribeTrails(&config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(TrailTables...) + tValues, err := c.transformTrails(output.TrailList) + if err != nil { + return err + } + c.db.ChunkedCreate(tValues) + c.log.Info("Fetched resources", zap.String("resource", "cloudtrail.trails"), zap.Int("count", len(output.TrailList))) + + return nil +} diff --git a/cloudwatch/alarms.go b/cloudwatch/alarms.go new file mode 100644 index 000000000..cc16773c2 --- /dev/null +++ b/cloudwatch/alarms.go @@ -0,0 +1,188 @@ +package cloudwatch + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + + +type MetricAlarm struct { + ID uint `gorm:"primarykey"` + AccountID string + Region string + ActionsEnabled *bool + AlarmActions []*MetricAlarmActions `gorm:"constraint:OnDelete:CASCADE;"` + AlarmArn *string + AlarmConfigurationUpdatedTimestamp *time.Time + AlarmDescription *string + AlarmName *string + ComparisonOperator *string + DatapointsToAlarm *int64 + EvaluateLowSampleCountPercentile *string + EvaluationPeriods *int64 + ExtendedStatistic *string + MetricName *string + Metrics []*MetricAlarmMetric `gorm:"constraint:OnDelete:CASCADE;"` + Namespace *string + Period *int64 + StateReason *string + StateReasonData *string + StateUpdatedTimestamp *time.Time + StateValue *string + Statistic *string + Threshold *float64 + ThresholdMetricId *string + TreatMissingData *string + Unit *string +} + +func (MetricAlarm) TableName() string { + return "aws_cloudwatch_metric_alarms" +} +type MetricAlarmActions struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + MetricAlarmID uint `neo:"ignore"` + Value * string +} + +func (MetricAlarmActions) TableName() string { + return "aws_cloudwatch_metric_alarm_actions" +} + +type MetricAlarmMetric struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + MetricAlarmID uint `neo:"ignore"` + Expression *string + ResourceID *string + Label *string + + Name *string + Namespace *string + + StatPeriod *int64 + StatStat *string + StatUnit *string + + Period *int64 + ReturnData *bool +} + +func (MetricAlarmMetric) TableName() string { + return "aws_cloudwatch_metric_alarm_metrics" +} + + +func (c *Client) transformMetricAlarms(values []*cloudwatch.MetricAlarm) []*MetricAlarm { + var tValues []*MetricAlarm + for _, value := range values { + tValue := MetricAlarm { + AccountID: c.accountID, + Region: c.region, + ActionsEnabled: value.ActionsEnabled, + AlarmActions: c.transformMetricAlarmActions(value.AlarmActions), + AlarmArn: value.AlarmArn, + AlarmConfigurationUpdatedTimestamp: value.AlarmConfigurationUpdatedTimestamp, + AlarmDescription: value.AlarmDescription, + AlarmName: value.AlarmName, + ComparisonOperator: value.ComparisonOperator, + DatapointsToAlarm: value.DatapointsToAlarm, + EvaluateLowSampleCountPercentile: value.EvaluateLowSampleCountPercentile, + EvaluationPeriods: value.EvaluationPeriods, + ExtendedStatistic: value.ExtendedStatistic, + MetricName: value.MetricName, + Metrics: c.transformMetricAlarmMetrics(value.Metrics), + Namespace: value.Namespace, + Period: value.Period, + StateReason: value.StateReason, + StateReasonData: value.StateReasonData, + StateUpdatedTimestamp: value.StateUpdatedTimestamp, + StateValue: value.StateValue, + Statistic: value.Statistic, + Threshold: value.Threshold, + ThresholdMetricId: value.ThresholdMetricId, + TreatMissingData: value.TreatMissingData, + Unit: value.Unit, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformMetricAlarmMetrics(values []*cloudwatch.MetricDataQuery) []*MetricAlarmMetric { + var tValues []*MetricAlarmMetric + for _, value := range values { + tValue := MetricAlarmMetric{ + AccountID: c.accountID, + Region: c.region, + Expression: value.Expression, + ResourceID: value.Id, + Label: value.Label, + Period: value.Period, + ReturnData: value.ReturnData, + } + if value.MetricStat != nil { + tValue.StatPeriod = value.MetricStat.Period + tValue.StatStat = value.MetricStat.Stat + tValue.StatUnit = value.MetricStat.Unit + if value.MetricStat.Metric != nil { + tValue.Name = value.MetricStat.Metric.MetricName + tValue.Namespace = value.MetricStat.Metric.Namespace + } + + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformMetricAlarmActions(values []*string) []*MetricAlarmActions { + var tValues []*MetricAlarmActions + for _, v := range values { + tValues = append(tValues, &MetricAlarmActions{ + Value: v, + }) + } + return tValues +} + +type MetricAlarmConfig struct { + Filter string +} + +var MetricAlarmTables = []interface{} { + &MetricAlarm{}, + &MetricAlarmActions{}, + &MetricAlarmMetric{}, +} + +func (c *Client)alarms(gConfig interface{}) error { + var config cloudwatch.DescribeAlarmsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(MetricAlarmTables...) + + for { + output, err := c.svc.DescribeAlarms(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformMetricAlarms(output.MetricAlarms)) + c.log.Info("Fetched resources", zap.String("resource", "cloudwatch.alarms"), zap.Int("count", len(output.MetricAlarms))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + + return nil +} + diff --git a/cloudwatch/client.go b/cloudwatch/client.go new file mode 100644 index 000000000..e057fc5d3 --- /dev/null +++ b/cloudwatch/client.go @@ -0,0 +1,41 @@ +package cloudwatch + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *cloudwatch.CloudWatch +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: cloudwatch.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "alarms": + return c.alarms(config) + default: + return fmt.Errorf("unsupported resource autoscaling.%s", resource) + } +} diff --git a/cloudwatchlogs/client.go b/cloudwatchlogs/client.go new file mode 100644 index 000000000..1c87786e9 --- /dev/null +++ b/cloudwatchlogs/client.go @@ -0,0 +1,41 @@ +package cloudwatchlogs + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *cloudwatchlogs.CloudWatchLogs +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: cloudwatchlogs.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "metric_filters": + return c.metricFilters(config) + default: + return fmt.Errorf("unsupported resource autoscaling.%s", resource) + } +} diff --git a/cloudwatchlogs/metric_filters.go b/cloudwatchlogs/metric_filters.go new file mode 100644 index 000000000..e54edd056 --- /dev/null +++ b/cloudwatchlogs/metric_filters.go @@ -0,0 +1,105 @@ +package cloudwatchlogs + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + + +type MetricFilter struct { + ID uint `gorm:"primarykey"` + AccountID string + Region string + CreationTime *int64 + FilterName *string + FilterPattern *string + LogGroupName *string + Transformations []*MetricFilterTransformations `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (MetricFilter) TableName() string { + return "aws_cloudwatchlogs_metric_filters" +} + +type MetricFilterTransformations struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + MetricFilterID uint `neo:"ignore"` + DefaultValue *float64 + MetricName *string + MetricNamespace *string + MetricValue *string +} + +func (MetricFilterTransformations) TableName() string { + return "aws_cloudwatchlogs_metric_filter_transformations" +} + +func (c *Client) transformMetricFilters(values []*cloudwatchlogs.MetricFilter) []*MetricFilter { + var tValues []*MetricFilter + for _, value := range values { + tValue := MetricFilter { + AccountID: c.accountID, + Region: c.region, + CreationTime: value.CreationTime, + FilterName: value.FilterName, + FilterPattern: value.FilterPattern, + LogGroupName: value.LogGroupName, + Transformations: c.transformMetricFilterMetricTransformations(value.MetricTransformations), + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformMetricFilterMetricTransformations(values []*cloudwatchlogs.MetricTransformation) []*MetricFilterTransformations { + var tValues []*MetricFilterTransformations + for _, value := range values { + tValue := MetricFilterTransformations{ + AccountID: c.accountID, + Region: c.region, + DefaultValue: value.DefaultValue, + MetricName: value.MetricName, + MetricNamespace: value.MetricNamespace, + MetricValue: value.MetricValue, + } + tValues = append(tValues, &tValue) + } + return tValues +} +type MetricFilterConfig struct { + Filter string +} + +var MetricFilterTables = []interface{} { + &MetricFilter{}, + &MetricFilterTransformations{}, +} + +func (c *Client)metricFilters(gConfig interface{}) error { + var config cloudwatchlogs.DescribeMetricFiltersInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(MetricFilterTables...) + + for { + output, err := c.svc.DescribeMetricFilters(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformMetricFilters(output.MetricFilters)) + c.log.Info("Fetched resources", zap.String("resource", "cloudwatchlogs.metric_filters"), zap.Int("count", len(output.MetricFilters))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + + return nil +} + diff --git a/config.go b/config.go new file mode 100644 index 000000000..4ee3ad138 --- /dev/null +++ b/config.go @@ -0,0 +1,52 @@ +package main + +const configYaml = ` + - name: aws +# accounts: # Optional. if you want to assume role to multiple account and fetch data from them +# - role_arn: +# regions: # Optional. if commented out assumes all regions +# - us-east-1 +# - us-west-2 +# log_level: debug # Optional. if commented out will enable AWS SDK debug logging. possible values: debug, debug_with_signing, debug_with_http_body, debug_with_request_retries, debug_with_request_error, debug_with_event_stream_body +# max_retries: # Optional. The maximum number of times that a request will be retried for failures. Defaults to -1, which defers the max retry setting to the service specific configuration. + resources: # You can comment resources your are not interested in for faster fetching. + - name: autoscaling.launch_configurations + - name: cloudtrail.trails + - name: cloudwatch.alarms + - name: cloudwatchlogs.metric_filters + - name: directconnect.gateways + - name: ec2.customer_gateways + - name: ec2.flow_logs + - name: ec2.images + - name: ec2.instances + - name: ec2.internet_gateways + - name: ec2.nat_gateways + - name: ec2.network_acls + - name: ec2.route_tables + - name: ec2.security_groups + - name: ec2.subnets + - name: ec2.vpc_peering_connections + - name: ec2.vpcs + - name: ecs.clusters + - name: ecr.images + - name: efs.filesystems + - name: elasticbeanstalk.environments + - name: elbv2.load_balancers + - name: elbv2.target_groups + - name: emr.clusters + - name: fsx.backups + - name: iam.groups + - name: iam.password_policies + - name: iam.policies + - name: iam.roles + - name: iam.users + - name: iam.virtual_mfa_devices + - name: kms.keys + - name: organizations.accounts + - name: rds.certificates + - name: rds.clusters + - name: rds.db_subnet_groups + - name: redshift.clusters + - name: redshift.cluster_subnet_groups + - name: s3.buckets + - name: sns.subscriptions` diff --git a/directconnect/client.go b/directconnect/client.go new file mode 100644 index 000000000..09b285e0b --- /dev/null +++ b/directconnect/client.go @@ -0,0 +1,41 @@ +package directconnect + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/directconnect" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *directconnect.DirectConnect +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: directconnect.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "gateways": + return c.gateways(config) + default: + return fmt.Errorf("unsupported resource directconnect.%s", resource) + } +} diff --git a/directconnect/gateways.go b/directconnect/gateways.go new file mode 100644 index 000000000..42abc83ab --- /dev/null +++ b/directconnect/gateways.go @@ -0,0 +1,72 @@ +package directconnect + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/directconnect" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type Gateway struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + AmazonSideAsn *int64 + DirectConnectGatewayId *string `neo:"unique"` + DirectConnectGatewayName *string + DirectConnectGatewayState *string + OwnerAccount *string + StateChangeError *string +} + +func (Gateway) TableName() string { + return "aws_directconnect_gateways" +} + +func (c *Client) transformGateway(value *directconnect.Gateway) *Gateway { + return &Gateway{ + Region: c.region, + AccountID: c.accountID, + AmazonSideAsn: value.AmazonSideAsn, + DirectConnectGatewayId: value.DirectConnectGatewayId, + DirectConnectGatewayName: value.DirectConnectGatewayName, + DirectConnectGatewayState: value.DirectConnectGatewayState, + OwnerAccount: value.OwnerAccount, + StateChangeError: value.StateChangeError, + } +} + +func (c *Client) transformGateways(values []*directconnect.Gateway) []*Gateway { + var tValues []*Gateway + for _, v := range values { + tValues = append(tValues, c.transformGateway(v)) + } + return tValues +} + +var GatewayTables = []interface{}{ + &Gateway{}, +} + +func (c *Client) gateways(gConfig interface{}) error { + var config directconnect.DescribeDirectConnectGatewaysInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(GatewayTables...) + for { + output, err := c.svc.DescribeDirectConnectGateways(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformGateways(output.DirectConnectGateways)) + c.log.Info("Fetched resources", zap.String("resource", "directconnect.gateways"), zap.Int("count", len(output.DirectConnectGateways))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/byoip_cidr.go b/ec2/byoip_cidr.go new file mode 100644 index 000000000..32457474d --- /dev/null +++ b/ec2/byoip_cidr.go @@ -0,0 +1,66 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "go.uber.org/zap" +) + +type ByoipCidr struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + Cidr *string + Description *string + State *string + StatusMessage *string +} + +func (ByoipCidr) TableName() string { + return "aws_ec2_byoip_cidrs" +} + +func (c *Client) transformByoipCidr(value *ec2.ByoipCidr) *ByoipCidr { + return &ByoipCidr{ + Region: c.region, + AccountID: c.accountID, + Cidr: value.Cidr, + Description: value.Description, + State: value.State, + StatusMessage: value.StatusMessage, + } +} + +func (c *Client) transformByoipCidrs(values []*ec2.ByoipCidr) []*ByoipCidr { + var tValues []*ByoipCidr + for _, v := range values { + tValues = append(tValues, c.transformByoipCidr(v)) + } + return tValues +} + +var ByoipCidrTables = []interface{}{ + &ByoipCidr{}, +} + +func (c *Client) byoipCidrs(_ interface{}) error { + MaxResults := int64(100) + config := ec2.DescribeByoipCidrsInput{ + MaxResults: &MaxResults, + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(ByoipCidrTables...) + for { + output, err := c.svc.DescribeByoipCidrs(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformByoipCidrs(output.ByoipCidrs)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.byoip_cidrs"), zap.Int("count", len(output.ByoipCidrs))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/client.go b/ec2/client.go new file mode 100644 index 000000000..d7d4c57f1 --- /dev/null +++ b/ec2/client.go @@ -0,0 +1,65 @@ +package ec2 + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *ec2.EC2 +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: ec2.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "images": + return c.images(config) + case "instances": + return c.instances(config) + case "byoip_cidrs": + return c.byoipCidrs(config) + case "customer_gateways": + return c.customerGateways(config) + case "internet_gateways": + return c.internetGateways(config) + case "nat_gateways": + return c.natGateways(config) + case "network_acls": + return c.networkAcls(config) + case "route_tables": + return c.routeTables(config) + case "security_groups": + return c.securityGroups(config) + case "vpcs": + return c.vpcs(config) + case "subnets": + return c.subnets(config) + case "flow_logs": + return c.FlowLogs(config) + case "vpc_peering_connections": + return c.vpcPeeringConnections(config) + default: + return fmt.Errorf("unsupported resource ec2.%s", resource) + } +} diff --git a/ec2/customer_gateways.go b/ec2/customer_gateways.go new file mode 100644 index 000000000..4aeceb13d --- /dev/null +++ b/ec2/customer_gateways.go @@ -0,0 +1,102 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type CustomerGateway struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + BgpAsn *string + CertificateArn *string + CustomerGatewayId *string `neo:"unique"` + DeviceName *string + IpAddress *string + State *string + Tags []*CustomerGatewayTag `gorm:"constraint:OnDelete:CASCADE;"` + Type *string +} + +func (CustomerGateway) TableName() string { + return "aws_ec2_customer_gateways" +} + +type CustomerGatewayTag struct { + ID uint `gorm:"primarykey"` + CustomerGatewayID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (CustomerGatewayTag) TableName() string { + return "aws_ec2_customer_gateway_tags" +} + +func (c *Client) transformCustomerGatewayTag(value *ec2.Tag) *CustomerGatewayTag { + return &CustomerGatewayTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformCustomerGatewayTags(values []*ec2.Tag) []*CustomerGatewayTag { + var tValues []*CustomerGatewayTag + for _, v := range values { + tValues = append(tValues, c.transformCustomerGatewayTag(v)) + } + return tValues +} + +func (c *Client) transformCustomerGateway(value *ec2.CustomerGateway) *CustomerGateway { + return &CustomerGateway{ + Region: c.region, + AccountID: c.accountID, + BgpAsn: value.BgpAsn, + CertificateArn: value.CertificateArn, + CustomerGatewayId: value.CustomerGatewayId, + DeviceName: value.DeviceName, + IpAddress: value.IpAddress, + State: value.State, + Tags: c.transformCustomerGatewayTags(value.Tags), + Type: value.Type, + } +} + +func (c *Client) transformCustomerGateways(values []*ec2.CustomerGateway) []*CustomerGateway { + var tValues []*CustomerGateway + for _, v := range values { + tValues = append(tValues, c.transformCustomerGateway(v)) + } + return tValues +} + +var CustomerGatewayTables = []interface{}{ + &CustomerGateway{}, + &CustomerGatewayTag{}, +} + +func (c *Client) customerGateways(gConfig interface{}) error { + var config ec2.DescribeCustomerGatewaysInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + + output, err := c.svc.DescribeCustomerGateways(&config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(CustomerGatewayTables...) + c.db.ChunkedCreate(c.transformCustomerGateways(output.CustomerGateways)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.customer_gateways"), zap.Int("count", len(output.CustomerGateways))) + return nil +} diff --git a/ec2/flow_logs.go b/ec2/flow_logs.go new file mode 100644 index 000000000..6a1a53803 --- /dev/null +++ b/ec2/flow_logs.go @@ -0,0 +1,122 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type FlowLog struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + CreationTime *time.Time + DeliverLogsErrorMessage *string + DeliverLogsPermissionArn *string + DeliverLogsStatus *string + FlowLogId *string `neo:"unique"` + FlowLogStatus *string + LogDestination *string + LogDestinationType *string + LogFormat *string + LogGroupName *string + MaxAggregationInterval *int64 + ResourceId *string + Tags []*FlowLogTag `gorm:"constraint:OnDelete:CASCADE;"` + TrafficType *string +} + +func (FlowLog) TableName() string { + return "aws_ec2_flow_logs" +} + +type FlowLogTag struct { + ID uint `gorm:"primarykey"` + FlowLogID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (FlowLogTag) TableName() string { + return "aws_ec2_flow_log_tags" +} + +func (c *Client) transformFlowLogTag(value *ec2.Tag) *FlowLogTag { + return &FlowLogTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformFlowLogTags(values []*ec2.Tag) []*FlowLogTag { + var tValues []*FlowLogTag + for _, v := range values { + tValues = append(tValues, c.transformFlowLogTag(v)) + } + return tValues +} + +func (c *Client) transformFlowLog(value *ec2.FlowLog) *FlowLog { + return &FlowLog{ + Region: c.region, + AccountID: c.accountID, + CreationTime: value.CreationTime, + DeliverLogsErrorMessage: value.DeliverLogsErrorMessage, + DeliverLogsPermissionArn: value.DeliverLogsPermissionArn, + DeliverLogsStatus: value.DeliverLogsStatus, + FlowLogId: value.FlowLogId, + FlowLogStatus: value.FlowLogStatus, + LogDestination: value.LogDestination, + LogDestinationType: value.LogDestinationType, + LogFormat: value.LogFormat, + LogGroupName: value.LogGroupName, + MaxAggregationInterval: value.MaxAggregationInterval, + ResourceId: value.ResourceId, + Tags: c.transformFlowLogTags(value.Tags), + TrafficType: value.TrafficType, + } +} + +func (c *Client) transformFlowLogs(values []*ec2.FlowLog) []*FlowLog { + var tValues []*FlowLog + for _, v := range values { + tValues = append(tValues, c.transformFlowLog(v)) + } + return tValues +} + +var FlowLogsTables = []interface{}{ + &FlowLog{}, + &FlowLogTag{}, +} + +func (c *Client) FlowLogs(gConfig interface{}) error { + var config ec2.DescribeFlowLogsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(FlowLogsTables...) + for { + output, err := c.svc.DescribeFlowLogs(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformFlowLogs(output.FlowLogs)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.flow_logs"), zap.Int("count", len(output.FlowLogs))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/images.go b/ec2/images.go new file mode 100644 index 000000000..504acf961 --- /dev/null +++ b/ec2/images.go @@ -0,0 +1,239 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type Image struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + Architecture *string + BlockDeviceMappings []*ImageBlockDeviceMapping `gorm:"constraint:OnDelete:CASCADE;"` + CreationDate *string + Description *string + EnaSupport *bool + Hypervisor *string + ImageId *string `neo:"unique"` + ImageLocation *string + ImageOwnerAlias *string + ImageType *string + KernelId *string + Name *string + OwnerId *string + Platform *string + PlatformDetails *string + ProductCodes []*ImageProductCode `gorm:"constraint:OnDelete:CASCADE;"` + Public *bool + RamdiskId *string + RootDeviceName *string + RootDeviceType *string + SriovNetSupport *string + State *string + + StateReasonCode *string + StateReasonMessage *string + + Tags []*ImageTag `gorm:"constraint:OnDelete:CASCADE;"` + UsageOperation *string + VirtualizationType *string +} + +func (Image) TableName() string { + return "aws_ec2_images" +} + +type ImageBlockDeviceMapping struct { + ID uint `gorm:"primarykey"` + ImageID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + DeviceName *string + EbsDeleteOnTermination *bool + EbsEncrypted *bool + EbsIops *int64 + EbsKmsKeyId *string + EbsSnapshotId *string + EbsVolumeSize *int64 + EbsVolumeType *string + + NoDevice *string + VirtualName *string +} + +func (ImageBlockDeviceMapping) TableName() string { + return "aws_ec2_image_block_device_mappings" +} + +type ImageProductCode struct { + ID uint `gorm:"primarykey"` + ImageID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ProductCodeId *string + ProductCodeType *string +} + +func (ImageProductCode) TableName() string { + return "aws_ec2_image_product_codes" +} + +type ImageTag struct { + ID uint `gorm:"primarykey"` + ImageID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (ImageTag) TableName() string { + return "aws_ec2_image_tags" +} + +func (c *Client) transformImageBlockDeviceMapping(value *ec2.BlockDeviceMapping) *ImageBlockDeviceMapping { + res := ImageBlockDeviceMapping{ + DeviceName: value.DeviceName, + AccountID: c.accountID, + Region: c.region, + NoDevice: value.NoDevice, + VirtualName: value.VirtualName, + } + + if value.Ebs != nil { + res.EbsDeleteOnTermination = value.Ebs.DeleteOnTermination + res.EbsEncrypted = value.Ebs.Encrypted + res.EbsIops = value.Ebs.Iops + res.EbsKmsKeyId = value.Ebs.KmsKeyId + res.EbsSnapshotId = value.Ebs.SnapshotId + res.EbsVolumeSize = value.Ebs.VolumeSize + res.EbsVolumeType = value.Ebs.VolumeType + } + + return &res +} + +func (c *Client) transformImageBlockDeviceMappings(values []*ec2.BlockDeviceMapping) []*ImageBlockDeviceMapping { + var tValues []*ImageBlockDeviceMapping + for _, v := range values { + tValues = append(tValues, c.transformImageBlockDeviceMapping(v)) + } + return tValues +} + +func (c *Client) transformImageProductCode(value *ec2.ProductCode) *ImageProductCode { + return &ImageProductCode{ + AccountID: c.accountID, + Region: c.region, + ProductCodeId: value.ProductCodeId, + ProductCodeType: value.ProductCodeType, + } +} + +func (c *Client) transformImageProductCodes(values []*ec2.ProductCode) []*ImageProductCode { + var tValues []*ImageProductCode + for _, v := range values { + tValues = append(tValues, c.transformImageProductCode(v)) + } + return tValues +} + +func (c *Client) transformImageTag(value *ec2.Tag) *ImageTag { + return &ImageTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformImageTags(values []*ec2.Tag) []*ImageTag { + var tValues []*ImageTag + for _, v := range values { + tValues = append(tValues, c.transformImageTag(v)) + } + return tValues +} + +func (c *Client) transformImage(value *ec2.Image) *Image { + res := Image{ + Region: c.region, + AccountID: c.accountID, + Architecture: value.Architecture, + BlockDeviceMappings: c.transformImageBlockDeviceMappings(value.BlockDeviceMappings), + CreationDate: value.CreationDate, + Description: value.Description, + EnaSupport: value.EnaSupport, + Hypervisor: value.Hypervisor, + ImageId: value.ImageId, + ImageLocation: value.ImageLocation, + ImageOwnerAlias: value.ImageOwnerAlias, + ImageType: value.ImageType, + KernelId: value.KernelId, + Name: value.Name, + OwnerId: value.OwnerId, + Platform: value.Platform, + PlatformDetails: value.PlatformDetails, + ProductCodes: c.transformImageProductCodes(value.ProductCodes), + Public: value.Public, + RamdiskId: value.RamdiskId, + RootDeviceName: value.RootDeviceName, + RootDeviceType: value.RootDeviceType, + SriovNetSupport: value.SriovNetSupport, + State: value.State, + Tags: c.transformImageTags(value.Tags), + UsageOperation: value.UsageOperation, + VirtualizationType: value.VirtualizationType, + } + + if value.StateReason != nil { + res.StateReasonCode = value.StateReason.Code + res.StateReasonMessage = value.StateReason.Message + } + return &res +} + +func (c *Client) transformImages(values []*ec2.Image) []*Image { + var tValues []*Image + for _, v := range values { + tValues = append(tValues, c.transformImage(v)) + } + return tValues +} + +var ImageTables = []interface{}{ + &Image{}, + &ImageBlockDeviceMapping{}, + &ImageProductCode{}, + &ImageTag{}, +} + +func (c *Client) images(gConfig interface{}) error { + var config ec2.DescribeImagesInput + err := mapstructure.Decode(gConfig, &config) + if config.Owners == nil { + self := "self" + config.Owners = append(config.Owners, &self) + } + if err != nil { + return err + } + + output, err := c.svc.DescribeImages(&config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(ImageTables...) + c.db.ChunkedCreate(c.transformImages(output.Images)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.images"), zap.Int("count", len(output.Images))) + return nil +} diff --git a/ec2/instances.go b/ec2/instances.go new file mode 100644 index 000000000..dd2036d50 --- /dev/null +++ b/ec2/instances.go @@ -0,0 +1,646 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Instance struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + AmiLaunchIndex *int64 + Architecture *string + BlockDeviceMappings []*InstanceBlockDeviceMapping `gorm:"constraint:OnDelete:CASCADE;"` + CapacityReservationId *string + + CapacityReservationPreference *string + CapacityReservationTargetId *string + CapacityReservationTargetGroupArn *string + + ClientToken *string + + CpuOptionsCoreCount *int64 + CpuOptionsThreadsPerCore *int64 + + EbsOptimized *bool + ElasticGpuAssociations []*InstanceElasticGpuAssociation `gorm:"constraint:OnDelete:CASCADE;"` + ElasticInferenceAcceleratorAssociations []*InstanceElasticInferenceAcceleratorAssociation `gorm:"constraint:OnDelete:CASCADE;"` + EnaSupport *bool + HibernationOptionsConfigured *bool + Hypervisor *string + + IamInstanceProfileArn *string + IamInstanceProfileId *string + + ImageId *string + InstanceId *string `neo:"unique"` + InstanceLifecycle *string + InstanceType *string + KernelId *string + KeyName *string + LaunchTime *time.Time + Licenses []*InstanceLicenseConfiguration `gorm:"constraint:OnDelete:CASCADE;"` + + MetadataOptionsHttpEndpoint *string + MetadataOptionsHttpPutResponseHopLimit *int64 + MetadataOptionsHttpTokens *string + MetadataOptionsState *string + + MonitoringState *string + + NetworkInterfaces []*InstanceNetworkInterface `gorm:"constraint:OnDelete:CASCADE;"` + OutpostArn *string + + PlacementAffinity *string + PlacementAvailabilityZone *string + PlacementGroupName *string + PlacementHostId *string + PlacementHostResourceGroupArn *string + PlacementPartitionNumber *int64 + PlacementSpreadDomain *string + PlacementTenancy *string + + Platform *string + PrivateDnsName *string + PrivateIpAddress *string + ProductCodes []*InstanceProductCode `gorm:"constraint:OnDelete:CASCADE;"` + PublicDnsName *string + PublicIpAddress *string + RamdiskId *string + RootDeviceName *string + RootDeviceType *string + SecurityGroups []*InstanceGroupIdentifier `gorm:"constraint:OnDelete:CASCADE;"` + SourceDestCheck *bool + SpotInstanceRequestId *string + SriovNetSupport *string + + StateCode *int64 + StateName *string + + StateReasonCode *string + StateReasonMessage *string + + StateTransitionReason *string + SubnetId *string + Tags []*InstanceTag `gorm:"constraint:OnDelete:CASCADE;"` + VirtualizationType *string + VpcId *string +} + +func (Instance) TableName() string { + return "aws_ec2_instances" +} + +type InstanceBlockDeviceMapping struct { + ID uint `gorm:"primarykey"` + InstanceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + DeviceName *string + + AttachTime *time.Time + DeleteOnTermination *bool + Status *string + VolumeId *string +} + +func (InstanceBlockDeviceMapping) TableName() string { + return "aws_ec2_instance_block_device_mappings" +} + +type InstanceElasticGpuAssociation struct { + ID uint `gorm:"primarykey"` + InstanceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ElasticGpuAssociationId *string + ElasticGpuAssociationState *string + ElasticGpuAssociationTime *string + ElasticGpuId *string +} + +func (InstanceElasticGpuAssociation) TableName() string { + return "aws_ec2_instance_elastic_gpu_associations" +} + +type InstanceElasticInferenceAcceleratorAssociation struct { + ID uint `gorm:"primarykey"` + InstanceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ElasticInferenceAcceleratorArn *string + ElasticInferenceAcceleratorAssociationId *string + ElasticInferenceAcceleratorAssociationState *string + ElasticInferenceAcceleratorAssociationTime *time.Time +} + +func (InstanceElasticInferenceAcceleratorAssociation) TableName() string { + return "aws_ec2_instance_elastic_inference_accelerator_associations" +} + +type InstanceLicenseConfiguration struct { + ID uint `gorm:"primarykey"` + InstanceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + LicenseConfigurationArn *string +} + +func (InstanceLicenseConfiguration) TableName() string { + return "aws_ec2_instance_license_configurations" +} + +type InstanceNetworkInterface struct { + ID uint `gorm:"primarykey"` + InstanceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AssociationCarrierIp *string + AssociationIpOwnerId *string + AssociationPublicDnsName *string + AssociationPublicIp *string + + AttachmentTime *time.Time + AttachmentId *string + AttachmentDeleteOnTermination *bool + AttachmentDeviceIndex *int64 + AttachmentStatus *string + + Description *string + InterfaceType *string + Ipv6Addresses []*InstanceIpv6Address `gorm:"constraint:OnDelete:CASCADE;"` + MacAddress *string + NetworkInterfaceId *string + OwnerId *string + PrivateDnsName *string + PrivateIpAddress *string + PrivateIpAddresses []*InstancePrivateIpAddress `gorm:"constraint:OnDelete:CASCADE;"` + SourceDestCheck *bool + Status *string + SubnetId *string + VpcId *string +} + +func (InstanceNetworkInterface) TableName() string { + return "aws_ec2_instance_network_interfaces" +} + +type InstanceGroupIdentifier struct { + ID uint `gorm:"primarykey"` + InstanceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + GroupId *string + GroupName *string +} + +func (InstanceGroupIdentifier) TableName() string { + return "aws_ec2_instance_group_identifiers" +} + +type InstanceIpv6Address struct { + ID uint `gorm:"primarykey"` + InstanceNetworkInterfaceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Ipv6Address *string +} + +func (InstanceIpv6Address) TableName() string { + return "aws_ec2_instance_ipv6_addresses" +} + +type InstancePrivateIpAddress struct { + ID uint `gorm:"primarykey"` + InstanceNetworkInterfaceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AssociationCarrierIp *string + AssociationIpOwnerId *string + AssociationPublicDnsName *string + AssociationPublicIp *string + + Primary *bool + PrivateDnsName *string + PrivateIpAddress *string +} + +func (InstancePrivateIpAddress) TableName() string { + return "aws_ec2_instance_private_ip_addresses" +} + +type InstanceProductCode struct { + ID uint `gorm:"primarykey"` + InstanceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ProductCodeId *string + ProductCodeType *string +} + +func (InstanceProductCode) TableName() string { + return "aws_ec2_instance_product_codes" +} + +type InstanceTag struct { + ID uint `gorm:"primarykey"` + InstanceID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (InstanceTag) TableName() string { + return "aws_ec2_instance_tags" +} + +func (c *Client) transformInstanceBlockDeviceMapping(value *ec2.InstanceBlockDeviceMapping) *InstanceBlockDeviceMapping { + res := InstanceBlockDeviceMapping{ + DeviceName: value.DeviceName, + AccountID: c.accountID, + Region: c.region, + } + + if value.Ebs != nil { + res.AttachTime = value.Ebs.AttachTime + res.DeleteOnTermination = value.Ebs.DeleteOnTermination + res.Status = value.Ebs.Status + res.VolumeId = value.Ebs.VolumeId + } + + return &res +} + +func (c *Client) transformInstanceBlockDeviceMappings(values []*ec2.InstanceBlockDeviceMapping) []*InstanceBlockDeviceMapping { + var tValues []*InstanceBlockDeviceMapping + for _, v := range values { + tValues = append(tValues, c.transformInstanceBlockDeviceMapping(v)) + } + return tValues +} + +func (c *Client) transformInstanceElasticGpuAssociation(value *ec2.ElasticGpuAssociation) *InstanceElasticGpuAssociation { + return &InstanceElasticGpuAssociation{ + AccountID: c.accountID, + Region: c.region, + ElasticGpuAssociationId: value.ElasticGpuAssociationId, + ElasticGpuAssociationState: value.ElasticGpuAssociationState, + ElasticGpuAssociationTime: value.ElasticGpuAssociationTime, + ElasticGpuId: value.ElasticGpuId, + } +} + +func (c *Client) transformInstanceElasticGpuAssociations(values []*ec2.ElasticGpuAssociation) []*InstanceElasticGpuAssociation { + var tValues []*InstanceElasticGpuAssociation + for _, v := range values { + tValues = append(tValues, c.transformInstanceElasticGpuAssociation(v)) + } + return tValues +} + +func (c *Client) transformInstanceElasticInferenceAcceleratorAssociation(value *ec2.ElasticInferenceAcceleratorAssociation) *InstanceElasticInferenceAcceleratorAssociation { + return &InstanceElasticInferenceAcceleratorAssociation{ + AccountID: c.accountID, + Region: c.region, + ElasticInferenceAcceleratorArn: value.ElasticInferenceAcceleratorArn, + ElasticInferenceAcceleratorAssociationId: value.ElasticInferenceAcceleratorAssociationId, + ElasticInferenceAcceleratorAssociationState: value.ElasticInferenceAcceleratorAssociationState, + ElasticInferenceAcceleratorAssociationTime: value.ElasticInferenceAcceleratorAssociationTime, + } +} + +func (c *Client) transformInstanceElasticInferenceAcceleratorAssociations(values []*ec2.ElasticInferenceAcceleratorAssociation) []*InstanceElasticInferenceAcceleratorAssociation { + var tValues []*InstanceElasticInferenceAcceleratorAssociation + for _, v := range values { + tValues = append(tValues, c.transformInstanceElasticInferenceAcceleratorAssociation(v)) + } + return tValues +} + +func (c *Client) transformInstanceLicenseConfiguration(value *ec2.LicenseConfiguration) *InstanceLicenseConfiguration { + return &InstanceLicenseConfiguration{ + AccountID: c.accountID, + Region: c.region, + LicenseConfigurationArn: value.LicenseConfigurationArn, + } +} + +func (c *Client) transformInstanceLicenseConfigurations(values []*ec2.LicenseConfiguration) []*InstanceLicenseConfiguration { + var tValues []*InstanceLicenseConfiguration + for _, v := range values { + tValues = append(tValues, c.transformInstanceLicenseConfiguration(v)) + } + return tValues +} + +func (c *Client) transformInstanceGroupIdentifier(value *ec2.GroupIdentifier) *InstanceGroupIdentifier { + return &InstanceGroupIdentifier{ + AccountID: c.accountID, + Region: c.region, + GroupId: value.GroupId, + GroupName: value.GroupName, + } +} + +func (c *Client) transformInstanceGroupIdentifiers(values []*ec2.GroupIdentifier) []*InstanceGroupIdentifier { + var tValues []*InstanceGroupIdentifier + for _, v := range values { + tValues = append(tValues, c.transformInstanceGroupIdentifier(v)) + } + return tValues +} + +func (c *Client) transformInstanceIpv6Address(value *ec2.InstanceIpv6Address) *InstanceIpv6Address { + return &InstanceIpv6Address{ + AccountID: c.accountID, + Region: c.region, + Ipv6Address: value.Ipv6Address, + } +} + +func (c *Client) transformInstanceIpv6Addresss(values []*ec2.InstanceIpv6Address) []*InstanceIpv6Address { + var tValues []*InstanceIpv6Address + for _, v := range values { + tValues = append(tValues, c.transformInstanceIpv6Address(v)) + } + return tValues +} + +func (c *Client) transformInstancePrivateIpAddress(value *ec2.InstancePrivateIpAddress) *InstancePrivateIpAddress { + res := InstancePrivateIpAddress{ + AccountID: c.accountID, + Region: c.region, + Primary: value.Primary, + PrivateDnsName: value.PrivateDnsName, + PrivateIpAddress: value.PrivateIpAddress, + } + + if value.Association != nil { + res.AssociationCarrierIp = value.Association.CarrierIp + res.AssociationIpOwnerId = value.Association.IpOwnerId + res.AssociationPublicDnsName = value.Association.PublicDnsName + res.AssociationPublicIp = value.Association.PublicIp + } + + return &res +} + +func (c *Client) transformInstancePrivateIpAddresss(values []*ec2.InstancePrivateIpAddress) []*InstancePrivateIpAddress { + var tValues []*InstancePrivateIpAddress + for _, v := range values { + tValues = append(tValues, c.transformInstancePrivateIpAddress(v)) + } + return tValues +} + +func (c *Client) transformInstanceNetworkInterface(value *ec2.InstanceNetworkInterface) *InstanceNetworkInterface { + res := InstanceNetworkInterface{ + AccountID: c.accountID, + Region: c.region, + Description: value.Description, + InterfaceType: value.InterfaceType, + Ipv6Addresses: c.transformInstanceIpv6Addresss(value.Ipv6Addresses), + MacAddress: value.MacAddress, + NetworkInterfaceId: value.NetworkInterfaceId, + OwnerId: value.OwnerId, + PrivateDnsName: value.PrivateDnsName, + PrivateIpAddress: value.PrivateIpAddress, + PrivateIpAddresses: c.transformInstancePrivateIpAddresss(value.PrivateIpAddresses), + SourceDestCheck: value.SourceDestCheck, + Status: value.Status, + SubnetId: value.SubnetId, + VpcId: value.VpcId, + } + + if value.Attachment != nil { + res.AttachmentTime = value.Attachment.AttachTime + res.AttachmentId = value.Attachment.AttachmentId + res.AttachmentDeleteOnTermination = value.Attachment.DeleteOnTermination + res.AttachmentDeviceIndex = value.Attachment.DeviceIndex + res.AttachmentStatus = value.Attachment.Status + } + + if value.Association != nil { + res.AssociationCarrierIp = value.Association.CarrierIp + res.AssociationIpOwnerId = value.Association.IpOwnerId + res.AssociationPublicDnsName = value.Association.PublicDnsName + res.AssociationPublicIp = value.Association.PublicIp + } + + return &res +} + +func (c *Client) transformInstanceNetworkInterfaces(values []*ec2.InstanceNetworkInterface) []*InstanceNetworkInterface { + var tValues []*InstanceNetworkInterface + for _, v := range values { + tValues = append(tValues, c.transformInstanceNetworkInterface(v)) + } + return tValues +} + +func (c *Client) transformInstanceProductCode(value *ec2.ProductCode) *InstanceProductCode { + return &InstanceProductCode{ + AccountID: c.accountID, + Region: c.region, + ProductCodeId: value.ProductCodeId, + ProductCodeType: value.ProductCodeType, + } +} + +func (c *Client) transformInstanceProductCodes(values []*ec2.ProductCode) []*InstanceProductCode { + var tValues []*InstanceProductCode + for _, v := range values { + tValues = append(tValues, c.transformInstanceProductCode(v)) + } + return tValues +} + +func (c *Client) transformInstanceTag(value *ec2.Tag) *InstanceTag { + return &InstanceTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformInstanceTags(values []*ec2.Tag) []*InstanceTag { + var tValues []*InstanceTag + for _, v := range values { + tValues = append(tValues, c.transformInstanceTag(v)) + } + return tValues +} + +func (c *Client) transformInstance(value *ec2.Instance) *Instance { + res := Instance{ + Region: c.region, + AccountID: c.accountID, + AmiLaunchIndex: value.AmiLaunchIndex, + Architecture: value.Architecture, + BlockDeviceMappings: c.transformInstanceBlockDeviceMappings(value.BlockDeviceMappings), + CapacityReservationId: value.CapacityReservationId, + ClientToken: value.ClientToken, + EbsOptimized: value.EbsOptimized, + ElasticGpuAssociations: c.transformInstanceElasticGpuAssociations(value.ElasticGpuAssociations), + ElasticInferenceAcceleratorAssociations: c.transformInstanceElasticInferenceAcceleratorAssociations(value.ElasticInferenceAcceleratorAssociations), + EnaSupport: value.EnaSupport, + Hypervisor: value.Hypervisor, + ImageId: value.ImageId, + InstanceId: value.InstanceId, + InstanceLifecycle: value.InstanceLifecycle, + InstanceType: value.InstanceType, + KernelId: value.KernelId, + KeyName: value.KeyName, + LaunchTime: value.LaunchTime, + Licenses: c.transformInstanceLicenseConfigurations(value.Licenses), + NetworkInterfaces: c.transformInstanceNetworkInterfaces(value.NetworkInterfaces), + OutpostArn: value.OutpostArn, + Platform: value.Platform, + PrivateDnsName: value.PrivateDnsName, + PrivateIpAddress: value.PrivateIpAddress, + ProductCodes: c.transformInstanceProductCodes(value.ProductCodes), + PublicDnsName: value.PublicDnsName, + PublicIpAddress: value.PublicIpAddress, + RamdiskId: value.RamdiskId, + RootDeviceName: value.RootDeviceName, + RootDeviceType: value.RootDeviceType, + SecurityGroups: c.transformInstanceGroupIdentifiers(value.SecurityGroups), + SourceDestCheck: value.SourceDestCheck, + SpotInstanceRequestId: value.SpotInstanceRequestId, + SriovNetSupport: value.SriovNetSupport, + StateTransitionReason: value.StateTransitionReason, + SubnetId: value.SubnetId, + Tags: c.transformInstanceTags(value.Tags), + VirtualizationType: value.VirtualizationType, + VpcId: value.VpcId, + } + + if value.CpuOptions != nil { + res.CpuOptionsCoreCount = value.CpuOptions.CoreCount + res.CpuOptionsThreadsPerCore = value.CpuOptions.ThreadsPerCore + } + + if value.CapacityReservationSpecification != nil { + res.CapacityReservationPreference = value.CapacityReservationSpecification.CapacityReservationPreference + if value.CapacityReservationSpecification.CapacityReservationTarget != nil { + res.CapacityReservationTargetId = value.CapacityReservationSpecification.CapacityReservationTarget.CapacityReservationId + res.CapacityReservationTargetGroupArn = value.CapacityReservationSpecification.CapacityReservationTarget.CapacityReservationResourceGroupArn + } + } + + if value.HibernationOptions != nil { + res.HibernationOptionsConfigured = value.HibernationOptions.Configured + } + + if value.IamInstanceProfile != nil { + res.IamInstanceProfileArn = value.IamInstanceProfile.Arn + res.IamInstanceProfileId = value.IamInstanceProfile.Id + } + + if value.MetadataOptions != nil { + res.MetadataOptionsHttpEndpoint = value.MetadataOptions.HttpEndpoint + res.MetadataOptionsHttpTokens = value.MetadataOptions.HttpTokens + res.MetadataOptionsState = value.MetadataOptions.State + res.MetadataOptionsHttpPutResponseHopLimit = value.MetadataOptions.HttpPutResponseHopLimit + } + + if value.Monitoring != nil { + res.MonitoringState = value.Monitoring.State + } + + if value.Placement != nil { + res.PlacementAffinity = value.Placement.Affinity + res.PlacementAvailabilityZone = value.Placement.AvailabilityZone + res.PlacementGroupName = value.Placement.GroupName + res.PlacementHostId = value.Placement.HostId + res.PlacementHostResourceGroupArn = value.Placement.HostResourceGroupArn + res.PlacementPartitionNumber = value.Placement.PartitionNumber + res.PlacementSpreadDomain = value.Placement.SpreadDomain + res.PlacementTenancy = value.Placement.Tenancy + } + + if value.State != nil { + res.StateCode = value.State.Code + res.StateName = value.State.Name + } + + if value.StateReason != nil { + res.StateReasonCode = value.StateReason.Code + res.StateReasonMessage = value.StateReason.Message + } + + return &res +} + +func (c *Client) transformInstances(values []*ec2.Instance) []*Instance { + var tValues []*Instance + for _, v := range values { + tValues = append(tValues, c.transformInstance(v)) + } + return tValues +} + +var InstanceTables = []interface{}{ + &Instance{}, + &InstanceBlockDeviceMapping{}, + &InstanceElasticGpuAssociation{}, + &InstanceElasticInferenceAcceleratorAssociation{}, + &InstanceLicenseConfiguration{}, + &InstanceNetworkInterface{}, + &InstanceGroupIdentifier{}, + &InstanceIpv6Address{}, + &InstancePrivateIpAddress{}, + &InstanceProductCode{}, + &InstanceTag{}, +} + +func (c *Client) instances(gConfig interface{}) error { + var config ec2.DescribeInstancesInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(InstanceTables...) + for { + output, err := c.svc.DescribeInstances(&config) + if err != nil { + return err + } + for _, reservation := range output.Reservations { + c.log.Info("Fetched resources", zap.String("resource", "ec2.instances"), zap.Int("count", len(reservation.Instances))) + c.db.ChunkedCreate(c.transformInstances(reservation.Instances)) + } + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/internet_gateways.go b/ec2/internet_gateways.go new file mode 100644 index 000000000..333e24098 --- /dev/null +++ b/ec2/internet_gateways.go @@ -0,0 +1,134 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type InternetGateway struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + Attachments []*InternetGatewayAttachment `gorm:"constraint:OnDelete:CASCADE;"` + InternetGatewayId *string `neo:"unique"` + OwnerId *string + Tags []*InternetGatewayTag `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (InternetGateway) TableName() string { + return "aws_ec2_internet_gateways" +} + +type InternetGatewayAttachment struct { + ID uint `gorm:"primarykey"` + InternetGatewayID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + State *string + VpcId *string +} + +func (InternetGatewayAttachment) TableName() string { + return "aws_ec2_internet_gateway_attachments" +} + +type InternetGatewayTag struct { + ID uint `gorm:"primarykey"` + InternetGatewayID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (InternetGatewayTag) TableName() string { + return "aws_ec2_internet_gateway_tags" +} + +func (c *Client) transformInternetGatewayAttachment(value *ec2.InternetGatewayAttachment) *InternetGatewayAttachment { + return &InternetGatewayAttachment{ + AccountID: c.accountID, + Region: c.region, + State: value.State, + VpcId: value.VpcId, + } +} + +func (c *Client) transformInternetGatewayAttachments(values []*ec2.InternetGatewayAttachment) []*InternetGatewayAttachment { + var tValues []*InternetGatewayAttachment + for _, v := range values { + tValues = append(tValues, c.transformInternetGatewayAttachment(v)) + } + return tValues +} + +func (c *Client) transformInternetGatewayTag(value *ec2.Tag) *InternetGatewayTag { + return &InternetGatewayTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformInternetGatewayTags(values []*ec2.Tag) []*InternetGatewayTag { + var tValues []*InternetGatewayTag + for _, v := range values { + tValues = append(tValues, c.transformInternetGatewayTag(v)) + } + return tValues +} + +func (c *Client) transformInternetGateway(value *ec2.InternetGateway) *InternetGateway { + return &InternetGateway{ + Region: c.region, + AccountID: c.accountID, + Attachments: c.transformInternetGatewayAttachments(value.Attachments), + InternetGatewayId: value.InternetGatewayId, + OwnerId: value.OwnerId, + Tags: c.transformInternetGatewayTags(value.Tags), + } +} + +func (c *Client) transformInternetGateways(values []*ec2.InternetGateway) []*InternetGateway { + var tValues []*InternetGateway + for _, v := range values { + tValues = append(tValues, c.transformInternetGateway(v)) + } + return tValues +} + +var InternetGatewayTables = []interface{}{ + &InternetGateway{}, + &InternetGatewayAttachment{}, + &InternetGatewayTag{}, +} + +func (c *Client) internetGateways(gConfig interface{}) error { + var config ec2.DescribeInternetGatewaysInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(InternetGatewayTables...) + for { + output, err := c.svc.DescribeInternetGateways(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformInternetGateways(output.InternetGateways)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.internet_gateways"), zap.Int("count", len(output.InternetGateways))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/nat_gateways.go b/ec2/nat_gateways.go new file mode 100644 index 000000000..4667c0e4f --- /dev/null +++ b/ec2/nat_gateways.go @@ -0,0 +1,168 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type NatGateway struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + CreateTime *time.Time + DeleteTime *time.Time + FailureCode *string + FailureMessage *string + NatGatewayAddresses []*NatGatewayAddress `gorm:"constraint:OnDelete:CASCADE;"` + NatGatewayId *string `neo:"unique"` + + ProvisionTime *time.Time + Provisioned *string + ProvisionedRequestTime *time.Time + ProvisionedRequested *string + ProvisionedStatus *string + + State *string + SubnetId *string + Tags []*NatGatewayTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string +} + +func (NatGateway) TableName() string { + return "aws_ec2_nat_gateways" +} + +type NatGatewayAddress struct { + ID uint `gorm:"primarykey"` + NatGatewayID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AllocationId *string + NetworkInterfaceId *string + PrivateIp *string + PublicIp *string +} + +func (NatGatewayAddress) TableName() string { + return "aws_ec2_nat_gateway_addresses" +} + +type NatGatewayTag struct { + ID uint `gorm:"primarykey"` + NatGatewayID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (NatGatewayTag) TableName() string { + return "aws_ec2_nat_gateway_tags" +} + +func (c *Client) transformNatGatewayAddress(value *ec2.NatGatewayAddress) *NatGatewayAddress { + return &NatGatewayAddress{ + AccountID: c.accountID, + Region: c.region, + AllocationId: value.AllocationId, + NetworkInterfaceId: value.NetworkInterfaceId, + PrivateIp: value.PrivateIp, + PublicIp: value.PublicIp, + } +} + +func (c *Client) transformNatGatewayAddresss(values []*ec2.NatGatewayAddress) []*NatGatewayAddress { + var tValues []*NatGatewayAddress + for _, v := range values { + tValues = append(tValues, c.transformNatGatewayAddress(v)) + } + return tValues +} + +func (c *Client) transformNatGatewayTag(value *ec2.Tag) *NatGatewayTag { + return &NatGatewayTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformNatGatewayTags(values []*ec2.Tag) []*NatGatewayTag { + var tValues []*NatGatewayTag + for _, v := range values { + tValues = append(tValues, c.transformNatGatewayTag(v)) + } + return tValues +} + +func (c *Client) transformNatGateway(value *ec2.NatGateway) *NatGateway { + res := NatGateway{ + Region: c.region, + AccountID: c.accountID, + CreateTime: value.CreateTime, + DeleteTime: value.DeleteTime, + FailureCode: value.FailureCode, + FailureMessage: value.FailureMessage, + NatGatewayAddresses: c.transformNatGatewayAddresss(value.NatGatewayAddresses), + NatGatewayId: value.NatGatewayId, + State: value.State, + SubnetId: value.SubnetId, + Tags: c.transformNatGatewayTags(value.Tags), + VpcId: value.VpcId, + } + + if value.ProvisionedBandwidth != nil { + res.ProvisionTime = value.ProvisionedBandwidth.ProvisionTime + res.Provisioned = value.ProvisionedBandwidth.Provisioned + res.ProvisionedRequestTime = value.ProvisionedBandwidth.RequestTime + res.ProvisionedRequested = value.ProvisionedBandwidth.Requested + res.ProvisionedStatus = value.ProvisionedBandwidth.Status + } + + return &res +} + +func (c *Client) transformNatGateways(values []*ec2.NatGateway) []*NatGateway { + var tValues []*NatGateway + for _, v := range values { + tValues = append(tValues, c.transformNatGateway(v)) + } + return tValues +} + +var NatGatewayTables = []interface{}{ + &NatGateway{}, + &NatGatewayAddress{}, + &NatGatewayTag{}, +} + +func (c *Client) natGateways(gConfig interface{}) error { + var config ec2.DescribeNatGatewaysInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(NatGatewayTables...) + for { + output, err := c.svc.DescribeNatGateways(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformNatGateways(output.NatGateways)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.nat_gateways"), zap.Int("count", len(output.NatGateways))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/network_acls.go b/ec2/network_acls.go new file mode 100644 index 000000000..1e7e6be09 --- /dev/null +++ b/ec2/network_acls.go @@ -0,0 +1,205 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type NetworkAcl struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + Associations []*NetworkAclAssociation `gorm:"constraint:OnDelete:CASCADE;"` + Entries []*NetworkAclEntry `gorm:"constraint:OnDelete:CASCADE;"` + IsDefault *bool + NetworkAclId *string `neo:"unique"` + OwnerId *string + Tags []*NetworkAclTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string +} + +func (NetworkAcl) TableName() string { + return "aws_ec2_network_acls" +} + +type NetworkAclAssociation struct { + ID uint `gorm:"primarykey"` + NetworkAclID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + NetworkAclAssociationId *string + NetworkAclId *string + SubnetId *string +} + +func (NetworkAclAssociation) TableName() string { + return "aws_ec2_network_acl_associations" +} + +type NetworkAclEntry struct { + ID uint `gorm:"primarykey"` + NetworkAclID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + CidrBlock *string + Egress *bool + + IcmpTypeCode *int64 + IcmpTypeType *int64 + + Ipv6CidrBlock *string + + PortRangeFrom *int64 + PortRangeTo *int64 + + Protocol *string + RuleAction *string + RuleNumber *int64 +} + +func (NetworkAclEntry) TableName() string { + return "aws_ec2_network_acl_entries" +} + +type NetworkAclTag struct { + ID uint `gorm:"primarykey"` + NetworkAclID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (NetworkAclTag) TableName() string { + return "aws_ec2_network_acl_tags" +} + +func (c *Client) transformNetworkAclAssociation(value *ec2.NetworkAclAssociation) *NetworkAclAssociation { + return &NetworkAclAssociation{ + AccountID: c.accountID, + Region: c.region, + NetworkAclAssociationId: value.NetworkAclAssociationId, + NetworkAclId: value.NetworkAclId, + SubnetId: value.SubnetId, + } +} + +func (c *Client) transformNetworkAclAssociations(values []*ec2.NetworkAclAssociation) []*NetworkAclAssociation { + var tValues []*NetworkAclAssociation + for _, v := range values { + tValues = append(tValues, c.transformNetworkAclAssociation(v)) + } + return tValues +} + +func (c *Client) transformNetworkAclEntry(value *ec2.NetworkAclEntry) *NetworkAclEntry { + res := NetworkAclEntry{ + AccountID: c.accountID, + Region: c.region, + + CidrBlock: value.CidrBlock, + Egress: value.Egress, + + Ipv6CidrBlock: value.Ipv6CidrBlock, + + Protocol: value.Protocol, + RuleAction: value.RuleAction, + RuleNumber: value.RuleNumber, + } + + if value.IcmpTypeCode != nil { + res.IcmpTypeCode = value.IcmpTypeCode.Code + res.IcmpTypeType = value.IcmpTypeCode.Type + } + + if value.PortRange != nil { + res.PortRangeFrom = value.PortRange.From + res.PortRangeTo = value.PortRange.To + } + + return &res +} + +func (c *Client) transformNetworkAclEntrys(values []*ec2.NetworkAclEntry) []*NetworkAclEntry { + var tValues []*NetworkAclEntry + for _, v := range values { + tValues = append(tValues, c.transformNetworkAclEntry(v)) + } + return tValues +} + +func (c *Client) transformNetworkAclTag(value *ec2.Tag) *NetworkAclTag { + return &NetworkAclTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformNetworkAclTags(values []*ec2.Tag) []*NetworkAclTag { + var tValues []*NetworkAclTag + for _, v := range values { + tValues = append(tValues, c.transformNetworkAclTag(v)) + } + return tValues +} + +func (c *Client) transformNetworkAcl(value *ec2.NetworkAcl) *NetworkAcl { + return &NetworkAcl{ + Region: c.region, + AccountID: c.accountID, + Associations: c.transformNetworkAclAssociations(value.Associations), + Entries: c.transformNetworkAclEntrys(value.Entries), + IsDefault: value.IsDefault, + NetworkAclId: value.NetworkAclId, + OwnerId: value.OwnerId, + Tags: c.transformNetworkAclTags(value.Tags), + VpcId: value.VpcId, + } +} + +func (c *Client) transformNetworkAcls(values []*ec2.NetworkAcl) []*NetworkAcl { + var tValues []*NetworkAcl + for _, v := range values { + tValues = append(tValues, c.transformNetworkAcl(v)) + } + return tValues +} + +var NetworkAclTables = []interface{}{ + &NetworkAcl{}, + &NetworkAclAssociation{}, + &NetworkAclEntry{}, + &NetworkAclTag{}, +} + +func (c *Client) networkAcls(gConfig interface{}) error { + var config ec2.DescribeNetworkAclsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(NetworkAclTables...) + for { + output, err := c.svc.DescribeNetworkAcls(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformNetworkAcls(output.NetworkAcls)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.network_acls"), zap.Int("count", len(output.NetworkAcls))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/route_tabls.go b/ec2/route_tabls.go new file mode 100644 index 000000000..501b9623b --- /dev/null +++ b/ec2/route_tabls.go @@ -0,0 +1,242 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type RouteTable struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + Associations []*RouteTableAssociation `gorm:"constraint:OnDelete:CASCADE;"` + OwnerId *string + PropagatingVgws []*RouteTablePropagatingVgw `gorm:"constraint:OnDelete:CASCADE;"` + RouteTableId *string `neo:"unique"` + Routes []*RouteTableRoute `gorm:"constraint:OnDelete:CASCADE;"` + Tags []*RouteTableTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string +} + +func (RouteTable) TableName() string { + return "aws_ec2_route_tables" +} + +type RouteTableAssociation struct { + ID uint `gorm:"primarykey"` + RouteTableID uint `neo:"ignore"` + AccountID string `gorm:"-" neo:"unique"` + Region string `gorm:"-" neo:"unique"` + + State *string + StatusMessage *string + + GatewayId *string + Main *bool + RouteTableAssociationId *string `neo:"unique"` + RouteTableId *string + SubnetId *string +} + +func (RouteTableAssociation) TableName() string { + return "aws_ec2_route_table_associations" +} + +type RouteTablePropagatingVgw struct { + ID uint `gorm:"primarykey"` + RouteTableID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + GatewayId *string +} + +func (RouteTablePropagatingVgw) TableName() string { + return "aws_ec2_route_table_propagation_vgws" +} + +type RouteTableRoute struct { + ID uint `gorm:"primarykey"` + RouteTableID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + CarrierGatewayId *string + DestinationCidrBlock *string + DestinationIpv6CidrBlock *string + DestinationPrefixListId *string + EgressOnlyInternetGatewayId *string + GatewayId *string + InstanceId *string + InstanceOwnerId *string + LocalGatewayId *string + NatGatewayId *string + NetworkInterfaceId *string + Origin *string + State *string + TransitGatewayId *string + VpcPeeringConnectionId *string +} + +func (RouteTableRoute) TableName() string { + return "aws_ec2_route_table_routes" +} + +type RouteTableTag struct { + ID uint `gorm:"primarykey"` + RouteTableID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (RouteTableTag) TableName() string { + return "aws_ec2_route_table_tags" +} + +func (c *Client) transformRouteTableAssociation(value *ec2.RouteTableAssociation) *RouteTableAssociation { + res := RouteTableAssociation{ + AccountID: c.accountID, + Region: c.region, + GatewayId: value.GatewayId, + Main: value.Main, + RouteTableAssociationId: value.RouteTableAssociationId, + RouteTableId: value.RouteTableId, + SubnetId: value.SubnetId, + } + + if value.AssociationState != nil { + res.State = value.AssociationState.State + res.StatusMessage = value.AssociationState.StatusMessage + } + + return &res +} + +func (c *Client) transformRouteTableAssociations(values []*ec2.RouteTableAssociation) []*RouteTableAssociation { + var tValues []*RouteTableAssociation + for _, v := range values { + tValues = append(tValues, c.transformRouteTableAssociation(v)) + } + return tValues +} + +func (c *Client) transformRouteTablePropagatingVgw(value *ec2.PropagatingVgw) *RouteTablePropagatingVgw { + return &RouteTablePropagatingVgw{ + AccountID: c.accountID, + Region: c.region, + GatewayId: value.GatewayId, + } +} + +func (c *Client) transformRouteTablePropagatingVgws(values []*ec2.PropagatingVgw) []*RouteTablePropagatingVgw { + var tValues []*RouteTablePropagatingVgw + for _, v := range values { + tValues = append(tValues, c.transformRouteTablePropagatingVgw(v)) + } + return tValues +} + +func (c *Client) transformRouteTableRoute(value *ec2.Route) *RouteTableRoute { + return &RouteTableRoute{ + AccountID: c.accountID, + Region: c.region, + CarrierGatewayId: value.CarrierGatewayId, + DestinationCidrBlock: value.DestinationCidrBlock, + DestinationIpv6CidrBlock: value.DestinationIpv6CidrBlock, + DestinationPrefixListId: value.DestinationPrefixListId, + EgressOnlyInternetGatewayId: value.EgressOnlyInternetGatewayId, + GatewayId: value.GatewayId, + InstanceId: value.InstanceId, + InstanceOwnerId: value.InstanceOwnerId, + LocalGatewayId: value.LocalGatewayId, + NatGatewayId: value.NatGatewayId, + NetworkInterfaceId: value.NetworkInterfaceId, + Origin: value.Origin, + State: value.State, + TransitGatewayId: value.TransitGatewayId, + VpcPeeringConnectionId: value.VpcPeeringConnectionId, + } +} + +func (c *Client) transformRouteTableRoutes(values []*ec2.Route) []*RouteTableRoute { + var tValues []*RouteTableRoute + for _, v := range values { + tValues = append(tValues, c.transformRouteTableRoute(v)) + } + return tValues +} + +func (c *Client) transformRouteTableTag(value *ec2.Tag) *RouteTableTag { + return &RouteTableTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformRouteTableTags(values []*ec2.Tag) []*RouteTableTag { + var tValues []*RouteTableTag + for _, v := range values { + tValues = append(tValues, c.transformRouteTableTag(v)) + } + return tValues +} + +func (c *Client) transformRouteTable(value *ec2.RouteTable) *RouteTable { + return &RouteTable{ + Region: c.region, + AccountID: c.accountID, + Associations: c.transformRouteTableAssociations(value.Associations), + OwnerId: value.OwnerId, + PropagatingVgws: c.transformRouteTablePropagatingVgws(value.PropagatingVgws), + RouteTableId: value.RouteTableId, + Routes: c.transformRouteTableRoutes(value.Routes), + Tags: c.transformRouteTableTags(value.Tags), + VpcId: value.VpcId, + } +} + +func (c *Client) transformRouteTables(values []*ec2.RouteTable) []*RouteTable { + var tValues []*RouteTable + for _, v := range values { + tValues = append(tValues, c.transformRouteTable(v)) + } + return tValues +} + +var RouteTableTables = []interface{}{ + &RouteTable{}, + &RouteTableAssociation{}, + &RouteTablePropagatingVgw{}, + &RouteTableRoute{}, + &RouteTableTag{}, +} + +func (c *Client) routeTables(gConfig interface{}) error { + var config ec2.DescribeRouteTablesInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(RouteTableTables...) + for { + output, err := c.svc.DescribeRouteTables(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformRouteTables(output.RouteTables)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.route_tables"), zap.Int("count", len(output.RouteTables))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/security_groups.go b/ec2/security_groups.go new file mode 100644 index 000000000..329077319 --- /dev/null +++ b/ec2/security_groups.go @@ -0,0 +1,279 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type SecurityGroup struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + Description *string + GroupId *string `neo:"unique"` + GroupName *string + IpPermissions []*SecurityGroupIpPermission `gorm:"constraint:OnDelete:CASCADE;"` + OwnerId *string + Tags []*SecurityGroupTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string +} + +func (SecurityGroup) TableName() string { + return "aws_ec2_security_groups" +} + +type SecurityGroupIpPermission struct { + ID uint `gorm:"primarykey"` + SecurityGroupID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + FromPort *int64 + IpProtocol *string + IpRanges []*SecurityGroupIpRange `gorm:"constraint:OnDelete:CASCADE;"` + Ipv6Ranges []*SecurityGroupIpv6Range `gorm:"constraint:OnDelete:CASCADE;"` + PrefixListIds []*SecurityGroupPrefixListId `gorm:"constraint:OnDelete:CASCADE;"` + ToPort *int64 + UserIdGroupPairs []*SecurityGroupUserIdGroupPair `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (SecurityGroupIpPermission) TableName() string { + return "aws_ec2_security_group_ip_permissions" +} + +type SecurityGroupIpRange struct { + ID uint `gorm:"primarykey"` + SecurityGroupIpPermissionID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + CidrIp *string + Description *string +} + +func (SecurityGroupIpRange) TableName() string { + return "aws_ec2_security_group_ip_ranges" +} + +type SecurityGroupIpv6Range struct { + ID uint `gorm:"primarykey"` + SecurityGroupIpPermissionID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + CidrIpv6 *string + Description *string +} + +func (SecurityGroupIpv6Range) TableName() string { + return "aws_ec2_security_group_ipv6_ranges" +} + +type SecurityGroupPrefixListId struct { + ID uint `gorm:"primarykey"` + SecurityGroupIpPermissionID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + Description *string + PrefixListId *string +} + +func (SecurityGroupPrefixListId) TableName() string { + return "aws_ec2_security_group_prefix_list_ids" +} + +type SecurityGroupUserIdGroupPair struct { + ID uint `gorm:"primarykey"` + SecurityGroupIpPermissionID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + Description *string + GroupId *string + GroupName *string + PeeringStatus *string + UserId *string + VpcId *string + VpcPeeringConnectionId *string +} + +func (SecurityGroupUserIdGroupPair) TableName() string { + return "aws_ec2_security_group_user_id_group_paris" +} + +type SecurityGroupTag struct { + ID uint `gorm:"primarykey"` + SecurityGroupID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + Key *string + Value *string +} + +func (SecurityGroupTag) TableName() string { + return "aws_ec2_security_group_tags" +} + +func (c *Client) transformSecurityGroupIpRange(value *ec2.IpRange) *SecurityGroupIpRange { + return &SecurityGroupIpRange{ + AccountID: c.accountID, + Region: c.region, + CidrIp: value.CidrIp, + Description: value.Description, + } +} + +func (c *Client) transformSecurityGroupIpRanges(values []*ec2.IpRange) []*SecurityGroupIpRange { + var tValues []*SecurityGroupIpRange + for _, v := range values { + tValues = append(tValues, c.transformSecurityGroupIpRange(v)) + } + return tValues +} + +func (c *Client) transformSecurityGroupIpv6Range(value *ec2.Ipv6Range) *SecurityGroupIpv6Range { + return &SecurityGroupIpv6Range{ + AccountID: c.accountID, + Region: c.region, + CidrIpv6: value.CidrIpv6, + Description: value.Description, + } +} + +func (c *Client) transformSecurityGroupIpv6Ranges(values []*ec2.Ipv6Range) []*SecurityGroupIpv6Range { + var tValues []*SecurityGroupIpv6Range + for _, v := range values { + tValues = append(tValues, c.transformSecurityGroupIpv6Range(v)) + } + return tValues +} + +func (c *Client) transformSecurityGroupPrefixListId(value *ec2.PrefixListId) *SecurityGroupPrefixListId { + return &SecurityGroupPrefixListId{ + AccountID: c.accountID, + Region: c.region, + Description: value.Description, + PrefixListId: value.PrefixListId, + } +} + +func (c *Client) transformSecurityGroupPrefixListIds(values []*ec2.PrefixListId) []*SecurityGroupPrefixListId { + var tValues []*SecurityGroupPrefixListId + for _, v := range values { + tValues = append(tValues, c.transformSecurityGroupPrefixListId(v)) + } + return tValues +} + +func (c *Client) transformSecurityGroupUserIdGroupPair(value *ec2.UserIdGroupPair) *SecurityGroupUserIdGroupPair { + return &SecurityGroupUserIdGroupPair{ + AccountID: c.accountID, + Region: c.region, + Description: value.Description, + GroupId: value.GroupId, + GroupName: value.GroupName, + PeeringStatus: value.PeeringStatus, + UserId: value.UserId, + VpcId: value.VpcId, + VpcPeeringConnectionId: value.VpcPeeringConnectionId, + } +} + +func (c *Client) transformSecurityGroupUserIdGroupPairs(values []*ec2.UserIdGroupPair) []*SecurityGroupUserIdGroupPair { + var tValues []*SecurityGroupUserIdGroupPair + for _, v := range values { + tValues = append(tValues, c.transformSecurityGroupUserIdGroupPair(v)) + } + return tValues +} + +func (c *Client) transformSecurityGroupIpPermission(value *ec2.IpPermission) *SecurityGroupIpPermission { + return &SecurityGroupIpPermission{ + AccountID: c.accountID, + Region: c.region, + FromPort: value.FromPort, + IpProtocol: value.IpProtocol, + IpRanges: c.transformSecurityGroupIpRanges(value.IpRanges), + Ipv6Ranges: c.transformSecurityGroupIpv6Ranges(value.Ipv6Ranges), + PrefixListIds: c.transformSecurityGroupPrefixListIds(value.PrefixListIds), + ToPort: value.ToPort, + UserIdGroupPairs: c.transformSecurityGroupUserIdGroupPairs(value.UserIdGroupPairs), + } +} + +func (c *Client) transformSecurityGroupIpPermissions(values []*ec2.IpPermission) []*SecurityGroupIpPermission { + var tValues []*SecurityGroupIpPermission + for _, v := range values { + tValues = append(tValues, c.transformSecurityGroupIpPermission(v)) + } + return tValues +} + +func (c *Client) transformSecurityGroupTag(value *ec2.Tag) *SecurityGroupTag { + return &SecurityGroupTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformSecurityGroupTags(values []*ec2.Tag) []*SecurityGroupTag { + var tValues []*SecurityGroupTag + for _, v := range values { + tValues = append(tValues, c.transformSecurityGroupTag(v)) + } + return tValues +} + +func (c *Client) transformSecurityGroup(value *ec2.SecurityGroup) *SecurityGroup { + return &SecurityGroup{ + Region: c.region, + AccountID: c.accountID, + Description: value.Description, + GroupId: value.GroupId, + GroupName: value.GroupName, + IpPermissions: c.transformSecurityGroupIpPermissions(value.IpPermissions), + OwnerId: value.OwnerId, + Tags: c.transformSecurityGroupTags(value.Tags), + VpcId: value.VpcId, + } +} + +func (c *Client) transformSecurityGroups(values []*ec2.SecurityGroup) []*SecurityGroup { + var tValues []*SecurityGroup + for _, v := range values { + tValues = append(tValues, c.transformSecurityGroup(v)) + } + return tValues +} + +var SecurityGroupTables = []interface{}{ + &SecurityGroup{}, + &SecurityGroupIpPermission{}, + &SecurityGroupIpRange{}, + &SecurityGroupIpv6Range{}, + &SecurityGroupPrefixListId{}, + &SecurityGroupUserIdGroupPair{}, + &SecurityGroupTag{}, +} + +func (c *Client) securityGroups(gConfig interface{}) error { + var config ec2.DescribeSecurityGroupsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(SecurityGroupTables...) + for { + output, err := c.svc.DescribeSecurityGroups(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformSecurityGroups(output.SecurityGroups)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.security_groups"), zap.Int("count", len(output.SecurityGroups))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/subnets.go b/ec2/subnets.go new file mode 100644 index 000000000..27b783302 --- /dev/null +++ b/ec2/subnets.go @@ -0,0 +1,165 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type Subnet struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + AssignIpv6AddressOnCreation *bool + AvailabilityZone *string + AvailabilityZoneId *string + AvailableIpAddressCount *int64 + CidrBlock *string + CustomerOwnedIpv4Pool *string + DefaultForAz *bool + Ipv6CidrBlockAssociationSet []*SubnetIpv6CidrBlockAssociation `gorm:"constraint:OnDelete:CASCADE;"` + MapCustomerOwnedIpOnLaunch *bool + MapPublicIpOnLaunch *bool + OutpostArn *string + OwnerId *string + State *string + SubnetArn *string + SubnetId *string `neo:"unique"` + Tags []*SubnetTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string +} + +func (Subnet) TableName() string { + return "aws_ec2_subnets" +} + +type SubnetIpv6CidrBlockAssociation struct { + ID uint `gorm:"primarykey"` + SubnetID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AssociationId *string + Ipv6CidrBlock *string + State *string + StatusMessage *string +} + +func (SubnetIpv6CidrBlockAssociation) TableName() string { + return "aws_ec2_subnet_ipv6_cidr_block_associations" +} + +type SubnetTag struct { + ID uint `gorm:"primarykey"` + SubnetID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + Key *string + Value *string +} + +func (SubnetTag) TableName() string { + return "aws_ec2_subnet_tags" +} + +func (c *Client) transformSubnetIpv6CidrBlockAssociation(value *ec2.SubnetIpv6CidrBlockAssociation) *SubnetIpv6CidrBlockAssociation { + res := SubnetIpv6CidrBlockAssociation{ + AccountID: c.accountID, + Region: c.region, + AssociationId: value.AssociationId, + Ipv6CidrBlock: value.Ipv6CidrBlock, + } + if value.Ipv6CidrBlock != nil { + res.State = value.Ipv6CidrBlockState.State + res.StatusMessage = value.Ipv6CidrBlockState.StatusMessage + } + + return &res +} + +func (c *Client) transformSubnetIpv6CidrBlockAssociations(values []*ec2.SubnetIpv6CidrBlockAssociation) []*SubnetIpv6CidrBlockAssociation { + var tValues []*SubnetIpv6CidrBlockAssociation + for _, v := range values { + tValues = append(tValues, c.transformSubnetIpv6CidrBlockAssociation(v)) + } + return tValues +} + +func (c *Client) transformSubnetTag(value *ec2.Tag) *SubnetTag { + return &SubnetTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformSubnetTags(values []*ec2.Tag) []*SubnetTag { + var tValues []*SubnetTag + for _, v := range values { + tValues = append(tValues, c.transformSubnetTag(v)) + } + return tValues +} + +func (c *Client) transformSubnet(value *ec2.Subnet) *Subnet { + return &Subnet{ + Region: c.region, + AccountID: c.accountID, + AssignIpv6AddressOnCreation: value.AssignIpv6AddressOnCreation, + AvailabilityZone: value.AvailabilityZone, + AvailabilityZoneId: value.AvailabilityZoneId, + AvailableIpAddressCount: value.AvailableIpAddressCount, + CidrBlock: value.CidrBlock, + CustomerOwnedIpv4Pool: value.CustomerOwnedIpv4Pool, + DefaultForAz: value.DefaultForAz, + Ipv6CidrBlockAssociationSet: c.transformSubnetIpv6CidrBlockAssociations(value.Ipv6CidrBlockAssociationSet), + MapCustomerOwnedIpOnLaunch: value.MapCustomerOwnedIpOnLaunch, + MapPublicIpOnLaunch: value.MapPublicIpOnLaunch, + OutpostArn: value.OutpostArn, + OwnerId: value.OwnerId, + State: value.State, + SubnetArn: value.SubnetArn, + SubnetId: value.SubnetId, + Tags: c.transformSubnetTags(value.Tags), + VpcId: value.VpcId, + } +} + +func (c *Client) transformSubnets(values []*ec2.Subnet) []*Subnet { + var tValues []*Subnet + for _, v := range values { + tValues = append(tValues, c.transformSubnet(v)) + } + return tValues +} + +var SubnetTables = []interface{}{ + &Subnet{}, + &SubnetIpv6CidrBlockAssociation{}, + &SubnetTag{}, +} + +func (c *Client) subnets(gConfig interface{}) error { + var config ec2.DescribeSubnetsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(SubnetTables...) + for { + output, err := c.svc.DescribeSubnets(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformSubnets(output.Subnets)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.subnets"), zap.Int("count", len(output.Subnets))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/vpc_peering_connections.go b/ec2/vpc_peering_connections.go new file mode 100644 index 000000000..b75d950a0 --- /dev/null +++ b/ec2/vpc_peering_connections.go @@ -0,0 +1,272 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type VpcPeeringConnection struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + + AccCidrBlock *string + AccCidrBlockSet []*VpcPeeringConnectionAccCidrBlock `gorm:"constraint:OnDelete:CASCADE;"` + AccIpv6CidrBlockSet []*VpcPeeringConnectionAccIpv6CidrBlock `gorm:"constraint:OnDelete:CASCADE;"` + AccOwnerId *string + + AccOptAllowDnsResolutionFromRemoteVpc *bool + AccOptAllowEgressFromLocalClassicLinkToRemoteVpc *bool + AccOptAllowEgressFromLocalVpcToRemoteClassicLink *bool + + AccRegion *string + AccVpcId *string + + ExpirationTime *time.Time + + ReqCidrBlock *string + ReqCidrBlockSet []*VpcPeeringConnectionReqCidrBlock `gorm:"constraint:OnDelete:CASCADE;"` + ReqIpv6CidrBlockSet []*VpcPeeringConnectionReqIpv6CidrBlock `gorm:"constraint:OnDelete:CASCADE;"` + ReqOwnerId *string + + ReqOptAllowDnsResolutionFromRemoteVpc *bool + ReqOptAllowEgressFromLocalClassicLinkToRemoteVpc *bool + ReqOptAllowEgressFromLocalVpcToRemoteClassicLink *bool + + ReqRegion *string + ReqVpcId *string + + StatusCode *string + StatusMessage *string + + Tags []*VpcPeeringConnectionTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcPeeringConnectionId *string `neo:"unique"` +} + +func (VpcPeeringConnection) TableName() string { + return "aws_ec2_vpc_peering_connections" +} + +type VpcPeeringConnectionAccCidrBlock struct { + ID uint `gorm:"primarykey"` + VpcPeeringConnectionID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + CidrBlock *string +} + +func (VpcPeeringConnectionAccCidrBlock) TableName() string { + return "aws_ec2_vpc_peering_connection_acc_cidr_blocks" +} + +type VpcPeeringConnectionAccIpv6CidrBlock struct { + ID uint `gorm:"primarykey"` + VpcPeeringConnectionID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Ipv6CidrBlock *string +} + +func (VpcPeeringConnectionAccIpv6CidrBlock) TableName() string { + return "aws_ec2_vpc_peering_connection_acc_ipv6_cidr_blocks" +} + +type VpcPeeringConnectionReqCidrBlock struct { + ID uint `gorm:"primarykey"` + VpcPeeringConnectionID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + CidrBlock *string +} + +func (VpcPeeringConnectionReqCidrBlock) TableName() string { + return "aws_ec2_vpc_peering_connection_req_cidr_blocks" +} + +type VpcPeeringConnectionReqIpv6CidrBlock struct { + ID uint `gorm:"primarykey"` + VpcPeeringConnectionID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Ipv6CidrBlock *string +} + +func (VpcPeeringConnectionReqIpv6CidrBlock) TableName() string { + return "aws_ec2_vpc_peering_connection_req_ipv6_cidr_blocks" +} + +type VpcPeeringConnectionTag struct { + ID uint `gorm:"primarykey"` + VpcPeeringConnectionID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (VpcPeeringConnectionTag) TableName() string { + return "aws_ec2_vpc_peering_connection_tags" +} + +func (c *Client) transformVpcPeeringConnectionAccepterCidrBlocks(values []*ec2.CidrBlock) []*VpcPeeringConnectionAccCidrBlock { + var tValues []*VpcPeeringConnectionAccCidrBlock + for _, v := range values { + tValues = append(tValues, &VpcPeeringConnectionAccCidrBlock{ + AccountID: c.accountID, + Region: c.region, + CidrBlock: v.CidrBlock, + }) + } + return tValues +} + +func (c *Client) transformVpcPeeringConnectionAccepterIpv6CidrBlocks(values []*ec2.Ipv6CidrBlock) []*VpcPeeringConnectionAccIpv6CidrBlock { + var tValues []*VpcPeeringConnectionAccIpv6CidrBlock + for _, v := range values { + tValues = append(tValues, &VpcPeeringConnectionAccIpv6CidrBlock{ + AccountID: c.accountID, + Region: c.region, + Ipv6CidrBlock: v.Ipv6CidrBlock, + }) + } + return tValues +} + +func (c *Client) transformVpcPeeringConnectionRequesterCidrBlocks(values []*ec2.CidrBlock) []*VpcPeeringConnectionReqCidrBlock { + var tValues []*VpcPeeringConnectionReqCidrBlock + for _, v := range values { + tValues = append(tValues, &VpcPeeringConnectionReqCidrBlock{ + AccountID: c.accountID, + Region: c.region, + CidrBlock: v.CidrBlock, + }) + } + return tValues +} + +func (c *Client) transformVpcPeeringConnectionRequesterIpv6CidrBlocks(values []*ec2.Ipv6CidrBlock) []*VpcPeeringConnectionReqIpv6CidrBlock { + var tValues []*VpcPeeringConnectionReqIpv6CidrBlock + for _, v := range values { + tValues = append(tValues, &VpcPeeringConnectionReqIpv6CidrBlock{ + AccountID: c.accountID, + Region: c.region, + Ipv6CidrBlock: v.Ipv6CidrBlock, + }) + } + return tValues +} + +func (c *Client) transformVpcPeeringConnectionTag(value *ec2.Tag) *VpcPeeringConnectionTag { + return &VpcPeeringConnectionTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformVpcPeeringConnectionTags(values []*ec2.Tag) []*VpcPeeringConnectionTag { + var tValues []*VpcPeeringConnectionTag + for _, v := range values { + tValues = append(tValues, c.transformVpcPeeringConnectionTag(v)) + } + return tValues +} + +func (c *Client) transformVpcPeeringConnection(value *ec2.VpcPeeringConnection) *VpcPeeringConnection { + res := VpcPeeringConnection{ + Region: c.region, + AccountID: c.accountID, + ExpirationTime: value.ExpirationTime, + Tags: c.transformVpcPeeringConnectionTags(value.Tags), + VpcPeeringConnectionId: value.VpcPeeringConnectionId, + } + + if value.Status != nil { + res.StatusMessage = value.Status.Message + res.StatusCode = value.Status.Code + } + + if value.AccepterVpcInfo != nil { + res.AccCidrBlock = value.AccepterVpcInfo.CidrBlock + res.AccCidrBlockSet = c.transformVpcPeeringConnectionAccepterCidrBlocks(value.AccepterVpcInfo.CidrBlockSet) + res.AccIpv6CidrBlockSet = c.transformVpcPeeringConnectionAccepterIpv6CidrBlocks(value.AccepterVpcInfo.Ipv6CidrBlockSet) + res.AccOwnerId = value.AccepterVpcInfo.OwnerId + res.AccRegion = value.AccepterVpcInfo.Region + res.AccVpcId = value.AccepterVpcInfo.VpcId + + if value.AccepterVpcInfo.PeeringOptions != nil { + res.AccOptAllowDnsResolutionFromRemoteVpc = value.AccepterVpcInfo.PeeringOptions.AllowDnsResolutionFromRemoteVpc + res.AccOptAllowEgressFromLocalClassicLinkToRemoteVpc = value.AccepterVpcInfo.PeeringOptions.AllowEgressFromLocalVpcToRemoteClassicLink + res.AccOptAllowEgressFromLocalVpcToRemoteClassicLink = value.AccepterVpcInfo.PeeringOptions.AllowEgressFromLocalClassicLinkToRemoteVpc + } + } + + if value.RequesterVpcInfo != nil { + res.ReqCidrBlock = value.RequesterVpcInfo.CidrBlock + res.ReqCidrBlockSet = c.transformVpcPeeringConnectionRequesterCidrBlocks(value.RequesterVpcInfo.CidrBlockSet) + res.ReqIpv6CidrBlockSet = c.transformVpcPeeringConnectionRequesterIpv6CidrBlocks(value.RequesterVpcInfo.Ipv6CidrBlockSet) + res.ReqOwnerId = value.RequesterVpcInfo.OwnerId + res.ReqRegion = value.RequesterVpcInfo.Region + res.ReqVpcId = value.RequesterVpcInfo.VpcId + if value.RequesterVpcInfo.PeeringOptions != nil { + res.ReqOptAllowDnsResolutionFromRemoteVpc = value.RequesterVpcInfo.PeeringOptions.AllowDnsResolutionFromRemoteVpc + res.ReqOptAllowEgressFromLocalClassicLinkToRemoteVpc = value.RequesterVpcInfo.PeeringOptions.AllowEgressFromLocalClassicLinkToRemoteVpc + res.ReqOptAllowEgressFromLocalVpcToRemoteClassicLink = value.RequesterVpcInfo.PeeringOptions.AllowEgressFromLocalVpcToRemoteClassicLink + } + } + + return &res +} + +func (c *Client) transformVpcPeeringConnections(values []*ec2.VpcPeeringConnection) []*VpcPeeringConnection { + var tValues []*VpcPeeringConnection + for _, v := range values { + tValues = append(tValues, c.transformVpcPeeringConnection(v)) + } + return tValues +} + +var VPCPeeringConnectionTables = []interface{}{ + &VpcPeeringConnection{}, + &VpcPeeringConnectionAccCidrBlock{}, + &VpcPeeringConnectionAccIpv6CidrBlock{}, + &VpcPeeringConnectionReqCidrBlock{}, + &VpcPeeringConnectionReqIpv6CidrBlock{}, + &VpcPeeringConnectionTag{}, +} + +func (c *Client) vpcPeeringConnections(gConfig interface{}) error { + var config ec2.DescribeVpcPeeringConnectionsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(VPCPeeringConnectionTables...) + for { + output, err := c.svc.DescribeVpcPeeringConnections(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformVpcPeeringConnections(output.VpcPeeringConnections)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.vpc_peering_connections"), zap.Int("count", len(output.VpcPeeringConnections))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ec2/vpcs.go b/ec2/vpcs.go new file mode 100644 index 000000000..adaf6cb94 --- /dev/null +++ b/ec2/vpcs.go @@ -0,0 +1,203 @@ +package ec2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type Vpc struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + CidrBlock *string + CidrBlockAssociations []*VpcCidrBlockAssociation `gorm:"constraint:OnDelete:CASCADE;"` + DhcpOptionsId *string + InstanceTenancy *string + Ipv6CidrBlockAssociations []*VpcIpv6CidrBlockAssociation `gorm:"constraint:OnDelete:CASCADE;"` + IsDefault *bool + OwnerId *string + State *string + Tags []*VpcTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string `neo:"unique"` +} + +func (Vpc) TableName() string { + return "aws_ec2_vpcs" +} + +type VpcCidrBlockAssociation struct { + ID uint `gorm:"primarykey"` + VpcID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AssociationId *string + CidrBlock *string + + State *string + StatusMessage *string +} + +func (VpcCidrBlockAssociation) TableName() string { + return "aws_ec2_vpc_cidr_block_associations" +} + +type VpcIpv6CidrBlockAssociation struct { + ID uint `gorm:"primarykey"` + VpcID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AssociationId *string + Ipv6CidrBlock *string + + State *string + StatusMessage *string + + Ipv6Pool *string + NetworkBorderGroup *string +} + +func (VpcIpv6CidrBlockAssociation) TableName() string { + return "aws_ec2_vpc_ipv6_cidr_block_associations" +} + +type VpcTag struct { + ID uint `gorm:"primarykey"` + VpcID uint `neo:"ignore"` + + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (VpcTag) TableName() string { + return "aws_ec2_vpc_tags" +} + +func (c *Client) transformVpcCidrBlockAssociation(value *ec2.VpcCidrBlockAssociation) *VpcCidrBlockAssociation { + res := VpcCidrBlockAssociation{ + AccountID: c.accountID, + Region: c.region, + AssociationId: value.AssociationId, + CidrBlock: value.CidrBlock, + } + if value.CidrBlockState != nil { + res.State = value.CidrBlockState.State + res.StatusMessage = value.CidrBlockState.State + } + + return &res +} + +func (c *Client) transformVpcCidrBlockAssociations(values []*ec2.VpcCidrBlockAssociation) []*VpcCidrBlockAssociation { + var tValues []*VpcCidrBlockAssociation + for _, v := range values { + tValues = append(tValues, c.transformVpcCidrBlockAssociation(v)) + } + return tValues +} + +func (c *Client) transformVpcIpv6CidrBlockAssociation(value *ec2.VpcIpv6CidrBlockAssociation) *VpcIpv6CidrBlockAssociation { + res := VpcIpv6CidrBlockAssociation{ + AccountID: c.accountID, + Region: c.region, + AssociationId: value.AssociationId, + Ipv6CidrBlock: value.Ipv6CidrBlock, + Ipv6Pool: value.Ipv6Pool, + NetworkBorderGroup: value.NetworkBorderGroup, + } + + if value.Ipv6CidrBlockState != nil { + res.State = value.Ipv6CidrBlockState.State + res.StatusMessage = value.Ipv6CidrBlockState.StatusMessage + } + + return &res +} + +func (c *Client) transformVpcIpv6CidrBlockAssociations(values []*ec2.VpcIpv6CidrBlockAssociation) []*VpcIpv6CidrBlockAssociation { + var tValues []*VpcIpv6CidrBlockAssociation + for _, v := range values { + tValues = append(tValues, c.transformVpcIpv6CidrBlockAssociation(v)) + } + return tValues +} + +func (c *Client) transformVpcTag(value *ec2.Tag) *VpcTag { + return &VpcTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformVpcTags(values []*ec2.Tag) []*VpcTag { + var tValues []*VpcTag + for _, v := range values { + tValues = append(tValues, c.transformVpcTag(v)) + } + return tValues +} + +func (c *Client) transformVpc(value *ec2.Vpc) *Vpc { + return &Vpc{ + Region: c.region, + AccountID: c.accountID, + CidrBlock: value.CidrBlock, + CidrBlockAssociations: c.transformVpcCidrBlockAssociations(value.CidrBlockAssociationSet), + DhcpOptionsId: value.DhcpOptionsId, + InstanceTenancy: value.InstanceTenancy, + Ipv6CidrBlockAssociations: c.transformVpcIpv6CidrBlockAssociations(value.Ipv6CidrBlockAssociationSet), + IsDefault: value.IsDefault, + OwnerId: value.OwnerId, + State: value.State, + Tags: c.transformVpcTags(value.Tags), + VpcId: value.VpcId, + } +} + +func (c *Client) transformVpcs(values []*ec2.Vpc) []*Vpc { + var tValues []*Vpc + for _, v := range values { + tValues = append(tValues, c.transformVpc(v)) + } + return tValues +} + +var VPCTables = []interface{}{ + &Vpc{}, + &VpcCidrBlockAssociation{}, + &VpcIpv6CidrBlockAssociation{}, + &VpcTag{}, +} + +func (c *Client) vpcs(gConfig interface{}) error { + var config ec2.DescribeVpcsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(VPCTables...) + for { + output, err := c.svc.DescribeVpcs(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformVpcs(output.Vpcs)) + c.log.Info("Fetched resources", zap.String("resource", "ec2.vpcs"), zap.Int("count", len(output.Vpcs))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/ecr/client.go b/ecr/client.go new file mode 100644 index 000000000..8d70334ed --- /dev/null +++ b/ecr/client.go @@ -0,0 +1,41 @@ +package ecr + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *ecr.ECR +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: ecr.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "images": + return c.images(config) + default: + return fmt.Errorf("unsupported resource ecr.%s", resource) + } +} diff --git a/ecr/images.go b/ecr/images.go new file mode 100644 index 000000000..f39d6bd13 --- /dev/null +++ b/ecr/images.go @@ -0,0 +1,219 @@ +package ecr + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecr" + "go.uber.org/zap" +) + + +type Repository struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + CreatedAt *time.Time + + EncryptionType *string + EncryptionKmsKey *string + + ImageScanningConfigurationScanOnPush *bool + ImageTagMutability *string + RegistryId *string + ARN *string + Name *string + URI *string + Images []*Image `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (Repository) TableName() string { + return "aws_ecr_repositories" +} + +type Image struct { + ID uint `gorm:"primarykey"` + RepositoryID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ArtifactMediaType *string + ImageDigest *string + ImageManifestMediaType *string + ImagePushedAt *time.Time + + ImageScanFindingsSeverityCounts []*ImageSeverityCount `gorm:"constraint:OnDelete:CASCADE;"` + ImageScanFindingsImageScanCompletedAt *time.Time + ImageScanFindingsVulnerabilitySourceUpdatedAt *time.Time + + ImageScanStatusDescription *string + ImageScanStatusStatus *string + ImageSizeInBytes *int64 + ImageTags []*ImageTags `gorm:"constraint:OnDelete:CASCADE;"` + RegistryId *string + RepositoryName *string +} + +func (Image) TableName() string { + return "aws_ecr_images" +} + +type ImageSeverityCount struct { + ID uint `gorm:"primarykey"` + ImageID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Severity string + Count *int64 +} + +func (ImageSeverityCount) TableName() string { + return "aws_ecr_image_severity_counts" +} + +type ImageTags struct { + ID uint `gorm:"primarykey"` + ImageID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Value *string +} + +func (ImageTags) TableName() string { + return "aws_ecr_image_tags" +} + +func (c *Client) transformRepository(value *ecr.Repository) *Repository { + tValue := Repository { + AccountID: c.accountID, + Region: c.region, + CreatedAt: value.CreatedAt, + ImageTagMutability: value.ImageTagMutability, + RegistryId: value.RegistryId, + ARN: value.RepositoryArn, + Name: value.RepositoryName, + URI: value.RepositoryUri, + } + + if value.EncryptionConfiguration != nil { + tValue.EncryptionType = value.EncryptionConfiguration.EncryptionType + tValue.EncryptionKmsKey = value.EncryptionConfiguration.KmsKey + } + + if value.ImageScanningConfiguration != nil { + tValue.ImageScanningConfigurationScanOnPush = value.ImageScanningConfiguration.ScanOnPush + } + + return &tValue +} + +func (c *Client) transformImages(values []*ecr.ImageDetail) []*Image { + var tValues []*Image + for _, value := range values { + tValue := Image{ + AccountID: c.accountID, + Region: c.region, + ArtifactMediaType: value.ArtifactMediaType, + ImageDigest: value.ImageDigest, + ImageManifestMediaType: value.ImageManifestMediaType, + ImagePushedAt: value.ImagePushedAt, + ImageSizeInBytes: value.ImageSizeInBytes, + ImageTags: c.transformImageTags(value.ImageTags), + RegistryId: value.RegistryId, + RepositoryName: value.RepositoryName, + } + + if value.ImageScanFindingsSummary != nil { + tValue.ImageScanFindingsImageScanCompletedAt = value.ImageScanFindingsSummary.ImageScanCompletedAt + tValue.ImageScanFindingsVulnerabilitySourceUpdatedAt = value.ImageScanFindingsSummary.VulnerabilitySourceUpdatedAt + for severity, count := range value.ImageScanFindingsSummary.FindingSeverityCounts { + tValue.ImageScanFindingsSeverityCounts = append(tValue.ImageScanFindingsSeverityCounts, + &ImageSeverityCount{ + AccountID: c.accountID, + Region: c.region, + Severity: severity, + Count: count, + }) + } + } + + if value.ImageScanStatus != nil { + tValue.ImageScanStatusDescription = value.ImageScanStatus.Description + tValue.ImageScanStatusStatus = value.ImageScanStatus.Status + } + tValues = append(tValues, &tValue) + } + return tValues +} +func (c *Client) transformImageTags(values []*string) []*ImageTags { + var tValues []*ImageTags + for _, v := range values { + tValues = append(tValues, &ImageTags{ + AccountID: c.accountID, + Region: c.region, + Value: v, + }) + } + return tValues +} + +type ImageConfig struct { + Filter string +} + +var ImageTables = []interface{}{ + &Repository{}, + &Image{}, + &ImageSeverityCount{}, + &ImageTags{}, +} + +func (c *Client) images(_ interface{}) error { + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(ImageTables...) + + var describeRepositoriesInput ecr.DescribeRepositoriesInput + var describeImagesInput ecr.DescribeImagesInput + var maxResults int64 + maxResults = 1000 + describeRepositoriesInput.MaxResults = &maxResults + describeImagesInput.MaxResults = &maxResults + totalImages := 0 + for { + outputRepos, err := c.svc.DescribeRepositories(&describeRepositoriesInput) + if err != nil { + return err + } + for _, repo := range outputRepos.Repositories { + describeImagesInput.RepositoryName = repo.RepositoryName + describeImagesInput.NextToken = nil + tRepo := c.transformRepository(repo) + for { + outputDescribeImages, err := c.svc.DescribeImages(&describeImagesInput) + if err != nil { + return err + } + tRepo.Images = append(tRepo.Images, c.transformImages(outputDescribeImages.ImageDetails)...) + totalImages += len(outputDescribeImages.ImageDetails) + c.log.Info("Fetched resources", zap.String("resource", "ecr.images"), zap.Int("count", len(outputDescribeImages.ImageDetails))) + + if aws.StringValue(outputDescribeImages.NextToken) == "" { + break + } + describeImagesInput.NextToken = outputDescribeImages.NextToken + } + c.db.InsertOne(tRepo) + } + if aws.StringValue(outputRepos.NextToken) == "" { + break + } + describeRepositoriesInput.NextToken = outputRepos.NextToken + } + + if totalImages == 0 { + c.log.Info("Fetched resources", zap.String("resource", "ecr.images"), zap.Int("count", 0)) + } + return nil +} diff --git a/ecs/client.go b/ecs/client.go new file mode 100644 index 000000000..5dfb0f68d --- /dev/null +++ b/ecs/client.go @@ -0,0 +1,41 @@ +package ecs + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *ecs.ECS +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: ecs.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "clusters": + return c.clusters(config) + default: + return fmt.Errorf("unsupported resource ecr.%s", resource) + } +} diff --git a/ecs/clusters.go b/ecs/clusters.go new file mode 100644 index 000000000..2d29fd9d4 --- /dev/null +++ b/ecs/clusters.go @@ -0,0 +1,238 @@ +package ecs + +import ( + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/cloudquery/cloudquery/providers/common" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type Cluster struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + ActiveServicesCount *int64 + AttachmentsStatus *string + CapacityProviders *string + ClusterArn *string `neo:"unique"` + ClusterName *string + DefaultCapacityProviderStrategy []*ClusterCapacityProviderStrategyItem `gorm:"constraint:OnDelete:CASCADE;"` + PendingTasksCount *int64 + RegisteredContainerInstancesCount *int64 + RunningTasksCount *int64 + Settings []*ClusterSetting `gorm:"constraint:OnDelete:CASCADE;"` + Statistics []*ClusterKeyValuePair `gorm:"constraint:OnDelete:CASCADE;"` + Status *string + Tags []*ClusterTag `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (Cluster) TableName() string { + return "aws_ecs_clusters" +} + +type ClusterKeyValuePair struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Name *string + Value *string +} + +func (ClusterKeyValuePair) TableName() string { + return "aws_ecs_cluster_key_value_pairs" +} + +type ClusterCapacityProviderStrategyItem struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Base *int64 + CapacityProvider *string + Weight *int64 +} + +func (ClusterCapacityProviderStrategyItem) TableName() string { + return "aws_ecs_cluster_capacity_provider_strategy_items" +} + +type ClusterSetting struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Name *string + Value *string +} + +func (ClusterSetting) TableName() string { + return "aws_ecs_cluster_settings" +} + +type ClusterTag struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (ClusterTag) TableName() string { + return "aws_ecs_cluster_tags" +} + +func (c *Client) transformClusterKeyValuePair(value *ecs.KeyValuePair) *ClusterKeyValuePair { + return &ClusterKeyValuePair{ + AccountID: c.accountID, + Region: c.region, + Name: value.Name, + Value: value.Value, + } +} + +func (c *Client) transformClusterKeyValuePairs(values []*ecs.KeyValuePair) []*ClusterKeyValuePair { + var tValues []*ClusterKeyValuePair + for _, v := range values { + tValues = append(tValues, c.transformClusterKeyValuePair(v)) + } + return tValues +} + +func (c *Client) transformClusterCapacityProviderStrategyItem(value *ecs.CapacityProviderStrategyItem) *ClusterCapacityProviderStrategyItem { + return &ClusterCapacityProviderStrategyItem{ + AccountID: c.accountID, + Region: c.region, + Base: value.Base, + CapacityProvider: value.CapacityProvider, + Weight: value.Weight, + } +} + +func (c *Client) transformClusterCapacityProviderStrategyItems(values []*ecs.CapacityProviderStrategyItem) []*ClusterCapacityProviderStrategyItem { + var tValues []*ClusterCapacityProviderStrategyItem + for _, v := range values { + tValues = append(tValues, c.transformClusterCapacityProviderStrategyItem(v)) + } + return tValues +} + +func (c *Client) transformClusterSetting(value *ecs.ClusterSetting) *ClusterSetting { + return &ClusterSetting{ + AccountID: c.accountID, + Region: c.region, + Name: value.Name, + Value: value.Value, + } +} + +func (c *Client) transformClusterSettings(values []*ecs.ClusterSetting) []*ClusterSetting { + var tValues []*ClusterSetting + for _, v := range values { + tValues = append(tValues, c.transformClusterSetting(v)) + } + return tValues +} + +func (c *Client) transformClusterTag(value *ecs.Tag) *ClusterTag { + return &ClusterTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformClusterTags(values []*ecs.Tag) []*ClusterTag { + var tValues []*ClusterTag + for _, v := range values { + tValues = append(tValues, c.transformClusterTag(v)) + } + return tValues +} + +func (c *Client) transformCluster(value *ecs.Cluster) *Cluster { + return &Cluster{ + Region: c.region, + AccountID: c.accountID, + ActiveServicesCount: value.ActiveServicesCount, + AttachmentsStatus: value.AttachmentsStatus, + CapacityProviders: common.StringListToString(value.CapacityProviders), + ClusterArn: value.ClusterArn, + ClusterName: value.ClusterName, + DefaultCapacityProviderStrategy: c.transformClusterCapacityProviderStrategyItems(value.DefaultCapacityProviderStrategy), + PendingTasksCount: value.PendingTasksCount, + RegisteredContainerInstancesCount: value.RegisteredContainerInstancesCount, + RunningTasksCount: value.RunningTasksCount, + Settings: c.transformClusterSettings(value.Settings), + Statistics: c.transformClusterKeyValuePairs(value.Statistics), + Status: value.Status, + Tags: c.transformClusterTags(value.Tags), + } +} + +func (c *Client) transformClusters(values []*ecs.Cluster) []*Cluster { + var tValues []*Cluster + for _, v := range values { + tValues = append(tValues, c.transformCluster(v)) + } + return tValues +} + +var ClusterTables = []interface{}{ + &Cluster{}, + &ClusterKeyValuePair{}, + &ClusterCapacityProviderStrategyItem{}, + &ClusterSetting{}, + &ClusterTag{}, + + &Service{}, + &ServiceSecurityGroups{}, + &ServiceSubnets{}, + &ServiceCapProviderStrategy{}, + &ServiceLoadBalancer{}, + &ServicePlacementConstraint{}, + &ServicePlacementStrategy{}, + &ServiceRegistry{}, + &ServiceTag{}, +} + +func (c *Client) clusters(gConfig interface{}) error { + var config ecs.DescribeClustersInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(ClusterTables...) + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(&Service{}) + var listConfig ecs.ListClustersInput + for { + listOutput, err := c.svc.ListClusters(&listConfig) + if err != nil { + return err + } + err = c.services(listOutput.ClusterArns) + if err != nil { + return err + } + config.Clusters = listOutput.ClusterArns + output, err := c.svc.DescribeClusters(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformClusters(output.Clusters)) + c.log.Info("Fetched resources", zap.String("resource", "ecs.cluster"), zap.Int("count", len(output.Clusters))) + + if listOutput.NextToken == nil { + break + } + listConfig.NextToken = listOutput.NextToken + } + return nil +} diff --git a/ecs/services.go b/ecs/services.go new file mode 100644 index 000000000..4db53fb35 --- /dev/null +++ b/ecs/services.go @@ -0,0 +1,361 @@ +package ecs + +import ( + "github.com/aws/aws-sdk-go/service/ecs" + "go.uber.org/zap" + "time" +) + +type Service struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + + CapProviderStrategy []*ServiceCapProviderStrategy `gorm:"constraint:OnDelete:CASCADE;"` + ClusterArn *string + CreatedAt *time.Time + CreatedBy *string + + DeploymentConfigurationMaximumPercent *int64 + DeploymentConfigurationMinimumHealthyPercent *int64 + + DeploymentControllerType *string + DesiredCount *int64 + EnableECSManagedTags *bool + + HealthCheckGracePeriodSeconds *int64 + LaunchType *string + LoadBalancers []*ServiceLoadBalancer `gorm:"constraint:OnDelete:CASCADE;"` + + AssignPublicIp *string + SecurityGroups []*ServiceSecurityGroups `gorm:"constraint:OnDelete:CASCADE;"` + Subnets []*ServiceSubnets `gorm:"constraint:OnDelete:CASCADE;"` + PendingCount *int64 + PlacementConstraints []*ServicePlacementConstraint `gorm:"constraint:OnDelete:CASCADE;"` + PlacementStrategy []*ServicePlacementStrategy `gorm:"constraint:OnDelete:CASCADE;"` + PlatformVersion *string + PropagateTags *string + RoleArn *string + RunningCount *int64 + SchedulingStrategy *string + ServiceArn *string `neo:"unique"` + ServiceName *string + ServiceRegistries []*ServiceRegistry `gorm:"constraint:OnDelete:CASCADE;"` + Status *string + Tags []*ServiceTag `gorm:"constraint:OnDelete:CASCADE;"` + TaskDefinition *string +} + +func (Service) TableName() string { + return "aws_ecs_services" +} + +type ServiceCapProviderStrategy struct { + ID uint `gorm:"primarykey"` + ServiceID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Base *int64 + CapacityProvider *string + Weight *int64 +} + +func (ServiceCapProviderStrategy) TableName() string { + return "aws_ecs_service_cap_provider_strategies" +} + +type ServiceSecurityGroups struct { + ID uint `gorm:"primarykey"` + ServiceID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Value *string +} + +func (ServiceSecurityGroups) TableName() string { + return "aws_ecs_service_security_groups" +} + +type ServiceSubnets struct { + ID uint `gorm:"primarykey"` + ServiceID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Value *string +} + +func (ServiceSubnets) TableName() string { + return "aws_ecs_service_subnets" +} + +type ServiceLoadBalancer struct { + ID uint `gorm:"primarykey"` + ServiceID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ContainerName *string + ContainerPort *int64 + LoadBalancerName *string + TargetGroupArn *string +} + +func (ServiceLoadBalancer) TableName() string { + return "aws_ecs_service_load_balancers" +} + +type ServicePlacementConstraint struct { + ID uint `gorm:"primarykey"` + ServiceID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Expression *string + Type *string +} + +func (ServicePlacementConstraint) TableName() string { + return "aws_ecs_service_placement_constraints" +} + +type ServicePlacementStrategy struct { + ID uint `gorm:"primarykey"` + ServiceID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Field *string + Type *string +} + +func (ServicePlacementStrategy) TableName() string { + return "aws_ecs_service_placement_strategies" +} + +type ServiceRegistry struct { + ID uint `gorm:"primarykey"` + ServiceID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ContainerName *string + ContainerPort *int64 + Port *int64 + RegistryArn *string +} + +func (ServiceRegistry) TableName() string { + return "aws_ecs_service_registries" +} + +type ServiceTag struct { + ID uint `gorm:"primarykey"` + ServiceID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (ServiceTag) TableName() string { + return "aws_ecs_service_tags" +} + +func (c *Client) transformServices(values []*ecs.Service) []*Service { + var tValues []*Service + for _, value := range values { + tValue := Service{ + AccountID: c.accountID, + Region: c.region, + CapProviderStrategy: c.transformServiceCapacityProviderStrategyItems(value.CapacityProviderStrategy), + ClusterArn: value.ClusterArn, + CreatedAt: value.CreatedAt, + CreatedBy: value.CreatedBy, + DesiredCount: value.DesiredCount, + EnableECSManagedTags: value.EnableECSManagedTags, + HealthCheckGracePeriodSeconds: value.HealthCheckGracePeriodSeconds, + LaunchType: value.LaunchType, + LoadBalancers: c.transformServiceLoadBalancers(value.LoadBalancers), + PendingCount: value.PendingCount, + PlacementConstraints: c.transformServicePlacementConstraints(value.PlacementConstraints), + PlacementStrategy: c.transformServicePlacementStrategies(value.PlacementStrategy), + PlatformVersion: value.PlatformVersion, + PropagateTags: value.PropagateTags, + RoleArn: value.RoleArn, + RunningCount: value.RunningCount, + SchedulingStrategy: value.SchedulingStrategy, + ServiceArn: value.ServiceArn, + ServiceName: value.ServiceName, + ServiceRegistries: c.transformServiceRegistries(value.ServiceRegistries), + Status: value.Status, + Tags: c.transformServiceTags(value.Tags), + TaskDefinition: value.TaskDefinition, + } + if value.DeploymentConfiguration != nil { + tValue.DeploymentConfigurationMaximumPercent = value.DeploymentConfiguration.MaximumPercent + tValue.DeploymentConfigurationMinimumHealthyPercent = value.DeploymentConfiguration.MinimumHealthyPercent + } + if value.DeploymentController != nil { + tValue.DeploymentControllerType = value.DeploymentController.Type + } + if value.NetworkConfiguration != nil && value.NetworkConfiguration.AwsvpcConfiguration != nil { + tValue.SecurityGroups = c.transformServiceAwsVpcConfigurationSecurityGroups(value.NetworkConfiguration.AwsvpcConfiguration.SecurityGroups) + tValue.Subnets = c.transformServiceAwsVpcConfigurationSubnets(value.NetworkConfiguration.AwsvpcConfiguration.Subnets) + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformServiceCapacityProviderStrategyItems(values []*ecs.CapacityProviderStrategyItem) []*ServiceCapProviderStrategy { + var tValues []*ServiceCapProviderStrategy + for _, value := range values { + tValue := ServiceCapProviderStrategy{ + AccountID: c.accountID, + Region: c.region, + Base: value.Base, + CapacityProvider: value.CapacityProvider, + Weight: value.Weight, + } + tValues = append(tValues, &tValue) + } + return tValues +} +func (c *Client) transformServiceAwsVpcConfigurationSecurityGroups(values []*string) []*ServiceSecurityGroups { + var tValues []*ServiceSecurityGroups + for _, v := range values { + tValues = append(tValues, &ServiceSecurityGroups{ + AccountID: c.accountID, + Region: c.region, + Value: v, + }) + } + return tValues +} + +func (c *Client) transformServiceAwsVpcConfigurationSubnets(values []*string) []*ServiceSubnets { + var tValues []*ServiceSubnets + for _, v := range values { + tValues = append(tValues, &ServiceSubnets{ + AccountID: c.accountID, + Region: c.region, + Value: v, + }) + } + return tValues +} + +func (c *Client) transformServiceLoadBalancers(values []*ecs.LoadBalancer) []*ServiceLoadBalancer { + var tValues []*ServiceLoadBalancer + for _, value := range values { + tValue := ServiceLoadBalancer{ + AccountID: c.accountID, + Region: c.region, + ContainerName: value.ContainerName, + ContainerPort: value.ContainerPort, + LoadBalancerName: value.LoadBalancerName, + TargetGroupArn: value.TargetGroupArn, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformServicePlacementConstraints(values []*ecs.PlacementConstraint) []*ServicePlacementConstraint { + var tValues []*ServicePlacementConstraint + for _, value := range values { + tValue := ServicePlacementConstraint{ + AccountID: c.accountID, + Region: c.region, + Expression: value.Expression, + Type: value.Type, + } + + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformServicePlacementStrategies(values []*ecs.PlacementStrategy) []*ServicePlacementStrategy { + var tValues []*ServicePlacementStrategy + for _, value := range values { + tValue := ServicePlacementStrategy{ + AccountID: c.accountID, + Region: c.region, + Field: value.Field, + Type: value.Type, + } + + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformServiceRegistries(values []*ecs.ServiceRegistry) []*ServiceRegistry { + var tValues []*ServiceRegistry + for _, value := range values { + tValue := ServiceRegistry{ + AccountID: c.accountID, + Region: c.region, + ContainerName: value.ContainerName, + ContainerPort: value.ContainerPort, + Port: value.Port, + RegistryArn: value.RegistryArn, + } + + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformServiceTags(values []*ecs.Tag) []*ServiceTag { + var tValues []*ServiceTag + for _, value := range values { + tValue := ServiceTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) services(clusters []*string) error { + + var listInput ecs.ListServicesInput + for _, cluster := range clusters { + listInput.Cluster = cluster + for { + listOutput, err := c.svc.ListServices(&listInput) + if err != nil { + return err + } + if len(listOutput.ServiceArns) == 0 { + c.log.Info("Fetched resources", zap.String("resource", "ecs.services"), zap.Int("count", 0)) + break + } + output, err := c.svc.DescribeServices(&ecs.DescribeServicesInput{ + Cluster: cluster, + Services: listOutput.ServiceArns, + }) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformServices(output.Services)) + c.log.Info("Fetched resources", zap.String("resource", "ecs.services"), zap.Int("count", len(output.Services))) + + if listOutput.NextToken == nil { + break + } + listInput.NextToken = listOutput.NextToken + } + } + return nil +} diff --git a/efs/client.go b/efs/client.go new file mode 100644 index 000000000..62910798f --- /dev/null +++ b/efs/client.go @@ -0,0 +1,41 @@ +package efs + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/efs" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *efs.EFS +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: efs.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "filesystems": + return c.fileSystems(config) + default: + return fmt.Errorf("unsupported resource ecr.%s", resource) + } +} diff --git a/efs/filesystems.go b/efs/filesystems.go new file mode 100644 index 000000000..f3d9a7c80 --- /dev/null +++ b/efs/filesystems.go @@ -0,0 +1,136 @@ +package efs + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/efs" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type FileSystemDescription struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + CreationTime *time.Time + CreationToken *string + Encrypted *bool + FileSystemArn *string `neo:"unique"` + FileSystemId *string + KmsKeyId *string + LifeCycleState *string + Name *string + NumberOfMountTargets *int64 + OwnerId *string + PerformanceMode *string + ProvisionedThroughputInMibps *float64 + + SizeInBytesTimestamp *time.Time + SizeInBytesValue *int64 + SizeInBytesValueInIA *int64 + SizeInBytesValueInStandard *int64 + + Tags []*FileSystemDescriptionTag `gorm:"constraint:OnDelete:CASCADE;"` + ThroughputMode *string +} + +func (FileSystemDescription) TableName() string { + return "aws_efs_file_system_descriptions" +} + +type FileSystemDescriptionTag struct { + ID uint `gorm:"primarykey"` + FileSystemDescriptionID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (FileSystemDescriptionTag) TableName() string { + return "aws_efs_file_system_description_tags" +} + +func (c *Client) transformFileSystemDescriptionTag(value *efs.Tag) *FileSystemDescriptionTag { + return &FileSystemDescriptionTag{ + Region: c.region, + AccountID: c.accountID, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformFileSystemDescriptionTags(values []*efs.Tag) []*FileSystemDescriptionTag { + var tValues []*FileSystemDescriptionTag + for _, v := range values { + tValues = append(tValues, c.transformFileSystemDescriptionTag(v)) + } + return tValues +} + +func (c *Client) transformFileSystemDescription(value *efs.FileSystemDescription) *FileSystemDescription { + res := FileSystemDescription{ + Region: c.region, + AccountID: c.accountID, + CreationTime: value.CreationTime, + CreationToken: value.CreationToken, + Encrypted: value.Encrypted, + FileSystemArn: value.FileSystemArn, + FileSystemId: value.FileSystemId, + KmsKeyId: value.KmsKeyId, + LifeCycleState: value.LifeCycleState, + Name: value.Name, + NumberOfMountTargets: value.NumberOfMountTargets, + OwnerId: value.OwnerId, + PerformanceMode: value.PerformanceMode, + ProvisionedThroughputInMibps: value.ProvisionedThroughputInMibps, + Tags: c.transformFileSystemDescriptionTags(value.Tags), + ThroughputMode: value.ThroughputMode, + } + + if value.SizeInBytes != nil { + res.SizeInBytesTimestamp = value.SizeInBytes.Timestamp + res.SizeInBytesValue = value.SizeInBytes.Value + res.SizeInBytesValueInIA = value.SizeInBytes.ValueInIA + res.SizeInBytesValueInStandard = value.SizeInBytes.ValueInStandard + } + + return &res +} + +func (c *Client) transformFileSystemDescriptions(values []*efs.FileSystemDescription) []*FileSystemDescription { + var tValues []*FileSystemDescription + for _, v := range values { + tValues = append(tValues, c.transformFileSystemDescription(v)) + } + return tValues +} + +var FileSystemTables = []interface{}{ + &FileSystemDescription{}, + &FileSystemDescriptionTag{}, +} + +func (c *Client) fileSystems(gConfig interface{}) error { + var config efs.DescribeFileSystemsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(FileSystemTables...) + for { + output, err := c.svc.DescribeFileSystems(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformFileSystemDescriptions(output.FileSystems)) + c.log.Info("Fetched resources", zap.String("resource", "efs.filesystems"), zap.Int("count", len(output.FileSystems))) + if aws.StringValue(output.NextMarker) == "" { + break + } + config.Marker = output.NextMarker + } + return nil +} diff --git a/elasticbeanstalk/client.go b/elasticbeanstalk/client.go new file mode 100644 index 000000000..559222571 --- /dev/null +++ b/elasticbeanstalk/client.go @@ -0,0 +1,41 @@ +package elasticbeanstalk + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/elasticbeanstalk" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *elasticbeanstalk.ElasticBeanstalk +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: elasticbeanstalk.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "environments": + return c.environments(config) + default: + return fmt.Errorf("unsupported resource autoscaling.%s", resource) + } +} diff --git a/elasticbeanstalk/enironments.go b/elasticbeanstalk/enironments.go new file mode 100644 index 000000000..a8db2f61a --- /dev/null +++ b/elasticbeanstalk/enironments.go @@ -0,0 +1,191 @@ +package elasticbeanstalk + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elasticbeanstalk" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Environment struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + AbortableOperationInProgress *bool + ApplicationName *string + CNAME *string + DateCreated *time.Time + DateUpdated *time.Time + Description *string + EndpointURL *string + EnvironmentArn *string `neo:"unique"` + EnvironmentId *string + EnvironmentLinks []*EnvironmentLink `gorm:"constraint:OnDelete:CASCADE;"` + EnvironmentName *string + Health *string + HealthStatus *string + OperationsRole *string + PlatformArn *string + + LoadBalancerDomain *string + LoadBalancerListeners []*EnvironmentListener `gorm:"constraint:OnDelete:CASCADE;"` + LoadBalancerName *string + + SolutionStackName *string + Status *string + TemplateName *string + + TierName *string + TierType *string + TierVersion *string + + VersionLabel *string +} + +func (Environment) TableName() string { + return "aws_elasticbeanstalk_environments" +} + +type EnvironmentLink struct { + ID uint `gorm:"primarykey"` + EnvironmentID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + EnvironmentName *string + LinkName *string +} + +func (EnvironmentLink) TableName() string { + return "aws_elasticbeanstalk_environment_links" +} + +type EnvironmentListener struct { + ID uint `gorm:"primarykey"` + EnvironmentID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Port *int64 + Protocol *string +} + +func (EnvironmentListener) TableName() string { + return "aws_elasticbeanstalk_environment_listeners" +} + +func (c *Client) transformEnvironmentLink(value *elasticbeanstalk.EnvironmentLink) *EnvironmentLink { + return &EnvironmentLink{ + AccountID: c.accountID, + Region: c.region, + EnvironmentName: value.EnvironmentName, + LinkName: value.LinkName, + } +} + +func (c *Client) transformEnvironmentDescriptionEnvironmentLinks(values []*elasticbeanstalk.EnvironmentLink) []*EnvironmentLink { + var tValues []*EnvironmentLink + for _, v := range values { + tValues = append(tValues, c.transformEnvironmentLink(v)) + } + return tValues +} + +func (c *Client) transformEnvironmentListener(value *elasticbeanstalk.Listener) *EnvironmentListener { + return &EnvironmentListener{ + AccountID: c.accountID, + Region: c.region, + Port: value.Port, + Protocol: value.Protocol, + } +} + +func (c *Client) transformEnvironmentListeners(values []*elasticbeanstalk.Listener) []*EnvironmentListener { + var tValues []*EnvironmentListener + for _, v := range values { + tValues = append(tValues, c.transformEnvironmentListener(v)) + } + return tValues +} + +func (c *Client) transformEnvironment(value *elasticbeanstalk.EnvironmentDescription) *Environment { + res := Environment{ + Region: c.region, + AccountID: c.accountID, + AbortableOperationInProgress: value.AbortableOperationInProgress, + ApplicationName: value.ApplicationName, + CNAME: value.CNAME, + DateCreated: value.DateCreated, + DateUpdated: value.DateUpdated, + Description: value.Description, + EndpointURL: value.EndpointURL, + EnvironmentArn: value.EnvironmentArn, + EnvironmentId: value.EnvironmentId, + EnvironmentName: value.EnvironmentName, + Health: value.Health, + HealthStatus: value.HealthStatus, + OperationsRole: value.OperationsRole, + PlatformArn: value.PlatformArn, + SolutionStackName: value.SolutionStackName, + Status: value.Status, + TemplateName: value.TemplateName, + VersionLabel: value.VersionLabel, + } + + if value.Tier != nil { + res.TierName = value.Tier.Name + res.TierType = value.Tier.Type + res.TierVersion = value.Tier.Version + } + + if value.EnvironmentLinks != nil { + res.EnvironmentLinks = c.transformEnvironmentDescriptionEnvironmentLinks(value.EnvironmentLinks) + } + + if value.Resources != nil && value.Resources.LoadBalancer != nil { + res.LoadBalancerDomain = value.Resources.LoadBalancer.Domain + res.LoadBalancerListeners = c.transformEnvironmentListeners(value.Resources.LoadBalancer.Listeners) + res.LoadBalancerName = value.Resources.LoadBalancer.LoadBalancerName + } + + return &res +} + +func (c *Client) transformEnvironments(values []*elasticbeanstalk.EnvironmentDescription) []*Environment { + var tValues []*Environment + for _, v := range values { + tValues = append(tValues, c.transformEnvironment(v)) + } + return tValues +} + +var EnvironmentTables = []interface{}{ + &Environment{}, + &EnvironmentLink{}, + &EnvironmentListener{}, +} + +func (c *Client) environments(gConfig interface{}) error { + var config elasticbeanstalk.DescribeEnvironmentsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(EnvironmentTables...) + for { + output, err := c.svc.DescribeEnvironments(&config) + if err != nil { + return err + } + + c.db.ChunkedCreate(c.transformEnvironments(output.Environments)) + c.log.Info("Fetched resources", zap.String("resource", "elasticbeanstalk.environments"), zap.Int("count", len(output.Environments))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/elbv2/client.go b/elbv2/client.go new file mode 100644 index 000000000..a3de52757 --- /dev/null +++ b/elbv2/client.go @@ -0,0 +1,43 @@ +package elbv2 + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *elbv2.ELBV2 +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: elbv2.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "load_balancers": + return c.loadBalancers(config) + case "target_groups": + return c.targetGroups(config) + default: + return fmt.Errorf("unsupported resource elbv2.%s", resource) + } +} diff --git a/elbv2/load_balancers.go b/elbv2/load_balancers.go new file mode 100644 index 000000000..ac7899533 --- /dev/null +++ b/elbv2/load_balancers.go @@ -0,0 +1,168 @@ +package elbv2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/cloudquery/cloudquery/providers/common" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type LoadBalancer struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + AvailabilityZones []*LoadBalancerAvailabilityZone `gorm:"constraint:OnDelete:CASCADE;"` + CanonicalHostedZoneId *string + CreatedTime *time.Time + CustomerOwnedIpv4Pool *string + DNSName *string + IpAddressType *string + LoadBalancerArn *string `neo:"unique"` + LoadBalancerName *string + Scheme *string + SecurityGroups *string + + StateCode *string + StateReason *string + + Type *string + VpcId *string +} + +func (LoadBalancer) TableName() string { + return "aws_elbv2_load_balancers" +} + +type LoadBalancerAvailabilityZone struct { + ID uint `gorm:"primarykey"` + LoadBalancerID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + LoadBalancerAddresses []*LoadBalancerAddress `gorm:"constraint:OnDelete:CASCADE;"` + OutpostId *string + SubnetId *string + ZoneName *string +} + +func (LoadBalancerAvailabilityZone) TableName() string { + return "aws_elbv2_load_balancer_av_zones" +} + +type LoadBalancerAddress struct { + ID uint `gorm:"primarykey"` + LoadBalancerAvailabilityZoneID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AllocationId *string + IpAddress *string + PrivateIPv4Address *string +} + +func (LoadBalancerAddress) TableName() string { + return "aws_elbv2_load_balancer_addresses" +} + +func (c *Client) transformLoadBalancerAddress(value *elbv2.LoadBalancerAddress) *LoadBalancerAddress { + return &LoadBalancerAddress{ + AccountID: c.accountID, + Region: c.region, + AllocationId: value.AllocationId, + IpAddress: value.IpAddress, + PrivateIPv4Address: value.PrivateIPv4Address, + } +} + +func (c *Client) transformLoadBalancerAddresss(values []*elbv2.LoadBalancerAddress) []*LoadBalancerAddress { + var tValues []*LoadBalancerAddress + for _, v := range values { + tValues = append(tValues, c.transformLoadBalancerAddress(v)) + } + return tValues +} + +func (c *Client) transformLoadBalancerAvailabilityZone(value *elbv2.AvailabilityZone) *LoadBalancerAvailabilityZone { + return &LoadBalancerAvailabilityZone{ + AccountID: c.accountID, + Region: c.region, + LoadBalancerAddresses: c.transformLoadBalancerAddresss(value.LoadBalancerAddresses), + OutpostId: value.OutpostId, + SubnetId: value.SubnetId, + ZoneName: value.ZoneName, + } +} + +func (c *Client) transformLoadBalancerAvailabilityZones(values []*elbv2.AvailabilityZone) []*LoadBalancerAvailabilityZone { + var tValues []*LoadBalancerAvailabilityZone + for _, v := range values { + tValues = append(tValues, c.transformLoadBalancerAvailabilityZone(v)) + } + return tValues +} + +func (c *Client) transformLoadBalancer(value *elbv2.LoadBalancer) *LoadBalancer { + res := LoadBalancer{ + Region: c.region, + AccountID: c.accountID, + AvailabilityZones: c.transformLoadBalancerAvailabilityZones(value.AvailabilityZones), + CanonicalHostedZoneId: value.CanonicalHostedZoneId, + CreatedTime: value.CreatedTime, + CustomerOwnedIpv4Pool: value.CustomerOwnedIpv4Pool, + DNSName: value.DNSName, + IpAddressType: value.IpAddressType, + LoadBalancerArn: value.LoadBalancerArn, + LoadBalancerName: value.LoadBalancerName, + Scheme: value.Scheme, + SecurityGroups: common.StringListToString(value.SecurityGroups), + Type: value.Type, + VpcId: value.VpcId, + } + + if value.State != nil { + res.StateReason = value.State.Reason + res.StateCode = value.State.Code + } + + return &res +} + +func (c *Client) transformLoadBalancers(values []*elbv2.LoadBalancer) []*LoadBalancer { + var tValues []*LoadBalancer + for _, v := range values { + tValues = append(tValues, c.transformLoadBalancer(v)) + } + return tValues +} + +var LoadBalancerTables = []interface{}{ + &LoadBalancer{}, + &LoadBalancerAvailabilityZone{}, + &LoadBalancerAddress{}, +} + +func (c *Client) loadBalancers(gConfig interface{}) error { + var config elbv2.DescribeLoadBalancersInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(LoadBalancerTables...) + + for { + output, err := c.svc.DescribeLoadBalancers(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformLoadBalancers(output.LoadBalancers)) + c.log.Info("Fetched resources", zap.String("resource", "elbv2.load_balancers"), zap.Int("count", len(output.LoadBalancers))) + if aws.StringValue(output.NextMarker) == "" { + break + } + config.Marker = output.NextMarker + } + return nil +} diff --git a/elbv2/target_groups.go b/elbv2/target_groups.go new file mode 100644 index 000000000..5fb8644bc --- /dev/null +++ b/elbv2/target_groups.go @@ -0,0 +1,118 @@ +package elbv2 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type TargetGroup struct { + ID uint `gorm:"primarykey"` + AccountID string + Region string + + HealthCheckEnabled *bool + HealthCheckIntervalSeconds *int64 + HealthCheckPath *string + HealthCheckPort *string + HealthCheckProtocol *string + HealthCheckTimeoutSeconds *int64 + HealthyThresholdCount *int64 + LBArns []*TargetGroupLoadBalancerArns `gorm:"constraint:OnDelete:CASCADE;"` + + MatcherHttpCode *string + Port *int64 + Protocol *string + TargetGroupArn *string `neo:"unique"` + TargetGroupName *string + TargetType *string + UnhealthyThresholdCount *int64 + VpcId *string +} + +func (TargetGroup) TableName() string { + return "aws_elbv2_target_groups" +} + +type TargetGroupLoadBalancerArns struct { + ID uint `gorm:"primarykey"` + TargetGroupID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + Value *string +} + +func (TargetGroupLoadBalancerArns) TableName() string { + return "aws_elbv2_target_group_load_balancer_arns" +} + +func (c *Client) transformTargetGroups(values []*elbv2.TargetGroup) []*TargetGroup { + var tValues []*TargetGroup + for _, value := range values { + tValue := TargetGroup{ + AccountID: c.accountID, + Region: c.region, + HealthCheckEnabled: value.HealthCheckEnabled, + HealthCheckIntervalSeconds: value.HealthCheckIntervalSeconds, + HealthCheckPath: value.HealthCheckPath, + HealthCheckPort: value.HealthCheckPort, + HealthCheckProtocol: value.HealthCheckProtocol, + HealthCheckTimeoutSeconds: value.HealthCheckTimeoutSeconds, + HealthyThresholdCount: value.HealthyThresholdCount, + LBArns: c.transformTargetGroupLoadBalancerArns(value.LoadBalancerArns), + Port: value.Port, + Protocol: value.Protocol, + TargetGroupArn: value.TargetGroupArn, + TargetGroupName: value.TargetGroupName, + TargetType: value.TargetType, + UnhealthyThresholdCount: value.UnhealthyThresholdCount, + VpcId: value.VpcId, + } + if value.Matcher != nil { + + tValue.MatcherHttpCode = value.Matcher.HttpCode + } + tValues = append(tValues, &tValue) + } + return tValues +} +func (c *Client) transformTargetGroupLoadBalancerArns(values []*string) []*TargetGroupLoadBalancerArns { + var tValues []*TargetGroupLoadBalancerArns + for _, v := range values { + tValues = append(tValues, &TargetGroupLoadBalancerArns{ + AccountID: c.accountID, + Region: c.region, + Value: v, + }) + } + return tValues +} + +var TargetGroupTables = []interface{}{ + &TargetGroup{}, + &TargetGroupLoadBalancerArns{}, +} + +func (c *Client) targetGroups(gConfig interface{}) error { + var config elbv2.DescribeTargetGroupsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(TargetGroupTables...) + for { + output, err := c.svc.DescribeTargetGroups(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformTargetGroups(output.TargetGroups)) + c.log.Info("Fetched resources", zap.String("resource", "elbv2.target_groups"), zap.Int("count", len(output.TargetGroups))) + if aws.StringValue(output.NextMarker) == "" { + break + } + config.Marker = output.NextMarker + } + + return nil +} diff --git a/emr/client.go b/emr/client.go new file mode 100644 index 000000000..b69dd5c64 --- /dev/null +++ b/emr/client.go @@ -0,0 +1,41 @@ +package emr + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/emr" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *emr.EMR +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: emr.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "clusters": + return c.clusters(config) + default: + return fmt.Errorf("unsupported resource autoscaling.%s", resource) + } +} diff --git a/emr/clusters.go b/emr/clusters.go new file mode 100644 index 000000000..12f449b0d --- /dev/null +++ b/emr/clusters.go @@ -0,0 +1,94 @@ +package emr + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/emr" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Cluster struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + ClusterArn *string `neo:"unique"` + ClusterID *string + Name *string + NormalizedInstanceHours *int64 + OutpostArn *string + + StatusState *string + StatusChangeReasonCode *string + StatusChangeReasonMessage *string + + StatusCreationDateTime *time.Time + StatusEndDateTime *time.Time + StatusReadyDateTime *time.Time +} + +func (Cluster) TableName() string { + return "aws_emr_clusters" +} + +func (c *Client) transformCluster(value *emr.ClusterSummary) *Cluster { + res := Cluster{ + Region: c.region, + AccountID: c.accountID, + ClusterArn: value.ClusterArn, + ClusterID: value.Id, + Name: value.Name, + NormalizedInstanceHours: value.NormalizedInstanceHours, + OutpostArn: value.OutpostArn, + } + + if value.Status != nil { + res.StatusState = value.Status.State + if value.Status.StateChangeReason != nil { + res.StatusChangeReasonCode = value.Status.StateChangeReason.Code + res.StatusChangeReasonMessage = value.Status.StateChangeReason.Message + } + if value.Status.Timeline != nil { + res.StatusCreationDateTime = value.Status.Timeline.CreationDateTime + res.StatusEndDateTime = value.Status.Timeline.EndDateTime + res.StatusReadyDateTime = value.Status.Timeline.ReadyDateTime + } + } + + return &res +} + +func (c *Client) transformClusters(values []*emr.ClusterSummary) []*Cluster { + var tValues []*Cluster + for _, v := range values { + tValues = append(tValues, c.transformCluster(v)) + } + return tValues +} + +var ClusterTables = []interface{}{ + &Cluster{}, +} + +func (c *Client) clusters(gConfig interface{}) error { + var config emr.ListClustersInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(ClusterTables...) + for { + output, err := c.svc.ListClusters(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformClusters(output.Clusters)) + c.log.Info("Fetched resources", zap.String("resource", "emr.clusters"), zap.Int("count", len(output.Clusters))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/fsx/backups.go b/fsx/backups.go new file mode 100644 index 000000000..322ba9ee5 --- /dev/null +++ b/fsx/backups.go @@ -0,0 +1,126 @@ +package fsx + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/fsx" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Backup struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + BackupId *string + CreationTime *time.Time + + ActiveDirectoryId *string + ActiveDirectoryDomainName *string + + FailureDetailsMessage *string + + KmsKeyId *string + Lifecycle *string + ProgressPercent *int64 + ResourceARN *string `neo:"unique"` + Tags []*BackupTag `gorm:"constraint:OnDelete:CASCADE;"` + Type *string +} + +func (Backup) TableName() string { + return "aws_fsx_backups" +} + +type BackupTag struct { + ID uint `gorm:"primarykey"` + BackupID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + Key *string + Value *string +} + +func (BackupTag) TableName() string { + return "aws_fsx_backup_tags" +} + +func (c *Client) transformBackupTag(value *fsx.Tag) *BackupTag { + return &BackupTag{ + Region: c.region, + AccountID: c.accountID, + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformBackupTags(values []*fsx.Tag) []*BackupTag { + var tValues []*BackupTag + for _, v := range values { + tValues = append(tValues, c.transformBackupTag(v)) + } + return tValues +} + +func (c *Client) transformBackup(value *fsx.Backup) *Backup { + res := Backup{ + Region: c.region, + AccountID: c.accountID, + BackupId: value.BackupId, + CreationTime: value.CreationTime, + KmsKeyId: value.KmsKeyId, + Lifecycle: value.Lifecycle, + ProgressPercent: value.ProgressPercent, + ResourceARN: value.ResourceARN, + Tags: c.transformBackupTags(value.Tags), + Type: value.Type, + } + + if value.DirectoryInformation != nil { + res.ActiveDirectoryDomainName = value.DirectoryInformation.DomainName + res.ActiveDirectoryId = value.DirectoryInformation.ActiveDirectoryId + } + + if value.FailureDetails != nil { + res.FailureDetailsMessage = value.FailureDetails.Message + } + + return &res +} + +func (c *Client) transformBackups(values []*fsx.Backup) []*Backup { + var tValues []*Backup + for _, v := range values { + tValues = append(tValues, c.transformBackup(v)) + } + return tValues +} + +var BackupTables = []interface{}{ + &Backup{}, + &BackupTag{}, +} + +func (c *Client) backups(gConfig interface{}) error { + var config fsx.DescribeBackupsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(BackupTables...) + + for { + output, err := c.svc.DescribeBackups(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformBackups(output.Backups)) + c.log.Info("Fetched resources", zap.String("resource", "fsx.backups"), zap.Int("count", len(output.Backups))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/fsx/client.go b/fsx/client.go new file mode 100644 index 000000000..4a5a74492 --- /dev/null +++ b/fsx/client.go @@ -0,0 +1,41 @@ +package fsx + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/fsx" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *fsx.FSx +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: fsx.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "backups": + return c.backups(config) + default: + return fmt.Errorf("unsupported resource autoscaling.%s", resource) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..49a500e2b --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/cloudquery/cq-provider-aws + +go 1.15 + +require ( + github.com/aws/aws-sdk-go v1.37.5 + github.com/cloudquery/cloudquery v0.8.12 + github.com/gocarina/gocsv v0.0.0-20201208093247-67c824bc04d4 + github.com/mitchellh/mapstructure v1.3.3 + go.uber.org/zap v1.16.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..edd91435f --- /dev/null +++ b/go.sum @@ -0,0 +1,845 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v49.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.11.13/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.8/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.5/go.mod h1:ptW4D47I+eIUe/lulFLYTVfG4rAARZoXIe1vmTQ+ol8= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-lambda-go v1.22.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU= +github.com/aws/aws-sdk-go v1.35.0/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go v1.37.5 h1:9zJ1aXRk1gLSWEeaMXa7Hbv1pIM915T2tpaIJi0+mkA= +github.com/aws/aws-sdk-go v1.37.5/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudquery/cloudquery v0.8.11 h1:2IaOyrQONT9SoqSTzSbWYBoLmHJ2jlgRA1/Z/fHC/BA= +github.com/cloudquery/cloudquery v0.8.11/go.mod h1:xWWn0jk0h6869KsKe+wiDmHO23WRTpmJ0Fvoihl+vXo= +github.com/cloudquery/cloudquery v0.8.12 h1:9kC+V6/tUEtdjIZc0BiNpeqFM/URektpIRYfuA7Q/jc= +github.com/cloudquery/cloudquery v0.8.12/go.mod h1:1E/+NeLQ46k8qyC2OHZqcz75dR+lwaLPfzz/xuGm/N0= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzqk8QCaRC4os14xoKDdbHqqlJtJA0oc1ZAjg= +github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gocarina/gocsv v0.0.0-20201208093247-67c824bc04d4 h1:Q7s2AN3DhFJKOnzO0uTKLhJTfXTEcXcvw5ylf2BHJw4= +github.com/gocarina/gocsv v0.0.0-20201208093247-67c824bc04d4/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.15.0 h1:qMuK0wxsoW4D0ddCCYwPSTm4KQv1X1ke3WmPWZ0Mvsk= +github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-plugin v1.4.0 h1:b0O7rs5uiJ99Iu9HugEzsM67afboErkHUWddUSpUO3A= +github.com/hashicorp/go-plugin v1.4.0/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U= +github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.7.0 h1:pwjzcYyfmz/HQOQlENvG1OcDqauTGaqlVahq934F0/U= +github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7JMeA= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.5 h1:NUbEWPmCQZbMmYlTjVoNPhc0CfnYyz2bfUAh6A5ZVJM= +github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.5.0 h1:jzBqRk2HFG2CV4AIwgCI2PwTgm6UUoCAK2ofHHRirtc= +github.com/jackc/pgtype v1.5.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.9.0 h1:6STjDqppM2ROy5p1wNDcsC7zJTjSHeuCsguZmXyzx7c= +github.com/jackc/pgx/v4 v4.9.0/go.mod h1:MNGWmViCgqbZck9ujOOBN63gK9XVGILXWCvKLGKmnms= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jarcoal/httpmock v1.0.6/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA= +github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2/go.mod h1:TjQg8pa4iejrUrjiz0MCtMV38jdMNW4doKSiBrEvCQQ= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/neo4j/neo4j-go-driver/v4 v4.2.1-0.20201216094814-003230e3e54b h1:AhhyvktLyFzPDiVMTWdqQhzWUfWKVpoU8GcC67NOMS8= +github.com/neo4j/neo4j-go-driver/v4 v4.2.1-0.20201216094814-003230e3e54b/go.mod h1:4e45lVy4oHcgLEQQrGHcc4MbyCeEPIQ33DhXqxf9AT4= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/okta/okta-sdk-golang/v2 v2.2.1/go.mod h1:ZGx20R+s/DQV3L8fxFN2OrQEIe2hBpwM7/443wLyQpc= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/ory/dockertest/v3 v3.6.3/go.mod h1:EFLcVUOl8qCwp9NyDAcCDtq/QviLtYswW/VbWzUnTNE= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY= +golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201014231627-1610a49f37af/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d h1:92D1fum1bJLKSdr11OJ+54YeCMCGYIygTA7R/YZxH5M= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.0.2 h1:xm21Um8cR/Cg+nMwSrajf8aBUxOIC+WmH72ir/ByYR8= +gorm.io/driver/mysql v1.0.2/go.mod h1:T+Fv7Rq/8+lpS3X1KKVUbj8Y/SzbPa5esK9KpPAKXR8= +gorm.io/driver/postgres v1.0.2 h1:mB5JjD4QglbCTdMT1aZDxQzHr87XDK1qh0MKIU3P96g= +gorm.io/driver/postgres v1.0.2/go.mod h1:FvRSYfBI9jEp6ZSjlpS9qNcSjxwYxFc03UOTrHdvvYA= +gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc= +gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c= +gorm.io/driver/sqlserver v1.0.4 h1:V15fszi0XAo7fbx3/cF50ngshDSN4QT0MXpWTylyPTY= +gorm.io/driver/sqlserver v1.0.4/go.mod h1:ciEo5btfITTBCj9BkoUVDvgQbUdLWQNqdFY5OGuGnRg= +gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.2/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.9 h1:M3aIZKXAC1PtPVu9t3WGwkBTE1le5c2telz3I/qjRNg= +gorm.io/gorm v1.20.9/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= +k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/iam/attached_policies.go b/iam/attached_policies.go new file mode 100644 index 000000000..1daabb54f --- /dev/null +++ b/iam/attached_policies.go @@ -0,0 +1,31 @@ +package iam +import ( + "github.com/aws/aws-sdk-go/service/iam" +) + + +type UserAttachedPolicy struct { + ID uint `gorm:"primarykey"` + UserID uint `neo:"ignore"` + AccountID string `gorm:"-"` + PolicyArn *string + PolicyName *string +} + +func (UserAttachedPolicy) TableName() string { + return "aws_iam_user_attached_policies" +} + +func (c *Client) transformAttachedPolicies(values []*iam.AttachedPolicy) []*UserAttachedPolicy { + var tValues []*UserAttachedPolicy + for _, value := range values { + tValue := UserAttachedPolicy{ + AccountID: c.accountID, + PolicyArn: value.PolicyArn, + PolicyName: value.PolicyName, + } + tValues = append(tValues, &tValue) + } + return tValues +} + diff --git a/iam/client.go b/iam/client.go new file mode 100644 index 000000000..9ab06218a --- /dev/null +++ b/iam/client.go @@ -0,0 +1,49 @@ +package iam + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + svc *iam.IAM +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, _ string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + svc: iam.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "users": + return c.users(config) + case "groups": + return c.groups(config) + case "policies": + return c.policies(config) + case "roles": + return c.roles(config) + case "password_policies": + return c.passwordPolicies(config) + case "virtual_mfa_devices": + return c.virtualMFADevices(config) + default: + return fmt.Errorf("unsupported resource iam.%s", resource) + } +} diff --git a/iam/groups.go b/iam/groups.go new file mode 100644 index 000000000..ecd819713 --- /dev/null +++ b/iam/groups.go @@ -0,0 +1,114 @@ +package iam + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Group struct { + ID uint `gorm:"primarykey"` + AccountID string + Arn *string `neo:"unique"` + CreateDate *time.Time + GroupId *string + GroupName *string + Path *string + Policies []*GroupPolicy `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (Group) TableName() string { + return "aws_iam_groups" +} + +type GroupPolicy struct { + ID uint `gorm:"primarykey"` + GroupID uint `neo:"ignore"` + AccountID string `gorm:"-"` + PolicyArn *string + PolicyName *string +} + +func (GroupPolicy) TableName() string { + return "aws_iam_group_policies" +} + +func (c *Client) transformGroupPolicies(values []*iam.AttachedPolicy) []*GroupPolicy { + var tValues []*GroupPolicy + for _, value := range values { + tValue := GroupPolicy{ + AccountID: c.accountID, + PolicyArn: value.PolicyArn, + PolicyName: value.PolicyName, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformGroups(values []*iam.Group) ([]*Group, error) { + var tValues []*Group + for _, value := range values { + tValue := &Group{ + AccountID: c.accountID, + Arn: value.Arn, + CreateDate: value.CreateDate, + GroupId: value.GroupId, + GroupName: value.GroupName, + Path: value.Path, + } + + listAttachedUserPoliciesInput := iam.ListAttachedGroupPoliciesInput{ + GroupName: value.GroupName, + } + for { + outputAttachedPolicies, err := c.svc.ListAttachedGroupPolicies(&listAttachedUserPoliciesInput) + if err != nil { + return nil, err + } + tValue.Policies = append(tValue.Policies, c.transformGroupPolicies(outputAttachedPolicies.AttachedPolicies)...) + if outputAttachedPolicies.Marker == nil { + break + } + listAttachedUserPoliciesInput.Marker = outputAttachedPolicies.Marker + } + + tValues = append(tValues, tValue) + + } + return tValues, nil +} + +var GroupTables = []interface{}{ + &Group{}, + &GroupPolicy{}, +} + +func (c *Client) groups(gConfig interface{}) error { + var config iam.ListGroupsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("account_id", c.accountID).Delete(GroupTables...) + + for { + output, err := c.svc.ListGroups(&config) + if err != nil { + return err + } + tValues, err := c.transformGroups(output.Groups) + if err != nil { + return err + } + c.db.ChunkedCreate(tValues) + c.log.Info("Fetched resources", zap.String("resource", "iam.groups"), zap.Int("count", len(output.Groups))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/iam/password_policies.go b/iam/password_policies.go new file mode 100644 index 000000000..ae9f9ddb8 --- /dev/null +++ b/iam/password_policies.go @@ -0,0 +1,69 @@ +package iam + +import ( + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type PasswordPolicy struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + AllowUsersToChangePassword *bool + ExpirePasswords *bool + HardExpiry *bool + MaxPasswordAge *int64 + MinimumPasswordLength *int64 + PasswordReusePrevention *int64 + RequireLowercaseCharacters *bool + RequireNumbers *bool + RequireSymbols *bool + RequireUppercaseCharacters *bool +} + +func (PasswordPolicy) TableName() string { + return "aws_iam_password_policies" +} + +func (c *Client) transformPasswordPolicy(value *iam.PasswordPolicy) *PasswordPolicy { + return &PasswordPolicy{ + AccountID: c.accountID, + AllowUsersToChangePassword: value.AllowUsersToChangePassword, + ExpirePasswords: value.ExpirePasswords, + HardExpiry: value.HardExpiry, + MaxPasswordAge: value.MaxPasswordAge, + MinimumPasswordLength: value.MinimumPasswordLength, + PasswordReusePrevention: value.PasswordReusePrevention, + RequireLowercaseCharacters: value.RequireLowercaseCharacters, + RequireNumbers: value.RequireNumbers, + RequireSymbols: value.RequireSymbols, + RequireUppercaseCharacters: value.RequireUppercaseCharacters, + } +} + +var PasswordPolicyTables = []interface{}{ + &PasswordPolicy{}, +} + +func (c *Client) passwordPolicies(gConfig interface{}) error { + var config iam.GetAccountPasswordPolicyInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + + output, err := c.svc.GetAccountPasswordPolicy(&config) + if err != nil { + if err.(awserr.Error).Code() == "NoSuchEntity" { + c.log.Info("Fetched PasswordPolicy", zap.Int("count", 0)) + return nil + } + return err + } + c.db.Where("account_id", c.accountID).Delete(PasswordPolicyTables...) + c.db.InsertOne(c.transformPasswordPolicy(output.PasswordPolicy)) + c.log.Info("Fetched resources", zap.String("resource", "iam.password_policies"), zap.Int("count", 1)) + + return nil +} diff --git a/iam/policies.go b/iam/policies.go new file mode 100644 index 000000000..5b03b5cfe --- /dev/null +++ b/iam/policies.go @@ -0,0 +1,141 @@ +package iam + +import ( + "net/url" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type Policy struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Arn *string `neo:"unique"` + AttachmentCount *int64 + CreateDate *time.Time + DefaultVersionId *string + Description *string + IsAttachable *bool + Path *string + PermissionsBoundaryUsageCount *int64 + PolicyId *string `neo:"unique"` + PolicyName *string + UpdateDate *time.Time + PolicyVersions []*PolicyVersion `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (Policy) TableName() string { + return "aws_iam_policies" +} + +type PolicyVersion struct { + ID uint `gorm:"primarykey` + PolicyID uint `neo:"ignore"` + AccountID string `gorm:"-"` + VersionID *string + CreateDate *time.Time + Document *string + IsDefaultVersion *bool +} + +func (PolicyVersion) TableName() string { + return "aws_iam_policy_versions" +} + +func (c *Client) transformPolicyVersionList(values []*iam.PolicyVersion) ([]*PolicyVersion, error) { + var tValues []*PolicyVersion + + for _, value := range values { + var decodedDocument string = "" + var err error = nil + if value.Document != nil { + decodedDocument, err = url.QueryUnescape(*value.Document) + if err != nil { + return nil, err + } + } + + tValues = append(tValues, &PolicyVersion{ + AccountID: c.accountID, + VersionID: value.VersionId, + CreateDate: value.CreateDate, + Document: &decodedDocument, + IsDefaultVersion: value.IsDefaultVersion, + }) + } + return tValues, nil +} + +func (c *Client) transformPolicy(value *iam.ManagedPolicyDetail) (*Policy, error) { + tValue := &Policy{ + AccountID: c.accountID, + Arn: value.Arn, + AttachmentCount: value.AttachmentCount, + CreateDate: value.CreateDate, + DefaultVersionId: value.DefaultVersionId, + Description: value.Description, + IsAttachable: value.IsAttachable, + Path: value.Path, + PermissionsBoundaryUsageCount: value.PermissionsBoundaryUsageCount, + PolicyId: value.PolicyId, + PolicyName: value.PolicyName, + UpdateDate: value.UpdateDate, + } + + policyVersions, err := c.transformPolicyVersionList(value.PolicyVersionList) + if err != nil { + return nil, err + } + tValue.PolicyVersions = append(tValue.PolicyVersions, policyVersions...) + + return tValue, nil +} + +func (c *Client) transformPolicies(values []*iam.ManagedPolicyDetail) ([]*Policy, error) { + var tValues []*Policy + for _, v := range values { + policy, err := c.transformPolicy(v) + if err != nil { + return nil, err + } + tValues = append(tValues, policy) + } + return tValues, nil +} + +var PolicyTables = []interface{}{ + &Policy{}, + &PolicyVersion{}, +} + +func (c *Client) policies(gConfig interface{}) error { + var config iam.GetAccountAuthorizationDetailsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + + c.db.Where("account_id", c.accountID).Delete(PolicyTables...) + for { + output, err := c.svc.GetAccountAuthorizationDetails(&config) + if err != nil { + return err + } + + tValues, err := c.transformPolicies(output.Policies) + if err != nil { + return err + } + c.db.ChunkedCreate(tValues) + + c.log.Info("Fetched resources", zap.String("resource", "iam.policies"), zap.Int("count", len(output.Policies))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/iam/roles.go b/iam/roles.go new file mode 100644 index 000000000..635cf7a0d --- /dev/null +++ b/iam/roles.go @@ -0,0 +1,188 @@ +package iam + +import ( + "net/url" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type Role struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Arn *string `neo:"unique"` + AssumeRolePolicyDocument *string + CreateDate *time.Time + Description *string + MaxSessionDuration *int64 + Path *string + + PermissionsBoundaryArn *string + PermissionsBoundaryType *string + + RoleId *string `neo:"unique"` + LastUsedDate *time.Time + LastUsedRegion *string + + RoleName *string + Tags []*RoleTag `gorm:"constraint:OnDelete:CASCADE;"` + Policies []*RolePolicy `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (Role) TableName() string { + return "aws_iam_roles" +} + +type RoleTag struct { + ID uint `gorm:"primarykey"` + RoleID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Key *string + Value *string +} + +func (RoleTag) TableName() string { + return "aws_iam_role_tags" +} + +type RolePolicy struct { + ID uint `gorm:"primarykey"` + RoleID uint `neo:"ignore"` + AccountID string `gorm:"-"` + PolicyArn *string + PolicyName *string +} + +func (RolePolicy) TableName() string { + return "aws_iam_role_policies" +} + +func (c *Client) transformRolesPolicies(values []*iam.AttachedPolicy) []*RolePolicy { + var tValues []*RolePolicy + for _, value := range values { + tValue := RolePolicy{ + AccountID: c.accountID, + PolicyArn: value.PolicyArn, + PolicyName: value.PolicyName, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformRoleTags(values []*iam.Tag) []*RoleTag { + var tValues []*RoleTag + for _, value := range values { + tValues = append(tValues, &RoleTag{ + AccountID: c.accountID, + Key: value.Key, + Value: value.Value, + }) + } + return tValues +} + +func (c *Client) transformRoles(values []*iam.Role) ([]*Role, error) { + var tValues []*Role + for _, value := range values { + var decodedDocument string = "" + var err error = nil + if value.AssumeRolePolicyDocument != nil { + decodedDocument, err = url.QueryUnescape(*value.AssumeRolePolicyDocument) + if err != nil { + return nil, err + } + } + + tValue := Role{ + AccountID: c.accountID, + Arn: value.Arn, + AssumeRolePolicyDocument: &decodedDocument, + CreateDate: value.CreateDate, + Description: value.Description, + MaxSessionDuration: value.MaxSessionDuration, + Path: value.Path, + RoleId: value.RoleId, + RoleName: value.RoleName, + } + + if value.PermissionsBoundary != nil { + tValue.PermissionsBoundaryArn = value.PermissionsBoundary.PermissionsBoundaryArn + tValue.PermissionsBoundaryType = value.PermissionsBoundary.PermissionsBoundaryType + } + + if value.RoleLastUsed != nil { + tValue.LastUsedDate = value.RoleLastUsed.LastUsedDate + tValue.LastUsedRegion = value.RoleLastUsed.Region + } + + listAttachedRolePoliciesInput := iam.ListAttachedRolePoliciesInput{ + RoleName: value.RoleName, + } + for { + outputAttachedPolicies, err := c.svc.ListAttachedRolePolicies(&listAttachedRolePoliciesInput) + if err != nil { + return nil, err + } + tValue.Policies = append(tValue.Policies, c.transformRolesPolicies(outputAttachedPolicies.AttachedPolicies)...) + if outputAttachedPolicies.Marker == nil { + break + } + listAttachedRolePoliciesInput.Marker = outputAttachedPolicies.Marker + } + + listRoleTagsInput := iam.ListRoleTagsInput{ + RoleName: value.RoleName, + } + for { + outputRoleTags, err := c.svc.ListRoleTags(&listRoleTagsInput) + if err != nil { + return nil, err + } + tValue.Tags = append(tValue.Tags, c.transformRoleTags(outputRoleTags.Tags)...) + if outputRoleTags.Marker == nil { + break + } + listRoleTagsInput.Marker = outputRoleTags.Marker + } + + tValues = append(tValues, &tValue) + } + return tValues, nil +} + +var RoleTables = []interface{}{ + &Role{}, + &RoleTag{}, + &RolePolicy{}, +} + +func (c *Client) roles(gConfig interface{}) error { + var config iam.ListRolesInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("account_id", c.accountID).Delete(RoleTables...) + + for { + output, err := c.svc.ListRoles(&config) + if err != nil { + return err + } + tValues, err := c.transformRoles(output.Roles) + if err != nil { + return err + } + c.db.ChunkedCreate(tValues) + c.log.Info("Fetched resources", zap.String("resource", "iam.roles"), zap.Int("count", len(output.Roles))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/iam/users.go b/iam/users.go new file mode 100644 index 000000000..a0a7a3361 --- /dev/null +++ b/iam/users.go @@ -0,0 +1,343 @@ +package iam + +import ( + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/gocarina/gocsv" + "go.uber.org/zap" +) + +type User struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Arn *string `neo:"unique"` + PasswordEnabled *bool + PasswordLastUsed *time.Time + PasswordLastChanged *time.Time + PasswordNextRotation *time.Time + MFAActive *bool + CreateDate *time.Time + Path *string + + PermissionsBoundaryArn *string + PermissionsBoundaryType *string + + Tags []*UserTag `gorm:"constraint:OnDelete:CASCADE;"` + UserId *string + UserName *string `csv:"user"` + AccessKeys []*UserAccessKey `gorm:"constraint:OnDelete:CASCADE;"` + AttachedPolicies []*UserAttachedPolicy `gorm:"constraint:OnDelete:CASCADE;"` + Groups []*UserGroup `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (User) TableName() string { + return "aws_iam_users" +} + +type UserGroup struct { + ID uint `gorm:"primarykey"` + UserID uint `neo:"ignore"` + AccountID string `gorm:"-"` + + Arn *string + CreateDate *time.Time + GroupId *string + GroupName *string + Path *string +} + +func (UserGroup) TableName() string { + return "aws_iam_user_groups" +} + +type UserAccessKey struct { + ID uint `gorm:"primarykey"` + UserID uint `neo:"ignore"` + AccountID string `gorm:"-"` + + AccessKeyId *string + CreateDate *time.Time + Status *string + LastUsed *time.Time + LastRotated *time.Time + LastUsedServiceName *string +} + +func (UserAccessKey) TableName() string { + return "aws_iam_user_access_keys" +} + +type UserTag struct { + ID uint `gorm:"primarykey"` + UserID uint `neo:"ignore"` + AccountID string `gorm:"-"` + + Key *string + Value *string +} + +func (UserTag) TableName() string { + return "aws_iam_user_tags" +} + +func (c *Client) transformUserGroups(values []*iam.Group) []*UserGroup { + var tValues []*UserGroup + for _, value := range values { + tValues = append(tValues, &UserGroup{ + AccountID: c.accountID, + Arn: value.Arn, + CreateDate: value.CreateDate, + GroupId: value.GroupId, + GroupName: value.GroupName, + Path: value.Path, + }) + } + return tValues +} + +func (c *Client) transformAccessKey(value *iam.AccessKeyMetadata) (*UserAccessKey, error) { + output, err := c.svc.GetAccessKeyLastUsed(&iam.GetAccessKeyLastUsedInput{AccessKeyId: value.AccessKeyId}) + if err != nil { + return nil, err + } + + res := UserAccessKey{ + AccessKeyId: value.AccessKeyId, + CreateDate: value.CreateDate, + Status: value.Status, + LastUsed: output.AccessKeyLastUsed.LastUsedDate, + LastUsedServiceName: output.AccessKeyLastUsed.ServiceName, + } + if output.AccessKeyLastUsed != nil { + res.LastUsed = output.AccessKeyLastUsed.LastUsedDate + res.LastUsedServiceName = output.AccessKeyLastUsed.ServiceName + } + return &res, nil +} + +func (c *Client) transformAccessKeys(values []*iam.AccessKeyMetadata) ([]*UserAccessKey, error) { + var tValues []*UserAccessKey + for _, v := range values { + tValue, err := c.transformAccessKey(v) + if err != nil { + return nil, err + } + tValues = append(tValues, tValue) + } + return tValues, nil +} + +func (c *Client) transformUserTag(value *iam.Tag) *UserTag { + return &UserTag{ + Key: value.Key, + Value: value.Value, + } +} + +func (c *Client) transformUserTags(values []*iam.Tag) []*UserTag { + var tValues []*UserTag + for _, v := range values { + tValues = append(tValues, c.transformUserTag(v)) + } + return tValues +} + +type ReportUser struct { + User string `csv:"user"` + ARN string `csv:"arn"` + UserCreationTime time.Time `csv:"user_creation_time"` + PasswordEnabled string `csv:"password_enabled"` + PasswordLastUsed string `csv:"password_last_used"` + PasswordLastChanged string `csv:"password_last_changed"` + PasswordNextRotation string `csv:"password_next_rotation"` + MFAActive bool `csv:"mfa_active"` + AccessKey1Active bool `csv:"access_key_1_active"` + AccessKey2Active bool `csv:"access_key_2_active"` + AccessKey1LastRotated string `csv:"access_key_1_last_rotated"` + AccessKey2LastRotated string `csv:"access_key_2_last_rotated"` +} + +func (c *Client) transformReportUser(reportUser *ReportUser) (*User, error) { + //var err error + location, err := time.LoadLocation("UTC") + if err != nil { + panic(err) + } + + createDate := reportUser.UserCreationTime.In(location) + res := User{ + AccountID: c.accountID, + Arn: &reportUser.ARN, + MFAActive: &reportUser.MFAActive, + CreateDate: &createDate, + UserName: &reportUser.User, + } + + if reportUser.User != "" { + output, err := c.svc.GetUser(&iam.GetUserInput{UserName: &reportUser.User}) + if err != nil { + return nil, err + } + res.Path = output.User.Path + if output.User.PermissionsBoundary != nil { + res.PermissionsBoundaryType = output.User.PermissionsBoundary.PermissionsBoundaryType + res.PermissionsBoundaryArn = output.User.PermissionsBoundary.PermissionsBoundaryArn + } + res.UserId = output.User.UserId + + outputAccessKeys, err := c.svc.ListAccessKeys(&iam.ListAccessKeysInput{ + UserName: &reportUser.User, + }) + if err != nil { + return nil, err + } + res.AccessKeys, err = c.transformAccessKeys(outputAccessKeys.AccessKeyMetadata) + if err != nil { + return nil, err + } + + listAttachedUserPoliciesInput := iam.ListAttachedUserPoliciesInput{ + UserName: &reportUser.User, + } + for { + outputAttachedPolicies, err := c.svc.ListAttachedUserPolicies(&listAttachedUserPoliciesInput) + if err != nil { + return nil, err + } + res.AttachedPolicies = append(res.AttachedPolicies, c.transformAttachedPolicies(outputAttachedPolicies.AttachedPolicies)...) + if outputAttachedPolicies.Marker == nil { + break + } + listAttachedUserPoliciesInput.Marker = outputAttachedPolicies.Marker + } + + listGroupsForUserInput := iam.ListGroupsForUserInput{ + UserName: &reportUser.User, + } + for { + outputListGroupsForUsers, err := c.svc.ListGroupsForUser(&listGroupsForUserInput) + if err != nil { + return nil, err + } + res.Groups = append(res.Groups, c.transformUserGroups(outputListGroupsForUsers.Groups)...) + if outputListGroupsForUsers.Marker == nil { + break + } + listGroupsForUserInput.Marker = outputListGroupsForUsers.Marker + } + + listUserTagsInput := iam.ListUserTagsInput{ + UserName: &reportUser.User, + } + for { + outputUserTags, err := c.svc.ListUserTags(&listUserTagsInput) + if err != nil { + return nil, err + } + res.Tags = append(res.Tags, c.transformUserTags(outputUserTags.Tags)...) + if outputUserTags.Marker == nil { + break + } + listUserTagsInput.Marker = outputUserTags.Marker + } + + } + + switch strings.ToLower(reportUser.PasswordEnabled) { + case "false": + passwordEnabled := false + res.PasswordEnabled = &passwordEnabled + case "true": + passwordEnabled := true + res.PasswordEnabled = &passwordEnabled + } + + passwordLastUsed, err := time.ParseInLocation(time.RFC3339, reportUser.PasswordLastUsed, location) + if err == nil { + res.PasswordLastUsed = &passwordLastUsed + } + passwordLastChanged, err := time.ParseInLocation(time.RFC3339, reportUser.PasswordLastChanged, location) + if err == nil { + res.PasswordLastChanged = &passwordLastChanged + } + + passwordNextRotation, err := time.ParseInLocation(time.RFC3339, reportUser.PasswordNextRotation, location) + if err == nil { + res.PasswordNextRotation = &passwordNextRotation + } + + lastRotated1, err := time.ParseInLocation(time.RFC3339, reportUser.AccessKey1LastRotated, location) + if err == nil && len(res.AccessKeys) > 0 { + res.AccessKeys[0].LastRotated = &lastRotated1 + } + + lastRotated2, err := time.ParseInLocation(time.RFC3339, reportUser.AccessKey2LastRotated, location) + if err == nil && len(res.AccessKeys) > 1 { + res.AccessKeys[1].LastRotated = &lastRotated2 + } + + return &res, nil +} + +func (c *Client) transformReportUsers(values []*ReportUser) ([]*User, error) { + var tValues []*User + for _, v := range values { + tValue, err := c.transformReportUser(v) + if err != nil { + return nil, err + } + tValues = append(tValues, tValue) + } + return tValues, nil +} + +var UserTables = []interface{}{ + &User{}, + &UserAccessKey{}, + &UserTag{}, + + &UserAttachedPolicy{}, + &UserGroup{}, +} + +func (c *Client) users(_ interface{}) error { + var err error + var reportOutput *iam.GetCredentialReportOutput + + c.db.Where("account_id", c.accountID).Delete(UserTables...) + for { + reportOutput, err = c.svc.GetCredentialReport(&iam.GetCredentialReportInput{}) + if err != nil { + if err.(awserr.Error).Code() != iam.ErrCodeCredentialReportNotPresentException || + err.(awserr.Error).Code() != iam.ErrCodeCredentialReportExpiredException { + _, err := c.svc.GenerateCredentialReport(&iam.GenerateCredentialReportInput{}) + if err != nil { + return err + } + } else if err.(awserr.Error).Code() != iam.ErrCodeCredentialReportNotReadyException { + c.log.Info("Waiting for credential report to be generated", zap.String("resource", "iam.users")) + time.Sleep(2 * time.Second) + } else { + return err + } + } else { + break + } + } + var users []*ReportUser + err = gocsv.UnmarshalBytes(reportOutput.Content, &users) + if err != nil { + return err + } + + tValues, err := c.transformReportUsers(users) + if err != nil { + return err + } + c.db.ChunkedCreate(tValues) + c.log.Info("Fetched resources", zap.String("resource", "iam.users"), zap.Int("count", len(users))) + return nil +} diff --git a/iam/virtual_mfa_devices.go b/iam/virtual_mfa_devices.go new file mode 100644 index 000000000..1069c41c4 --- /dev/null +++ b/iam/virtual_mfa_devices.go @@ -0,0 +1,63 @@ +package iam + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type VirtualMFADevice struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + SerialNumber *string `neo:"unique"` + UserARN *string + EnableDate *time.Time +} + +func (VirtualMFADevice) TableName() string { + return "aws_iam_virtual_mfa_devices" +} + +var VirtualMFADeviceTables = []interface{}{ + &VirtualMFADevice{}, +} + +func (c *Client) transformMFADevices(values []*iam.VirtualMFADevice) []*VirtualMFADevice { + var tValues []*VirtualMFADevice + for _, v := range values { + tValue := VirtualMFADevice{ + SerialNumber: v.SerialNumber, + EnableDate: v.EnableDate, + } + if v.User != nil { + tValue.UserARN = v.User.Arn + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) virtualMFADevices(gConfig interface{}) error { + var config iam.ListVirtualMFADevicesInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("account_id", c.accountID).Delete(VirtualMFADeviceTables...) + + for { + output, err := c.svc.ListVirtualMFADevices(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformMFADevices(output.VirtualMFADevices)) + c.log.Info("Fetched resources", zap.String("resource", "iam.virtual_mfa_devices"), zap.Int("count", len(output.VirtualMFADevices))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/kms/client.go b/kms/client.go new file mode 100644 index 000000000..33deec566 --- /dev/null +++ b/kms/client.go @@ -0,0 +1,41 @@ +package kms + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *kms.KMS +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: kms.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "keys": + return c.keys(config) + default: + return fmt.Errorf("unsupported resource kms.%s", resource) + } +} diff --git a/kms/keys.go b/kms/keys.go new file mode 100644 index 000000000..152e3c489 --- /dev/null +++ b/kms/keys.go @@ -0,0 +1,170 @@ +package kms + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Key struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + + Arn *string `neo:"unique"` + KeyId *string + RotationEnabled *bool + CloudHsmClusterId *string + CreationDate *time.Time + CustomKeyStoreId *string + CustomerMasterKeySpec *string + DeletionDate *time.Time + Description *string + Enabled *bool + EncryptionAlgorithms []*KeyEncryptionAlgorithm `gorm:"constraint:OnDelete:CASCADE;"` + ExpirationModel *string + Manager *string + KeyState *string + KeyUsage *string + Origin *string + SigningAlgorithms []*KeySigningAlgorithm `gorm:"constraint:OnDelete:CASCADE;"` + ValidTo *time.Time +} + +func (Key) TableName() string { + return "aws_kms_keys" +} + +type KeyEncryptionAlgorithm struct { + ID uint `gorm:"primarykey"` + KeyID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + name string +} + +func (KeyEncryptionAlgorithm) TableName() string { + return "aws_kms_key_encryption_algorithms" +} + +type KeySigningAlgorithm struct { + ID uint `gorm:"primarykey"` + KeyID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + name string +} + +func (KeySigningAlgorithm) TableName() string { + return "aws_kms_key_signing_algorithms" +} + +func (c *Client) transformKeyListEntry(value *kms.KeyListEntry) (*Key, error) { + output, err := c.svc.DescribeKey(&kms.DescribeKeyInput{ + KeyId: value.KeyId, + }) + if err != nil { + return nil, err + } + var outputKeyRotation *kms.GetKeyRotationStatusOutput + if aws.StringValue(output.KeyMetadata.Origin) != "EXTERNAL" { + outputKeyRotation, err = c.svc.GetKeyRotationStatus(&kms.GetKeyRotationStatusInput{ + KeyId: value.KeyId, + }) + if err != nil { + return nil, err + } + } + + res := Key{ + Region: c.region, + AccountID: c.accountID, + Arn: value.KeyArn, + KeyId: value.KeyId, + CloudHsmClusterId: output.KeyMetadata.CloudHsmClusterId, + CreationDate: output.KeyMetadata.CreationDate, + CustomKeyStoreId: output.KeyMetadata.CustomKeyStoreId, + CustomerMasterKeySpec: output.KeyMetadata.CustomerMasterKeySpec, + DeletionDate: output.KeyMetadata.DeletionDate, + Description: output.KeyMetadata.Description, + Enabled: output.KeyMetadata.Enabled, + ExpirationModel: output.KeyMetadata.ExpirationModel, + Manager: output.KeyMetadata.KeyManager, + KeyState: output.KeyMetadata.KeyState, + KeyUsage: output.KeyMetadata.KeyUsage, + Origin: output.KeyMetadata.Origin, + ValidTo: output.KeyMetadata.ValidTo, + } + + if outputKeyRotation != nil { + res.RotationEnabled = outputKeyRotation.KeyRotationEnabled + } + + for _, algorithm := range output.KeyMetadata.EncryptionAlgorithms { + res.EncryptionAlgorithms = append(res.EncryptionAlgorithms, &KeyEncryptionAlgorithm{ + Region: c.region, + AccountID: c.accountID, + name: aws.StringValue(algorithm), + }) + } + + for _, algorithm := range output.KeyMetadata.SigningAlgorithms { + res.SigningAlgorithms = append(res.SigningAlgorithms, &KeySigningAlgorithm{ + Region: c.region, + AccountID: c.accountID, + name: aws.StringValue(algorithm), + }) + } + + return &res, nil +} + +func (c *Client) transformKeyListEntrys(values []*kms.KeyListEntry) ([]*Key, error) { + var tValues []*Key + for _, v := range values { + tValue, err := c.transformKeyListEntry(v) + if err != nil { + return nil, err + } + tValues = append(tValues, tValue) + } + return tValues, nil +} + +var KeyTables = []interface{}{ + &Key{}, + &KeySigningAlgorithm{}, + &KeyEncryptionAlgorithm{}, +} + +func (c *Client) keys(gConfig interface{}) error { + var config kms.ListKeysInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(KeyTables...) + + for { + output, err := c.svc.ListKeys(&config) + if err != nil { + return err + } + tValues, err := c.transformKeyListEntrys(output.Keys) + if err != nil { + return err + } + c.db.ChunkedCreate(tValues) + c.log.Info("Fetched resources", zap.String("resource", "kms.keys"), zap.Int("count", len(output.Keys))) + if aws.StringValue(output.NextMarker) == "" { + break + } + config.Marker = output.NextMarker + } + return nil +} diff --git a/organizations/accounts.go b/organizations/accounts.go new file mode 100644 index 000000000..3cc2f3262 --- /dev/null +++ b/organizations/accounts.go @@ -0,0 +1,76 @@ +package organizations + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/organizations" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type Account struct { + ID uint `gorm:"primarykey"` + CallerAccountID string + AccountID *string `neo:"unique"` + Arn *string `neo:"unique" gorm:"unique"` + Email *string + JoinedMethod *string + JoinedTimestamp *time.Time + Name *string + Status *string +} + +func (Account) TableName() string { + return "aws_organizations_accounts" +} + +func (c *Client) transformAccounts(values []*organizations.Account) ([]*Account, error) { + var tValues []*Account + for _, value := range values { + tValues = append(tValues, &Account{ + CallerAccountID: c.accountID, + AccountID: value.Id, + Arn: value.Arn, + Email: value.Email, + JoinedMethod: value.JoinedMethod, + JoinedTimestamp: value.JoinedTimestamp, + Name: value.Name, + Status: value.Status, + }) + } + return tValues, nil +} + +var AccountTables = []interface{}{ + &Account{}, +} + +func (c *Client) accounts(gConfig interface{}) error { + var config organizations.ListAccountsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + + // TODO: This doesn't work, since the account ids are not coming from the client but from the sdk call + c.db.Where("caller_account_id", c.accountID).Delete(AccountTables...) + + for { + output, err := c.svc.ListAccounts(&config) + if err != nil { + return err + } + tValues, err := c.transformAccounts(output.Accounts) + if err != nil { + return err + } + c.db.ChunkedUpsert(tValues) + c.log.Info("Fetched resources", zap.String("resource", "organizations.accounts"), zap.Int("count", len(output.Accounts))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + return nil +} diff --git a/organizations/client.go b/organizations/client.go new file mode 100644 index 000000000..bfa777bbe --- /dev/null +++ b/organizations/client.go @@ -0,0 +1,40 @@ +package organizations + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/organizations" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + svc *organizations.Organizations +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, _ string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + svc: organizations.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "accounts": + return c.accounts(config) + default: + return fmt.Errorf("unsupported resource organizations.%s", resource) + } +} diff --git a/provider.go b/provider.go new file mode 100644 index 000000000..db54a65b1 --- /dev/null +++ b/provider.go @@ -0,0 +1,339 @@ +package main + +import ( + "fmt" + "github.com/cloudquery/cloudquery/sdk" + "log" + "strings" + "sync" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/autoscaling" + "github.com/cloudquery/cloudquery/providers/aws/cloudtrail" + "github.com/cloudquery/cloudquery/providers/aws/cloudwatch" + "github.com/cloudquery/cloudquery/providers/aws/cloudwatchlogs" + "github.com/cloudquery/cloudquery/providers/aws/directconnect" + "github.com/cloudquery/cloudquery/providers/aws/ec2" + "github.com/cloudquery/cloudquery/providers/aws/ecr" + "github.com/cloudquery/cloudquery/providers/aws/ecs" + "github.com/cloudquery/cloudquery/providers/aws/efs" + "github.com/cloudquery/cloudquery/providers/aws/elasticbeanstalk" + "github.com/cloudquery/cloudquery/providers/aws/elbv2" + "github.com/cloudquery/cloudquery/providers/aws/emr" + "github.com/cloudquery/cloudquery/providers/aws/fsx" + "github.com/cloudquery/cloudquery/providers/aws/iam" + "github.com/cloudquery/cloudquery/providers/aws/kms" + "github.com/cloudquery/cloudquery/providers/aws/organizations" + "github.com/cloudquery/cloudquery/providers/aws/rds" + "github.com/cloudquery/cloudquery/providers/aws/redshift" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "github.com/cloudquery/cloudquery/providers/aws/s3" + "github.com/cloudquery/cloudquery/providers/aws/sns" + "go.uber.org/zap" + "gopkg.in/yaml.v3" +) + +type Provider struct { + session *session.Session + cred *credentials.Credentials + region string + db *database.Database + config Config + accountID string + resourceClients map[string]resource.ClientInterface + log *zap.Logger + logLevel aws.LogLevelType +} + +type Account struct { + ID string + RoleARN string +} + +type Config struct { + Regions []string + Accounts []Account + LogLevel *string + MaxRetries *int + Resources []struct { + Name string + Other map[string]interface{} `yaml:",inline"` + } +} + +var globalCollectedResources = map[string]bool{} + +type ServiceNewFunction func(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, accountID string, region string) resource.ClientInterface + +var globalServices = map[string]ServiceNewFunction{ + "iam": iam.NewClient, + "s3": s3.NewClient, + "organizations": organizations.NewClient, +} + +var regionalServices = map[string]ServiceNewFunction{ + "autoscaling": autoscaling.NewClient, + "cloudtrail": cloudtrail.NewClient, + "cloudwatchlogs": cloudwatchlogs.NewClient, + "cloudwatch": cloudwatch.NewClient, + "directconnect": directconnect.NewClient, + "ec2": ec2.NewClient, + "ecr": ecr.NewClient, + "ecs": ecs.NewClient, + "efs": efs.NewClient, + "elasticbeanstalk": elasticbeanstalk.NewClient, + "elbv2": elbv2.NewClient, + "emr": emr.NewClient, + "fsx": fsx.NewClient, + "kms": kms.NewClient, + "rds": rds.NewClient, + "redshift": redshift.NewClient, + "sns": sns.NewClient, +} + +var tablesArr = [][]interface{}{ + autoscaling.LaunchConfigurationTables, + cloudtrail.TrailTables, + cloudwatchlogs.MetricFilterTables, + cloudwatch.MetricAlarmTables, + directconnect.GatewayTables, + ec2.ByoipCidrTables, + ec2.CustomerGatewayTables, + ec2.FlowLogsTables, + ec2.ImageTables, + ec2.InstanceTables, + ec2.InternetGatewayTables, + ec2.NatGatewayTables, + ec2.NetworkAclTables, + ec2.RouteTableTables, + ec2.SecurityGroupTables, + ec2.SubnetTables, + ec2.VPCPeeringConnectionTables, + ec2.VPCTables, + ecr.ImageTables, + ecs.ClusterTables, + efs.FileSystemTables, + elasticbeanstalk.EnvironmentTables, + elbv2.LoadBalancerTables, + elbv2.TargetGroupTables, + emr.ClusterTables, + fsx.BackupTables, + iam.GroupTables, + iam.PasswordPolicyTables, + iam.PolicyTables, + iam.RoleTables, + iam.UserTables, + iam.VirtualMFADeviceTables, + kms.KeyTables, + organizations.AccountTables, + rds.ClusterTables, + rds.CertificateTables, + rds.DBSubnetGroupTables, + redshift.ClusterTables, + redshift.ClusterSubnetGroupTables, + s3.BucketTables, + sns.SubscriptionTables, +} + +func (p *Provider) Init(driver string, dsn string, verbose bool) error { + var err error + p.db, err = database.Open(driver, dsn) + if err != nil { + return err + } + + zapLogger, err := sdk.NewLogger(verbose) + p.log = zapLogger + p.resourceClients = map[string]resource.ClientInterface{} + p.log.Info("Creating tables if needed") + for _, tables := range tablesArr { + err := p.db.AutoMigrate(tables...) + if err != nil { + return err + } + } + return nil +} + +func (p *Provider) GenConfig() (string, error) { + return configYaml, nil +} + +func (p *Provider) parseLogLevel() { + if p.config.LogLevel == nil { + return + } + switch *p.config.LogLevel { + case "debug", "debug_with_signing": + p.logLevel = aws.LogDebug + case "debug_with_http_body": + p.logLevel = aws.LogDebugWithSigning + case "debug_with_request_retries": + p.logLevel = aws.LogDebugWithRequestRetries + case "debug_with_request_error": + p.logLevel = aws.LogDebugWithRequestErrors + case "debug_with_event_stream_body": + p.logLevel = aws.LogDebugWithEventStreamBody + default: + log.Fatalf("unknown log_level %s", *p.config.LogLevel) + } +} + +func (p *Provider) Fetch(data []byte) error { + err := yaml.Unmarshal(data, &p.config) + if err != nil { + return err + } + + if len(p.config.Resources) == 0 { + p.log.Info("no resources specified. See available resources: see: https://docs.cloudquery.io/aws/tables-reference") + return nil + } + regions := p.config.Regions + if len(regions) == 0 { + resolver := endpoints.DefaultResolver() + partitions := resolver.(endpoints.EnumPartitions).Partitions() + for _, p := range partitions { + if p.ID() == "aws" { + for id, _ := range p.Regions() { + regions = append(regions, id) + } + } + } + p.log.Info(fmt.Sprintf("No regions specified in config.yml. Assuming all %d regions", len(regions))) + } + + if len(p.config.Accounts) == 0 { + p.config.Accounts = append(p.config.Accounts, Account{ + ID: "default", + RoleARN: "default", + }) + } + + p.parseLogLevel() + + for _, account := range p.config.Accounts { + + if account.ID != "default" && account.RoleARN != "" { + // assume role if specified (SDK takes it from default or env var: AWS_PROFILE) + p.session, err = session.NewSession() + cred := stscreds.NewCredentials(p.session, account.RoleARN) + p.session, err = session.NewSession(&aws.Config{ + Credentials: cred, + }) + if err != nil { + return err + } + } else if account.ID != "default" { + p.session, err = session.NewSession(&aws.Config{Credentials: credentials.NewSharedCredentials("", account.ID)}) + } else { + p.session, err = session.NewSession() + } + if err != nil { + return err + } + + for _, region := range regions { + p.region = region + + svc := sts.New(p.session, &aws.Config{ + Region: aws.String(region), + }) + output, err := svc.GetCallerIdentity(&sts.GetCallerIdentityInput{}) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "InvalidClientTokenId" { + p.log.Debug("Region is disabled (to enable see: https://docs.aws.amazon.com/general/latest/gr/rande-manage.html#rande-manage-enable). skipping...", + zap.String("account_id", p.accountID), + zap.String("region", region)) + continue + } + } + return err + } + p.accountID = aws.StringValue(output.Account) + p.initRegionalClients() + var wg sync.WaitGroup + for _, resource := range p.config.Resources { + wg.Add(1) + go p.collectResource(&wg, resource.Name, resource.Other) + } + wg.Wait() + } + globalCollectedResources = map[string]bool{} + p.resourceClients = map[string]resource.ClientInterface{} + } + + return nil +} + +func (p *Provider) initRegionalClients() { + zapLog := p.log.With(zap.String("account_id", p.accountID), zap.String("region", p.region)) + for serviceName, newFunc := range regionalServices { + p.resourceClients[serviceName] = newFunc(p.session, + &aws.Config{ + Region: aws.String(p.region), + LogLevel: &p.logLevel, + MaxRetries: p.config.MaxRetries, + }, + p.db, zapLog, p.accountID, p.region) + } +} + +var lock = sync.RWMutex{} + +func (p *Provider) collectResource(wg *sync.WaitGroup, fullResourceName string, config interface{}) { + defer wg.Done() + resourcePath := strings.Split(fullResourceName, ".") + if len(resourcePath) != 2 { + log.Fatalf("resource %s should be in format {service}.{resource}", fullResourceName) + } + service := resourcePath[0] + resourceName := resourcePath[1] + + if globalServices[service] != nil { + lock.Lock() + if globalCollectedResources[fullResourceName] { + lock.Unlock() + return + } + globalCollectedResources[fullResourceName] = true + if p.resourceClients[service] == nil { + zapLog := p.log.With(zap.String("account_id", p.accountID)) + p.resourceClients[service] = globalServices[service](p.session, + &aws.Config{Region: aws.String(p.region), + LogLevel: &p.logLevel, + MaxRetries: p.config.MaxRetries, + }, + p.db, zapLog, p.accountID, p.region) + } + lock.Unlock() + } + + if p.resourceClients[service] == nil { + log.Fatalf("unsupported service %s for resource %s", service, resourceName) + } + + err := p.resourceClients[service].CollectResource(resourceName, config) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + switch awsErr.Code() { + case "AccessDenied", "AccessDeniedException", "UnauthorizedOperation": + p.log.Info("Skipping resource. Access denied", zap.String("account_id", p.accountID), zap.String("resource", fullResourceName), zap.Error(err)) + return + } + } + log.Fatal(err) + } +} + +func main() { + sdk.ServePlugin(&Provider{}) +} \ No newline at end of file diff --git a/rds/certificates.go b/rds/certificates.go new file mode 100644 index 000000000..370f34e1a --- /dev/null +++ b/rds/certificates.go @@ -0,0 +1,78 @@ +package rds + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Certificate struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + CertificateArn *string `neo:"unique"` + CertificateIdentifier *string + CertificateType *string + CustomerOverride *bool + CustomerOverrideValidTill *time.Time + Thumbprint *string + ValidFrom *time.Time + ValidTill *time.Time +} + +func (Certificate) TableName() string { + return "aws_rds_certificates" +} + +func (c *Client) transformCertificate(value *rds.Certificate) *Certificate { + return &Certificate{ + Region: c.region, + AccountID: c.accountID, + CertificateArn: value.CertificateArn, + CertificateIdentifier: value.CertificateIdentifier, + CertificateType: value.CertificateType, + CustomerOverride: value.CustomerOverride, + CustomerOverrideValidTill: value.CustomerOverrideValidTill, + Thumbprint: value.Thumbprint, + ValidFrom: value.ValidFrom, + ValidTill: value.ValidTill, + } +} + +func (c *Client) transformCertificates(values []*rds.Certificate) []*Certificate { + var tValues []*Certificate + for _, v := range values { + tValues = append(tValues, c.transformCertificate(v)) + } + return tValues +} + +var CertificateTables = []interface{}{ + &Certificate{}, +} + +func (c *Client) certificates(gConfig interface{}) error { + var config rds.DescribeCertificatesInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(CertificateTables...) + + for { + output, err := c.svc.DescribeCertificates(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformCertificates(output.Certificates)) + c.log.Info("Fetched resources", zap.String("resource", "rds.certificates"), zap.Int("count", len(output.Certificates))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/rds/client.go b/rds/client.go new file mode 100644 index 000000000..e2935cd5c --- /dev/null +++ b/rds/client.go @@ -0,0 +1,45 @@ +package rds + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *rds.RDS +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: rds.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "certificates": + return c.certificates(config) + case "clusters": + return c.clusters(config) + case "db_subnet_groups": + return c.dbSubnetGroups(config) + default: + return fmt.Errorf("unsupported resource rds.%s", resource) + } +} diff --git a/rds/clusters.go b/rds/clusters.go new file mode 100644 index 000000000..f5bae60dd --- /dev/null +++ b/rds/clusters.go @@ -0,0 +1,356 @@ +package rds + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/cloudquery/cloudquery/providers/common" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Cluster struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string + Region string + ActivityStreamKinesisStreamName *string + ActivityStreamKmsKeyId *string + ActivityStreamMode *string + ActivityStreamStatus *string + AllocatedStorage *int64 + AssociatedRoles []*ClusterRole `gorm:"constraint:OnDelete:CASCADE;"` + AvailabilityZones *string + BacktrackConsumedChangeRecords *int64 + BacktrackWindow *int64 + BackupRetentionPeriod *int64 + Capacity *int64 + CharacterSetName *string + CloneGroupId *string + ClusterCreateTime *time.Time + CopyTagsToSnapshot *bool + CrossAccountClone *bool + CustomEndpoints *string + ClusterArn *string `neo:"unique"` + ClusterIdentifier *string + ClusterMembers []*ClusterMember `gorm:"constraint:OnDelete:CASCADE;"` + ClusterOptionGroupMemberships []*ClusterOptionGroupStatus `gorm:"constraint:OnDelete:CASCADE;"` + ClusterParameterGroup *string + SubnetGroup *string + DatabaseName *string + DbClusterResourceId *string + DeletionProtection *bool + DomainMemberships []*ClusterDomainMembership `gorm:"constraint:OnDelete:CASCADE;"` + EarliestBacktrackTime *time.Time + EarliestRestorableTime *time.Time + EnabledCloudwatchLogsExports *string + Endpoint *string + Engine *string + EngineMode *string + EngineVersion *string + GlobalWriteForwardingRequested *bool + GlobalWriteForwardingStatus *string + HostedZoneId *string + HttpEndpointEnabled *bool + IAMDatabaseAuthenticationEnabled *bool + KmsKeyId *string + LatestRestorableTime *time.Time + MasterUsername *string + MultiAZ *bool + PercentProgress *string + Port *int64 + PreferredBackupWindow *string + PreferredMaintenanceWindow *string + ReadReplicaIdentifiers *string + ReaderEndpoint *string + ReplicationSourceIdentifier *string + + ScalingConfigAutoPause *bool + ScalingConfigMaxCapacity *int64 + ScalingConfigMinCapacity *int64 + ScalingConfigSecondsUntilAutoPause *int64 + ScalingConfigTimeoutAction *string + + Status *string + StorageEncrypted *bool + VpcSecurityGroups []*ClusterVpcSecurityGroupMembership `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (Cluster) TableName() string { + return "aws_rds_clusters" +} + +type ClusterRole struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + FeatureName *string + RoleArn *string + Status *string +} + +func (ClusterRole) TableName() string { + return "aws_rds_cluster_roles" +} + +type ClusterMember struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ClusterParameterGroupStatus *string + InstanceIdentifier *string + IsClusterWriter *bool + PromotionTier *int64 +} + +func (ClusterMember) TableName() string { + return "aws_rds_cluster_members" +} + +type ClusterOptionGroupStatus struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + ClusterOptionGroupName *string + Status *string +} + +func (ClusterOptionGroupStatus) TableName() string { + return "aws_rds_cluster_option_group_statuses" +} + +type ClusterDomainMembership struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Domain *string + FQDN *string + IAMRoleName *string + Status *string +} + +func (ClusterDomainMembership) TableName() string { + return "aws_rds_cluster_domain_membership" +} + +type ClusterVpcSecurityGroupMembership struct { + ID uint `gorm:"primarykey"` + ClusterID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Status *string + VpcSecurityGroupId *string +} + +func (ClusterVpcSecurityGroupMembership) TableName() string { + return "aws_rds_cluster_vpc_security_group_memberships" +} + +func (c *Client) transformClusterRole(value *rds.DBClusterRole) *ClusterRole { + return &ClusterRole{ + AccountID: c.accountID, + Region: c.region, + FeatureName: value.FeatureName, + RoleArn: value.RoleArn, + Status: value.Status, + } +} + +func (c *Client) transformClusterRoles(values []*rds.DBClusterRole) []*ClusterRole { + var tValues []*ClusterRole + for _, v := range values { + tValues = append(tValues, c.transformClusterRole(v)) + } + return tValues +} + +func (c *Client) transformClusterMember(value *rds.DBClusterMember) *ClusterMember { + return &ClusterMember{ + AccountID: c.accountID, + Region: c.region, + ClusterParameterGroupStatus: value.DBClusterParameterGroupStatus, + InstanceIdentifier: value.DBInstanceIdentifier, + IsClusterWriter: value.IsClusterWriter, + PromotionTier: value.PromotionTier, + } +} + +func (c *Client) transformClusterMembers(values []*rds.DBClusterMember) []*ClusterMember { + var tValues []*ClusterMember + for _, v := range values { + tValues = append(tValues, c.transformClusterMember(v)) + } + return tValues +} + +func (c *Client) transformClusterOptionGroupStatus(value *rds.DBClusterOptionGroupStatus) *ClusterOptionGroupStatus { + return &ClusterOptionGroupStatus{ + AccountID: c.accountID, + Region: c.region, + ClusterOptionGroupName: value.DBClusterOptionGroupName, + Status: value.Status, + } +} + +func (c *Client) transformClusterOptionGroupStatuss(values []*rds.DBClusterOptionGroupStatus) []*ClusterOptionGroupStatus { + var tValues []*ClusterOptionGroupStatus + for _, v := range values { + tValues = append(tValues, c.transformClusterOptionGroupStatus(v)) + } + return tValues +} + +func (c *Client) transformClusterDomainMembership(value *rds.DomainMembership) *ClusterDomainMembership { + return &ClusterDomainMembership{ + AccountID: c.accountID, + Region: c.region, + Domain: value.Domain, + FQDN: value.FQDN, + IAMRoleName: value.IAMRoleName, + Status: value.Status, + } +} + +func (c *Client) transformClusterDomainMemberships(values []*rds.DomainMembership) []*ClusterDomainMembership { + var tValues []*ClusterDomainMembership + for _, v := range values { + tValues = append(tValues, c.transformClusterDomainMembership(v)) + } + return tValues +} + +func (c *Client) transformClusterVpcSecurityGroupMembership(value *rds.VpcSecurityGroupMembership) *ClusterVpcSecurityGroupMembership { + return &ClusterVpcSecurityGroupMembership{ + AccountID: c.accountID, + Region: c.region, + Status: value.Status, + VpcSecurityGroupId: value.VpcSecurityGroupId, + } +} + +func (c *Client) transformClusterVpcSecurityGroupMemberships(values []*rds.VpcSecurityGroupMembership) []*ClusterVpcSecurityGroupMembership { + var tValues []*ClusterVpcSecurityGroupMembership + for _, v := range values { + tValues = append(tValues, c.transformClusterVpcSecurityGroupMembership(v)) + } + return tValues +} + +func (c *Client) transformCluster(value *rds.DBCluster) *Cluster { + res := Cluster{ + Region: c.region, + AccountID: c.accountID, + ActivityStreamKinesisStreamName: value.ActivityStreamKinesisStreamName, + ActivityStreamKmsKeyId: value.ActivityStreamKmsKeyId, + ActivityStreamMode: value.ActivityStreamMode, + ActivityStreamStatus: value.ActivityStreamStatus, + AllocatedStorage: value.AllocatedStorage, + AssociatedRoles: c.transformClusterRoles(value.AssociatedRoles), + AvailabilityZones: common.StringListToString(value.AvailabilityZones), + BacktrackConsumedChangeRecords: value.BacktrackConsumedChangeRecords, + BacktrackWindow: value.BacktrackWindow, + BackupRetentionPeriod: value.BackupRetentionPeriod, + Capacity: value.Capacity, + CharacterSetName: value.CharacterSetName, + CloneGroupId: value.CloneGroupId, + ClusterCreateTime: value.ClusterCreateTime, + CopyTagsToSnapshot: value.CopyTagsToSnapshot, + CrossAccountClone: value.CrossAccountClone, + CustomEndpoints: common.StringListToString(value.CustomEndpoints), + ClusterArn: value.DBClusterArn, + ClusterIdentifier: value.DBClusterIdentifier, + ClusterMembers: c.transformClusterMembers(value.DBClusterMembers), + ClusterOptionGroupMemberships: c.transformClusterOptionGroupStatuss(value.DBClusterOptionGroupMemberships), + ClusterParameterGroup: value.DBClusterParameterGroup, + SubnetGroup: value.DBSubnetGroup, + DatabaseName: value.DatabaseName, + DbClusterResourceId: value.DbClusterResourceId, + DeletionProtection: value.DeletionProtection, + DomainMemberships: c.transformClusterDomainMemberships(value.DomainMemberships), + EarliestBacktrackTime: value.EarliestBacktrackTime, + EarliestRestorableTime: value.EarliestRestorableTime, + EnabledCloudwatchLogsExports: common.StringListToString(value.EnabledCloudwatchLogsExports), + Endpoint: value.Endpoint, + Engine: value.Engine, + EngineMode: value.EngineMode, + EngineVersion: value.EngineVersion, + GlobalWriteForwardingRequested: value.GlobalWriteForwardingRequested, + GlobalWriteForwardingStatus: value.GlobalWriteForwardingStatus, + HostedZoneId: value.HostedZoneId, + HttpEndpointEnabled: value.HttpEndpointEnabled, + IAMDatabaseAuthenticationEnabled: value.IAMDatabaseAuthenticationEnabled, + KmsKeyId: value.KmsKeyId, + LatestRestorableTime: value.LatestRestorableTime, + MasterUsername: value.MasterUsername, + MultiAZ: value.MultiAZ, + PercentProgress: value.PercentProgress, + Port: value.Port, + PreferredBackupWindow: value.PreferredBackupWindow, + PreferredMaintenanceWindow: value.PreferredMaintenanceWindow, + ReadReplicaIdentifiers: common.StringListToString(value.ReadReplicaIdentifiers), + ReaderEndpoint: value.ReaderEndpoint, + ReplicationSourceIdentifier: value.ReplicationSourceIdentifier, + Status: value.Status, + StorageEncrypted: value.StorageEncrypted, + VpcSecurityGroups: c.transformClusterVpcSecurityGroupMemberships(value.VpcSecurityGroups), + } + + if value.ScalingConfigurationInfo != nil { + res.ScalingConfigAutoPause = value.ScalingConfigurationInfo.AutoPause + res.ScalingConfigMaxCapacity = value.ScalingConfigurationInfo.MaxCapacity + res.ScalingConfigMinCapacity = value.ScalingConfigurationInfo.MinCapacity + res.ScalingConfigSecondsUntilAutoPause = value.ScalingConfigurationInfo.SecondsUntilAutoPause + res.ScalingConfigTimeoutAction = value.ScalingConfigurationInfo.TimeoutAction + } + + return &res +} + +func (c *Client) transformClusters(values []*rds.DBCluster) []*Cluster { + var tValues []*Cluster + for _, v := range values { + tValues = append(tValues, c.transformCluster(v)) + } + return tValues +} + +var ClusterTables = []interface{}{ + &Cluster{}, + &ClusterRole{}, + &ClusterMember{}, + &ClusterOptionGroupStatus{}, + &ClusterDomainMembership{}, + &ClusterVpcSecurityGroupMembership{}, +} + +func (c *Client) clusters(gConfig interface{}) error { + var config rds.DescribeDBClustersInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(ClusterTables...) + + for { + output, err := c.svc.DescribeDBClusters(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformClusters(output.DBClusters)) + c.log.Info("Fetched resources", zap.String("resource", "rds.clusters"), zap.Int("count", len(output.DBClusters))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/rds/subnet_groups.go b/rds/subnet_groups.go new file mode 100644 index 000000000..1f4465d45 --- /dev/null +++ b/rds/subnet_groups.go @@ -0,0 +1,114 @@ +package rds + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type DBSubnetGroup struct { + ID uint `gorm:"primarykey"` + AccountID string + Region string + DBSubnetGroupArn *string `neo:"unique"` + DBSubnetGroupDescription *string + DBSubnetGroupName *string + SubnetGroupStatus *string + Subnets []*DBSubnetGroupSubnet `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string +} + +func (DBSubnetGroup) TableName() string { + return "aws_rds_db_subnet_groups" +} + +type DBSubnetGroupSubnet struct { + ID uint `gorm:"primarykey"` + DBSubnetGroupID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AvailabilityZoneName *string + Identifier *string + OutpostArn *string + Status *string +} + +func (DBSubnetGroupSubnet) TableName() string { + return "aws_rds_db_subnet_group_subnets" +} + +func (c *Client) transformDBSubnetGroupSubnet(value *rds.Subnet) *DBSubnetGroupSubnet { + res := DBSubnetGroupSubnet{ + Identifier: value.SubnetIdentifier, + Status: value.SubnetStatus, + } + + if value.SubnetAvailabilityZone != nil { + res.AvailabilityZoneName = value.SubnetAvailabilityZone.Name + } + + if value.SubnetOutpost != nil { + res.OutpostArn = value.SubnetOutpost.Arn + } + + return &res +} + +func (c *Client) transformDBSubnetGroupSubnets(values []*rds.Subnet) []*DBSubnetGroupSubnet { + var tValues []*DBSubnetGroupSubnet + for _, v := range values { + tValues = append(tValues, c.transformDBSubnetGroupSubnet(v)) + } + return tValues +} + +func (c *Client) transformDBSubnetGroup(value *rds.DBSubnetGroup) *DBSubnetGroup { + return &DBSubnetGroup{ + Region: c.region, + AccountID: c.accountID, + DBSubnetGroupArn: value.DBSubnetGroupArn, + DBSubnetGroupDescription: value.DBSubnetGroupDescription, + DBSubnetGroupName: value.DBSubnetGroupName, + SubnetGroupStatus: value.SubnetGroupStatus, + Subnets: c.transformDBSubnetGroupSubnets(value.Subnets), + VpcId: value.VpcId, + } +} + +func (c *Client) transformDBSubnetGroups(values []*rds.DBSubnetGroup) []*DBSubnetGroup { + var tValues []*DBSubnetGroup + for _, v := range values { + tValues = append(tValues, c.transformDBSubnetGroup(v)) + } + return tValues +} + +var DBSubnetGroupTables = []interface{}{ + &DBSubnetGroup{}, + &DBSubnetGroupSubnet{}, +} + +func (c *Client) dbSubnetGroups(gConfig interface{}) error { + var config rds.DescribeDBSubnetGroupsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(DBSubnetGroupTables...) + + for { + output, err := c.svc.DescribeDBSubnetGroups(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformDBSubnetGroups(output.DBSubnetGroups)) + c.log.Info("Fetched resources", zap.String("resource", "rds.subnet_groups"), zap.Int("count", len(output.DBSubnetGroups))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/redshift/client.go b/redshift/client.go new file mode 100644 index 000000000..01d0c8c2f --- /dev/null +++ b/redshift/client.go @@ -0,0 +1,43 @@ +package redshift + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/redshift" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *redshift.Redshift +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: redshift.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "clusters": + return c.clusters(config) + case "cluster_subnet_groups": + return c.clusterSubnetGroups(config) + default: + return fmt.Errorf("unsupported resource redshift.%s", resource) + } +} diff --git a/redshift/clusters.go b/redshift/clusters.go new file mode 100644 index 000000000..be6ec821f --- /dev/null +++ b/redshift/clusters.go @@ -0,0 +1,509 @@ +package redshift + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/redshift" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Cluster struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + AllowVersionUpgrade *bool + AutomatedSnapshotRetentionPeriod *int64 + AvailabilityZone *string + ClusterAvailabilityStatus *string + ClusterCreateTime *time.Time + ClusterIdentifier *string `neo:"unique"` + ClusterNodes []*ClusterNode `gorm:"constraint:OnDelete:CASCADE;"` + ClusterParameterGroups []*ClusterParameterGroupStatus `gorm:"constraint:OnDelete:CASCADE;"` + ClusterPublicKey *string + ClusterRevisionNumber *string + ClusterSecurityGroups []*ClusterSecurityGroupMembership `gorm:"constraint:OnDelete:CASCADE;"` + + ClusterSnapshotCopyStatusDestinationRegion *string + ClusterSnapshotCopyStatusManualSnapshotRetentionPeriod *int64 + ClusterSnapshotCopyStatusRetentionPeriod *int64 + ClusterSnapshotCopyStatusSnapshotCopyGrantName *string + + ClusterStatus *string + ClusterSubnetGroupName *string + ClusterVersion *string + DBName *string + + DataTransferProgressCurrentRateInMegaBytesPerSecond *float64 + DataTransferProgressDataTransferredInMegaBytes *int64 + DataTransferProgressElapsedTimeInSeconds *int64 + DataTransferProgressEstimatedTimeToCompletionInSeconds *int64 + DataTransferProgressStatus *string + DataTransferProgressTotalDataInMegaBytes *int64 + + DeferredMaintenanceWindows []*ClusterDeferredMaintenanceWindow `gorm:"constraint:OnDelete:CASCADE;"` + + ElasticIpStatusElasticIp *string + ElasticIpStatusStatus *string + + ElasticResizeNumberOfNodeOptions *string + Encrypted *bool + + EndpointAddress *string + EndpointPort *int64 + + EnhancedVpcRouting *bool + ExpectedNextSnapshotScheduleTime *time.Time + ExpectedNextSnapshotScheduleTimeStatus *string + + HsmStatusHsmClientCertificateIdentifier *string + HsmStatusHsmConfigurationIdentifier *string + HsmStatusStatus *string + + IamRoles []*ClusterIamRole `gorm:"constraint:OnDelete:CASCADE;"` + KmsKeyId *string + MaintenanceTrackName *string + ManualSnapshotRetentionPeriod *int64 + MasterUsername *string + ModifyStatus *string + NextMaintenanceWindowStartTime *time.Time + NodeType *string + NumberOfNodes *int64 + PendingActions []*ClusterPendingActions `gorm:"constraint:OnDelete:CASCADE;"` + + PendingModifiedValuesAutomatedSnapshotRetentionPeriod *int64 + PendingModifiedValuesClusterIdentifier *string + PendingModifiedValuesClusterType *string + PendingModifiedValuesClusterVersion *string + PendingModifiedValuesEncryptionType *string + PendingModifiedValuesEnhancedVpcRouting *bool + PendingModifiedValuesMaintenanceTrackName *string + PendingModifiedValuesMasterUserPassword *string + PendingModifiedValuesNodeType *string + PendingModifiedValuesNumberOfNodes *int64 + PendingModifiedValuesPubliclyAccessible *bool + + PreferredMaintenanceWindow *string + PubliclyAccessible *bool + + ResizeInfoAllowCancelResize *bool + ResizeInfoResizeType *string + + RestoreStatusCurrentRestoreRateInMegaBytesPerSecond *float64 + RestoreStatusElapsedTimeInSeconds *int64 + RestoreStatusEstimatedTimeToCompletionInSeconds *int64 + RestoreStatusProgressInMegaBytes *int64 + RestoreStatusSnapshotSizeInMegaBytes *int64 + RestoreStatusStatus *string + + SnapshotScheduleIdentifier *string + SnapshotScheduleState *string + Tags []*ClusterTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string + VpcSecurityGroups []*ClusterVpcSecurityGroupMembership `gorm:"constraint:OnDelete:CASCADE;"` +} + +func (Cluster) TableName() string { + return "aws_redshift_clusters" +} + +type ClusterNode struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + ClusterID uint `neo:"ignore"` + NodeRole *string + PrivateIPAddress *string + PublicIPAddress *string +} + +func (ClusterNode) TableName() string { + return "aws_redshift_cluster_nodes" +} + +type ClusterParameterStatus struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + ClusterParameterGroupStatusID uint `neo:"ignore"` + ParameterApplyErrorDescription *string + ParameterApplyStatus *string + ParameterName *string +} + +func (ClusterParameterStatus) TableName() string { + return "aws_redshift_cluster_parameter_statuses" +} + +type ClusterParameterGroupStatus struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + ClusterID uint `neo:"ignore"` + List []*ClusterParameterStatus `gorm:"constraint:OnDelete:CASCADE;"` + ParameterApplyStatus *string + ParameterGroupName *string +} + +func (ClusterParameterGroupStatus) TableName() string { + return "aws_redshift_cluster_parameter_group_statuses" +} + +type ClusterSecurityGroupMembership struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + ClusterID uint `neo:"ignore"` + ClusterSecurityGroupName *string + Status *string +} + +func (ClusterSecurityGroupMembership) TableName() string { + return "aws_redshift_cluster_security_group_memberships" +} + +type ClusterDeferredMaintenanceWindow struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + ClusterID uint `neo:"ignore"` + DeferMaintenanceEndTime *time.Time + DeferMaintenanceIdentifier *string + DeferMaintenanceStartTime *time.Time +} + +func (ClusterDeferredMaintenanceWindow) TableName() string { + return "aws_redshift_cluster_deferred_maintenance_windows" +} + +type ClusterIamRole struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + ClusterID uint `neo:"ignore"` + ApplyStatus *string + IamRoleArn *string +} + +func (ClusterIamRole) TableName() string { + return "aws_redshift_cluster_iam_roles" +} + +type ClusterPendingActions struct { + ID uint `gorm:"primarykey"` + ClusterID uint + Value *string +} + +func (ClusterPendingActions) TableName() string { + return "aws_redshift_cluster_pending_actions" +} + +type ClusterTag struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + ClusterID uint `neo:"ignore"` + Key *string + Value *string +} + +func (ClusterTag) TableName() string { + return "aws_redshift_cluster_tags" +} + +type ClusterVpcSecurityGroupMembership struct { + ID uint `gorm:"primarykey"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + ClusterID uint `neo:"ignore"` + Status *string + VpcSecurityGroupId *string +} + +func (ClusterVpcSecurityGroupMembership) TableName() string { + return "aws_redshift_cluster_vpc_security_group_memberships" +} + +func (c *Client) transformClusters(values []*redshift.Cluster) []*Cluster { + var tValues []*Cluster + for _, value := range values { + tValue := Cluster{ + AccountID: c.accountID, + Region: c.region, + AllowVersionUpgrade: value.AllowVersionUpgrade, + AutomatedSnapshotRetentionPeriod: value.AutomatedSnapshotRetentionPeriod, + AvailabilityZone: value.AvailabilityZone, + ClusterAvailabilityStatus: value.ClusterAvailabilityStatus, + ClusterCreateTime: value.ClusterCreateTime, + ClusterIdentifier: value.ClusterIdentifier, + ClusterNodes: c.transformClusterNodes(value.ClusterNodes), + ClusterParameterGroups: c.transformClusterParameterGroupStatuss(value.ClusterParameterGroups), + ClusterPublicKey: value.ClusterPublicKey, + ClusterRevisionNumber: value.ClusterRevisionNumber, + ClusterSecurityGroups: c.transformClusterSecurityGroupMemberships(value.ClusterSecurityGroups), + ClusterStatus: value.ClusterStatus, + ClusterSubnetGroupName: value.ClusterSubnetGroupName, + ClusterVersion: value.ClusterVersion, + DBName: value.DBName, + DeferredMaintenanceWindows: c.transformClusterDeferredMaintenanceWindows(value.DeferredMaintenanceWindows), + ElasticResizeNumberOfNodeOptions: value.ElasticResizeNumberOfNodeOptions, + Encrypted: value.Encrypted, + EnhancedVpcRouting: value.EnhancedVpcRouting, + ExpectedNextSnapshotScheduleTime: value.ExpectedNextSnapshotScheduleTime, + ExpectedNextSnapshotScheduleTimeStatus: value.ExpectedNextSnapshotScheduleTimeStatus, + IamRoles: c.transformClusterIamRoles(value.IamRoles), + KmsKeyId: value.KmsKeyId, + MaintenanceTrackName: value.MaintenanceTrackName, + ManualSnapshotRetentionPeriod: value.ManualSnapshotRetentionPeriod, + MasterUsername: value.MasterUsername, + ModifyStatus: value.ModifyStatus, + NextMaintenanceWindowStartTime: value.NextMaintenanceWindowStartTime, + NodeType: value.NodeType, + NumberOfNodes: value.NumberOfNodes, + PendingActions: c.transformClusterPendingActions(value.PendingActions), + PreferredMaintenanceWindow: value.PreferredMaintenanceWindow, + PubliclyAccessible: value.PubliclyAccessible, + SnapshotScheduleIdentifier: value.SnapshotScheduleIdentifier, + SnapshotScheduleState: value.SnapshotScheduleState, + Tags: c.transformClusterTags(value.Tags), + VpcId: value.VpcId, + VpcSecurityGroups: c.transformClusterVpcSecurityGroupMemberships(value.VpcSecurityGroups), + } + if value.ClusterSnapshotCopyStatus != nil { + + tValue.ClusterSnapshotCopyStatusDestinationRegion = value.ClusterSnapshotCopyStatus.DestinationRegion + tValue.ClusterSnapshotCopyStatusManualSnapshotRetentionPeriod = value.ClusterSnapshotCopyStatus.ManualSnapshotRetentionPeriod + tValue.ClusterSnapshotCopyStatusRetentionPeriod = value.ClusterSnapshotCopyStatus.RetentionPeriod + tValue.ClusterSnapshotCopyStatusSnapshotCopyGrantName = value.ClusterSnapshotCopyStatus.SnapshotCopyGrantName + + } + if value.DataTransferProgress != nil { + + tValue.DataTransferProgressCurrentRateInMegaBytesPerSecond = value.DataTransferProgress.CurrentRateInMegaBytesPerSecond + tValue.DataTransferProgressDataTransferredInMegaBytes = value.DataTransferProgress.DataTransferredInMegaBytes + tValue.DataTransferProgressElapsedTimeInSeconds = value.DataTransferProgress.ElapsedTimeInSeconds + tValue.DataTransferProgressEstimatedTimeToCompletionInSeconds = value.DataTransferProgress.EstimatedTimeToCompletionInSeconds + tValue.DataTransferProgressStatus = value.DataTransferProgress.Status + tValue.DataTransferProgressTotalDataInMegaBytes = value.DataTransferProgress.TotalDataInMegaBytes + + } + if value.ElasticIpStatus != nil { + + tValue.ElasticIpStatusElasticIp = value.ElasticIpStatus.ElasticIp + tValue.ElasticIpStatusStatus = value.ElasticIpStatus.Status + + } + if value.Endpoint != nil { + + tValue.EndpointAddress = value.Endpoint.Address + tValue.EndpointPort = value.Endpoint.Port + + } + if value.HsmStatus != nil { + + tValue.HsmStatusHsmClientCertificateIdentifier = value.HsmStatus.HsmClientCertificateIdentifier + tValue.HsmStatusHsmConfigurationIdentifier = value.HsmStatus.HsmConfigurationIdentifier + tValue.HsmStatusStatus = value.HsmStatus.Status + + } + if value.PendingModifiedValues != nil { + + tValue.PendingModifiedValuesAutomatedSnapshotRetentionPeriod = value.PendingModifiedValues.AutomatedSnapshotRetentionPeriod + tValue.PendingModifiedValuesClusterIdentifier = value.PendingModifiedValues.ClusterIdentifier + tValue.PendingModifiedValuesClusterType = value.PendingModifiedValues.ClusterType + tValue.PendingModifiedValuesClusterVersion = value.PendingModifiedValues.ClusterVersion + tValue.PendingModifiedValuesEncryptionType = value.PendingModifiedValues.EncryptionType + tValue.PendingModifiedValuesEnhancedVpcRouting = value.PendingModifiedValues.EnhancedVpcRouting + tValue.PendingModifiedValuesMaintenanceTrackName = value.PendingModifiedValues.MaintenanceTrackName + tValue.PendingModifiedValuesMasterUserPassword = value.PendingModifiedValues.MasterUserPassword + tValue.PendingModifiedValuesNodeType = value.PendingModifiedValues.NodeType + tValue.PendingModifiedValuesNumberOfNodes = value.PendingModifiedValues.NumberOfNodes + tValue.PendingModifiedValuesPubliclyAccessible = value.PendingModifiedValues.PubliclyAccessible + + } + if value.ResizeInfo != nil { + + tValue.ResizeInfoAllowCancelResize = value.ResizeInfo.AllowCancelResize + tValue.ResizeInfoResizeType = value.ResizeInfo.ResizeType + + } + if value.RestoreStatus != nil { + + tValue.RestoreStatusCurrentRestoreRateInMegaBytesPerSecond = value.RestoreStatus.CurrentRestoreRateInMegaBytesPerSecond + tValue.RestoreStatusElapsedTimeInSeconds = value.RestoreStatus.ElapsedTimeInSeconds + tValue.RestoreStatusEstimatedTimeToCompletionInSeconds = value.RestoreStatus.EstimatedTimeToCompletionInSeconds + tValue.RestoreStatusProgressInMegaBytes = value.RestoreStatus.ProgressInMegaBytes + tValue.RestoreStatusSnapshotSizeInMegaBytes = value.RestoreStatus.SnapshotSizeInMegaBytes + tValue.RestoreStatusStatus = value.RestoreStatus.Status + + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterNodes(values []*redshift.ClusterNode) []*ClusterNode { + var tValues []*ClusterNode + for _, value := range values { + tValue := ClusterNode{ + AccountID: c.accountID, + Region: c.region, + NodeRole: value.NodeRole, + PrivateIPAddress: value.PrivateIPAddress, + PublicIPAddress: value.PublicIPAddress, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterParameterStatuss(values []*redshift.ClusterParameterStatus) []*ClusterParameterStatus { + var tValues []*ClusterParameterStatus + for _, value := range values { + tValue := ClusterParameterStatus{ + AccountID: c.accountID, + Region: c.region, + ParameterApplyErrorDescription: value.ParameterApplyErrorDescription, + ParameterApplyStatus: value.ParameterApplyStatus, + ParameterName: value.ParameterName, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterParameterGroupStatuss(values []*redshift.ClusterParameterGroupStatus) []*ClusterParameterGroupStatus { + var tValues []*ClusterParameterGroupStatus + for _, value := range values { + tValue := ClusterParameterGroupStatus{ + AccountID: c.accountID, + Region: c.region, + List: c.transformClusterParameterStatuss(value.ClusterParameterStatusList), + ParameterApplyStatus: value.ParameterApplyStatus, + ParameterGroupName: value.ParameterGroupName, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterSecurityGroupMemberships(values []*redshift.ClusterSecurityGroupMembership) []*ClusterSecurityGroupMembership { + var tValues []*ClusterSecurityGroupMembership + for _, value := range values { + tValue := ClusterSecurityGroupMembership{ + AccountID: c.accountID, + Region: c.region, + ClusterSecurityGroupName: value.ClusterSecurityGroupName, + Status: value.Status, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterDeferredMaintenanceWindows(values []*redshift.DeferredMaintenanceWindow) []*ClusterDeferredMaintenanceWindow { + var tValues []*ClusterDeferredMaintenanceWindow + for _, value := range values { + tValue := ClusterDeferredMaintenanceWindow{ + AccountID: c.accountID, + Region: c.region, + DeferMaintenanceEndTime: value.DeferMaintenanceEndTime, + DeferMaintenanceIdentifier: value.DeferMaintenanceIdentifier, + DeferMaintenanceStartTime: value.DeferMaintenanceStartTime, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterIamRoles(values []*redshift.ClusterIamRole) []*ClusterIamRole { + var tValues []*ClusterIamRole + for _, value := range values { + tValue := ClusterIamRole{ + AccountID: c.accountID, + Region: c.region, + ApplyStatus: value.ApplyStatus, + IamRoleArn: value.IamRoleArn, + } + tValues = append(tValues, &tValue) + } + return tValues +} +func (c *Client) transformClusterPendingActions(values []*string) []*ClusterPendingActions { + var tValues []*ClusterPendingActions + for _, v := range values { + tValues = append(tValues, &ClusterPendingActions{ + Value: v, + }) + } + return tValues +} + +func (c *Client) transformClusterTags(values []*redshift.Tag) []*ClusterTag { + var tValues []*ClusterTag + for _, value := range values { + tValue := ClusterTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterVpcSecurityGroupMemberships(values []*redshift.VpcSecurityGroupMembership) []*ClusterVpcSecurityGroupMembership { + var tValues []*ClusterVpcSecurityGroupMembership + for _, value := range values { + tValue := ClusterVpcSecurityGroupMembership{ + AccountID: c.accountID, + Region: c.region, + Status: value.Status, + VpcSecurityGroupId: value.VpcSecurityGroupId, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +type ClusterConfig struct { + Filter string +} + +var ClusterTables = []interface{}{ + &Cluster{}, + &ClusterNode{}, + &ClusterParameterGroupStatus{}, + &ClusterParameterStatus{}, + &ClusterSecurityGroupMembership{}, + &ClusterDeferredMaintenanceWindow{}, + &ClusterIamRole{}, + &ClusterTag{}, + &ClusterVpcSecurityGroupMembership{}, +} + +func (c *Client) clusters(gConfig interface{}) error { + var config redshift.DescribeClustersInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(ClusterTables...) + + for { + output, err := c.svc.DescribeClusters(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformClusters(output.Clusters)) + c.log.Info("Fetched resources", zap.String("resource", "redshift.clusters"), zap.Int("count", len(output.Clusters))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + return nil +} diff --git a/redshift/subnet_groups.go b/redshift/subnet_groups.go new file mode 100644 index 000000000..4a48363d7 --- /dev/null +++ b/redshift/subnet_groups.go @@ -0,0 +1,162 @@ +package redshift + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/redshift" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + +type ClusterSubnetGroup struct { + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + + Name *string `neo:"unique"` + Description *string + Status *string + Subnets []*ClusterSubnetGroupSubnet `gorm:"constraint:OnDelete:CASCADE;"` + Tags []*ClusterSubnetGroupTag `gorm:"constraint:OnDelete:CASCADE;"` + VpcId *string +} + +func (ClusterSubnetGroup) TableName() string { + return "aws_redshift_cluster_subnet_groups" +} + +type ClusterSubnetGroupTag struct { + ID uint `gorm:"primarykey"` + ClusterSubnetGroupID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Key *string + Value *string +} + +func (ClusterSubnetGroupTag) TableName() string { + return "aws_redshift_cluster_subnet_group_tags" +} + +type ClusterSubnetGroupSubnet struct { + ID uint `gorm:"primarykey"` + ClusterSubnetGroupID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + AZName *string + AZPlatforms []*ClusterSubnetGroupSupportedPlatform `gorm:"constraint:OnDelete:CASCADE;"` + Identifier *string + Status *string +} + +func (ClusterSubnetGroupSubnet) TableName() string { + return "aws_redshift_cluster_subnet_group_subnets" +} + +type ClusterSubnetGroupSupportedPlatform struct { + ID uint `gorm:"primarykey"` + ClusterSubnetGroupSubnetID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + Name *string +} + +func (ClusterSubnetGroupSupportedPlatform) TableName() string { + return "aws_redshift_cluster_subnet_group_supported_platforms" +} + +func (c *Client) transformClusterSubnetGroups(values []*redshift.ClusterSubnetGroup) []*ClusterSubnetGroup { + var tValues []*ClusterSubnetGroup + for _, value := range values { + tValue := ClusterSubnetGroup{ + AccountID: c.accountID, + Region: c.region, + Name: value.ClusterSubnetGroupName, + Description: value.Description, + Status: value.SubnetGroupStatus, + Subnets: c.transformClusterSubnetGroupSubnets(value.Subnets), + Tags: c.transformClusterSubnetGroupTags(value.Tags), + VpcId: value.VpcId, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterSubnetGroupSupportedPlatforms(values []*redshift.SupportedPlatform) []*ClusterSubnetGroupSupportedPlatform { + var tValues []*ClusterSubnetGroupSupportedPlatform + for _, value := range values { + tValue := ClusterSubnetGroupSupportedPlatform{ + AccountID: c.accountID, + Region: c.region, + Name: value.Name, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterSubnetGroupSubnets(values []*redshift.Subnet) []*ClusterSubnetGroupSubnet { + var tValues []*ClusterSubnetGroupSubnet + for _, value := range values { + tValue := ClusterSubnetGroupSubnet{ + AccountID: c.accountID, + Region: c.region, + Identifier: value.SubnetIdentifier, + Status: value.SubnetStatus, + } + if value.SubnetAvailabilityZone != nil { + tValue.AZName = value.SubnetAvailabilityZone.Name + tValue.AZPlatforms = c.transformClusterSubnetGroupSupportedPlatforms(value.SubnetAvailabilityZone.SupportedPlatforms) + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformClusterSubnetGroupTags(values []*redshift.Tag) []*ClusterSubnetGroupTag { + var tValues []*ClusterSubnetGroupTag + for _, value := range values { + tValue := ClusterSubnetGroupTag{ + AccountID: c.accountID, + Region: c.region, + Key: value.Key, + Value: value.Value, + } + tValues = append(tValues, &tValue) + } + return tValues +} + +var ClusterSubnetGroupTables = []interface{}{ + &ClusterSubnetGroup{}, + &ClusterSubnetGroupSubnet{}, + &ClusterSubnetGroupSupportedPlatform{}, + &ClusterSubnetGroupTag{}, +} + +func (c *Client) clusterSubnetGroups(gConfig interface{}) error { + var config redshift.DescribeClusterSubnetGroupsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(ClusterSubnetGroupTables...) + + for { + output, err := c.svc.DescribeClusterSubnetGroups(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformClusterSubnetGroups(output.ClusterSubnetGroups)) + c.log.Info("Fetched resources", zap.String("resource", "redshift.cluster_subnet_groups"), zap.Int("count", len(output.ClusterSubnetGroups))) + if aws.StringValue(output.Marker) == "" { + break + } + config.Marker = output.Marker + } + + return nil +} diff --git a/resource/client.go b/resource/client.go new file mode 100644 index 000000000..6a18034a4 --- /dev/null +++ b/resource/client.go @@ -0,0 +1,5 @@ +package resource + +type ClientInterface interface { + CollectResource(resource string, config interface{}) error +} diff --git a/s3/buckets.go b/s3/buckets.go new file mode 100644 index 000000000..4401cb605 --- /dev/null +++ b/s3/buckets.go @@ -0,0 +1,297 @@ +package s3 + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/cloudquery/cloudquery/providers/common" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" + "time" +) + +type Bucket struct { + _ interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"` + ID uint `gorm:"primarykey"` + AccountID string `neo:"unique"` + Region string `neo:"unique"` + CreationDate *time.Time + Name *string `neo:"unique"` + Grants []*BucketGrant `gorm:"constraint:OnDelete:CASCADE;"` + CORSRules []*BucketCorsRule `gorm:"constraint:OnDelete:CASCADE;"` + EncryptionRules []*BucketEncryptionRule `gorm:"constraint:OnDelete:CASCADE;"` + // The bucket policy as a JSON document. + Policy *string + + // Specifies whether MFA delete is enabled in the bucket versioning configuration. + // This element is only returned if the bucket has been configured with MFA + // delete. If the bucket has never been so configured, this element is not returned. + MFADelete *string + + // The versioning state of the bucket. + Status *string + + // Logging + LoggingTargetPrefix *string + LoggingTargetBucket *string +} + +func (Bucket) TableName() string { + return "aws_s3_buckets" +} + +type BucketEncryptionRule struct { + ID uint `gorm:"primarykey"` + BucketID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + KMSMasterKeyID *string + SSEAlgorithm *string +} + +func (BucketEncryptionRule) TableName() string { + return "aws_s3_bucket_encryption_rules" +} + +type BucketGrant struct { + ID uint `gorm:"primarykey"` + BucketID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + // The person being granted permissions. + GranteeDisplayName *string + GranteeEmailAddress *string + GranteeID *string + GranteeType *string + GranteeURI *string + + // Specifies the permission given to the grantee. + Permission *string +} + +func (BucketGrant) TableName() string { + return "aws_s3_bucket_grants" +} + +type BucketCorsRule struct { + ID uint `gorm:"primarykey"` + BucketID uint `neo:"ignore"` + AccountID string `gorm:"-"` + Region string `gorm:"-"` + + // Headers that are specified in the Access-Control-Request-Headers header. + // These headers are allowed in a preflight OPTIONS request. In response to + // any preflight OPTIONS request, Amazon S3 returns any requested headers that + // are allowed. + AllowedHeaders *string + + // An HTTP method that you allow the origin to execute. Valid values are GET, + // PUT, HEAD, POST, and DELETE. + // + // AllowedMethods is a required field + AllowedMethods *string + + // One or more origins you want customers to be able to access the bucket from. + // + // AllowedOrigins is a required field + AllowedOrigins *string + + // One or more headers in the response that you want customers to be able to + // access from their applications (for example, from a JavaScript XMLHttpRequest + // object). + ExposeHeaders *string + + // The time in seconds that your browser is to cache the preflight response + // for the specified resource. + MaxAgeSeconds *int64 +} + +func (BucketCorsRule) TableName() string { + return "aws_s3_bucket_cors_rules" +} + +func (c *Client) transformGrants(values []*s3.Grant) []*BucketGrant { + var tValues []*BucketGrant + for _, v := range values { + tValue := BucketGrant{ + AccountID: c.accountID, + Region: c.region, + Permission: v.Permission, + } + if v.Grantee != nil { + tValue.GranteeDisplayName = v.Grantee.DisplayName + tValue.GranteeType = v.Grantee.Type + tValue.GranteeID = v.Grantee.ID + tValue.GranteeEmailAddress = v.Grantee.EmailAddress + tValue.GranteeURI = v.Grantee.URI + } + tValues = append(tValues, &tValue) + } + return tValues +} + +func (c *Client) transformBucketCorsRules(values []*s3.CORSRule) []*BucketCorsRule { + var tValues []*BucketCorsRule + for _, v := range values { + tValues = append(tValues, &BucketCorsRule{ + AccountID: c.accountID, + Region: c.region, + AllowedHeaders: common.StringListToString(v.AllowedHeaders), + AllowedMethods: common.StringListToString(v.AllowedMethods), + AllowedOrigins: common.StringListToString(v.AllowedOrigins), + ExposeHeaders: common.StringListToString(v.ExposeHeaders), + MaxAgeSeconds: v.MaxAgeSeconds, + }) + } + return tValues +} + +func (c *Client) transformEncryptionRules(values []*s3.ServerSideEncryptionRule) []*BucketEncryptionRule { + var tValues []*BucketEncryptionRule + for _, v := range values { + if v.ApplyServerSideEncryptionByDefault != nil { + tValues = append(tValues, &BucketEncryptionRule{ + AccountID: c.accountID, + Region: c.region, + KMSMasterKeyID: v.ApplyServerSideEncryptionByDefault.KMSMasterKeyID, + SSEAlgorithm: v.ApplyServerSideEncryptionByDefault.SSEAlgorithm, + }) + } + } + return tValues +} + +func (c *Client) transformBucket(value *s3.Bucket) (*Bucket, error) { + loggingOutput, err := c.svc.GetBucketLogging(&s3.GetBucketLoggingInput{Bucket: value.Name}) + if err != nil { + return nil, err + } + + aclOutput, err := c.svc.GetBucketAcl(&s3.GetBucketAclInput{Bucket: value.Name}) + if err != nil { + return nil, err + } + + CORSOutput, err := c.svc.GetBucketCors(&s3.GetBucketCorsInput{ + Bucket: value.Name, + }) + if err != nil && err.(awserr.Error).Code() != "NoSuchCORSConfiguration" { + return nil, err + } + + policyOutput, err := c.svc.GetBucketPolicy(&s3.GetBucketPolicyInput{ + Bucket: value.Name, + }) + if err != nil && err.(awserr.Error).Code() != "NoSuchBucketPolicy" { + return nil, err + } + + versioningOutput, err := c.svc.GetBucketVersioning(&s3.GetBucketVersioningInput{ + Bucket: value.Name, + }) + if err != nil { + return nil, err + } + + encryptionOutput, err := c.svc.GetBucketEncryption(&s3.GetBucketEncryptionInput{ + Bucket: value.Name, + }) + if err != nil { + var aerr awserr.Error + var ok bool + if aerr, ok = err.(awserr.Error); !ok { + return nil, err + } + if aerr.Code() != "ServerSideEncryptionConfigurationNotFoundError" { + return nil, err + } + } + var EncryptionRules []*BucketEncryptionRule + if encryptionOutput.ServerSideEncryptionConfiguration != nil { + EncryptionRules = c.transformEncryptionRules(encryptionOutput.ServerSideEncryptionConfiguration.Rules) + } + + res := Bucket{ + Region: c.region, + AccountID: c.accountID, + CreationDate: value.CreationDate, + Name: value.Name, + Grants: c.transformGrants(aclOutput.Grants), + CORSRules: c.transformBucketCorsRules(CORSOutput.CORSRules), + EncryptionRules: EncryptionRules, + Policy: policyOutput.Policy, + Status: versioningOutput.Status, + MFADelete: versioningOutput.MFADelete, + } + + if loggingOutput.LoggingEnabled != nil { + res.LoggingTargetBucket = loggingOutput.LoggingEnabled.TargetBucket + res.LoggingTargetPrefix = loggingOutput.LoggingEnabled.TargetPrefix + } + + return &res, nil +} + +func (c *Client) transformBuckets(values []*s3.Bucket) ([]*Bucket, error) { + var tValues []*Bucket + for _, v := range values { + output, err := c.svc.GetBucketLocation(&s3.GetBucketLocationInput{ + Bucket: v.Name, + }) + + if err != nil { + if err.(awserr.Error).Code() == "NoSuchBucket" { + // https://aws.amazon.com/premiumsupport/knowledge-center/s3-listing-deleted-bucket/ + // deleted buckets may show up + c.log.Debug("Skipping bucket (already deleted)", zap.String("bucket", *v.Name)) + continue + } + return nil, err + } + c.region = "us-east-1" + if output.LocationConstraint != nil { + // This is a weird corner case by AWS API https://github.com/aws/aws-sdk-net/issues/323#issuecomment-196584538 + c.region = aws.StringValue(output.LocationConstraint) + } + c.awsConfig.Region = aws.String(c.region) + c.svc = s3.New(c.session, c.awsConfig) + + tBucket, err := c.transformBucket(v) + if err != nil { + return nil, err + } + tValues = append(tValues, tBucket) + } + return tValues, nil +} + +var BucketTables = []interface{}{ + &Bucket{}, + &BucketGrant{}, + &BucketCorsRule{}, + &BucketEncryptionRule{}, +} + +func (c *Client) buckets(gConfig interface{}) error { + var config s3.ListBucketsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + + output, err := c.svc.ListBuckets(&config) + if err != nil { + return err + } + c.db.Where("account_id", c.accountID).Delete(BucketTables...) + tBuckets, err := c.transformBuckets(output.Buckets) + if err != nil { + return err + } + c.db.ChunkedCreate(tBuckets) + c.log.Info("Fetched resources", zap.String("resource", "s3.buckets"), zap.Int("count", len(tBuckets))) + + return nil +} diff --git a/s3/client.go b/s3/client.go new file mode 100644 index 000000000..7bce2d4ee --- /dev/null +++ b/s3/client.go @@ -0,0 +1,45 @@ +package s3 + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *s3.S3 + awsConfig *aws.Config +} + +func NewClient(sess *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, _ string) resource.ClientInterface { + globalRegion := "us-east-1" + awsConfig.Region = &globalRegion + return &Client{ + session: sess, + db: db, + log: log, + accountID: accountID, + region: "us-east-1", + svc: s3.New(sess, awsConfig), + awsConfig: awsConfig, + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "buckets": + return c.buckets(config) + default: + return fmt.Errorf("unsupported resource buckets.%s", resource) + } +} diff --git a/sns/client.go b/sns/client.go new file mode 100644 index 000000000..c65941657 --- /dev/null +++ b/sns/client.go @@ -0,0 +1,41 @@ +package sns + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sns" + "github.com/cloudquery/cloudquery/database" + "github.com/cloudquery/cloudquery/providers/aws/resource" + "go.uber.org/zap" +) + +type Client struct { + session *session.Session + db *database.Database + log *zap.Logger + accountID string + region string + svc *sns.SNS +} + +func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger, + accountID string, region string) resource.ClientInterface { + return &Client{ + session: session, + db: db, + log: log, + accountID: accountID, + region: region, + svc: sns.New(session, awsConfig), + } +} + +func (c *Client) CollectResource(resource string, config interface{}) error { + switch resource { + case "subscriptions": + return c.subscriptions(config) + default: + return fmt.Errorf("unsupported resource ec2.%s", resource) + } +} diff --git a/sns/subscriptions.go b/sns/subscriptions.go new file mode 100644 index 000000000..4c4254da5 --- /dev/null +++ b/sns/subscriptions.go @@ -0,0 +1,72 @@ +package sns +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sns" + "github.com/mitchellh/mapstructure" + "go.uber.org/zap" +) + + +type Subscription struct { + ID uint `gorm:"primarykey"` + AccountID string + Region string + Endpoint *string + Owner *string + Protocol *string + SubscriptionArn *string + TopicArn *string +} + +func (Subscription) TableName() string { + return "aws_sns_subscriptions" +} + +func (c *Client) transformSubscriptions(values []*sns.Subscription) []*Subscription { + var tValues []*Subscription + for _, value := range values { + tValue := Subscription { + AccountID: c.accountID, + Region: c.region, + Endpoint: value.Endpoint, + Owner: value.Owner, + Protocol: value.Protocol, + SubscriptionArn: value.SubscriptionArn, + TopicArn: value.TopicArn, + } + tValues = append(tValues, &tValue) + } + return tValues +} +type SubscriptionConfig struct { + Filter string +} + +var SubscriptionTables = []interface{} { + &Subscription{}, +} + +func (c *Client)subscriptions(gConfig interface{}) error { + var config sns.ListSubscriptionsInput + err := mapstructure.Decode(gConfig, &config) + if err != nil { + return err + } + c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(SubscriptionTables...) + + for { + output, err := c.svc.ListSubscriptions(&config) + if err != nil { + return err + } + c.db.ChunkedCreate(c.transformSubscriptions(output.Subscriptions)) + c.log.Info("Fetched resources", zap.String("resource", "sns.subscriptions"), zap.Int("count", len(output.Subscriptions))) + if aws.StringValue(output.NextToken) == "" { + break + } + config.NextToken = output.NextToken + } + + return nil +} +