Migrate AWS provider to standalone repository as a plugin

Previous history is at https://github.com/cloudquery/cloudquery

.github/workflows/release.yml

@@ -0,0 +1,24 @@
+name: release
+on:
+  push:
+    tags:
+      - '*'
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+      - run: git fetch --prune --unshallow
+      - name: setup dependencies
+        uses: actions/setup-go@v2
+      - name: release dry run
+        run: make release-dry-run
+      - name: setup release environment
+        run: |-
+          echo "${{secrets.GORELEASER_GITHUB_TOKEN}}" | docker login ghcr.io --username $GITHUB_ACTOR --password-stdin
+          echo 'GITHUB_TOKEN=${{secrets.GORELEASER_GITHUB_TOKEN}}' > .release-env
+      - name: release publish
+        run: make release

.github/workflows/test.yml

@@ -0,0 +1,34 @@
+name: test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches: [ main ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.14
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Build
+      run: go build -v .
+
+    - name: Test
+      run: go test -v ./... --tags=integration
+

.gitignore

@@ -0,0 +1,13 @@
+.DS_store
+crash.log
+.idea
+.vscode
+cloudquery
+cloudquery.db
+cloudquery.zip
+/config.yml
+dist/
+vendor/
+bin/
+.release-env
+cq-provider-aws

.goreleaser.yaml

@@ -0,0 +1,33 @@
+before:
+  hooks:
+    - go mod download
+    - go generate ./...
+builds:
+  - flags:
+      - -buildmode=exe
+    env:
+      - CGO_ENABLED=1
+      - GO111MODULE=on
+      - CC=$PWD/gencc.sh
+      - CXX=$PWD/gencpp.sh
+    main: ./provider.go
+    ldflags:
+     - -s -w -X github.com/cloudquery/cloudquery/cmd.Version={{.Version}} -X github.com/cloudquery/cloudquery/cmd.Commit={{.Commit}} -X github.com/cloudquery/cloudquery/cmd.Date={{.Date}}
+    goos:
+      - windows
+      - linux
+      - darwin
+    goarch:
+      - amd64
+archives:
+  -
+    name_template: "{{ .Binary }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+    format: binary
+checksum:
+  name_template: 'checksums.txt'
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'

Makefile

@@ -0,0 +1,31 @@
+PACKAGE_NAME          := github.com/troian/golang-cross-example
+GOLANG_CROSS_VERSION  ?= v1.15.2
+
+.PHONY: release-dry-run
+release-dry-run:
+	@docker run \
+		--privileged \
+		-e CGO_ENABLED=1 \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		-v `pwd`:/go/src/$(PACKAGE_NAME) \
+		-w /go/src/$(PACKAGE_NAME) \
+		troian/golang-cross:${GOLANG_CROSS_VERSION} \
+		--rm-dist --skip-validate --skip-publish
+
+.PHONY: release
+release:
+	@if [ ! -f ".release-env" ]; then \
+		echo "\033[91m.release-env is required for release\033[0m";\
+		exit 1;\
+	fi
+	docker run \
+		--rm \
+		--privileged \
+		-e CGO_ENABLED=1 \
+		--env-file .release-env \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		-v ~/.docker:/root/.docker \
+		-v `pwd`:/go/src/$(PACKAGE_NAME) \
+		-w /go/src/$(PACKAGE_NAME) \
+		troian/golang-cross:${GOLANG_CROSS_VERSION} \
+		release --rm-dist

autoscaling/client.go

@@ -0,0 +1,41 @@
+package autoscaling
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/cloudquery/cloudquery/database"
+	"github.com/cloudquery/cloudquery/providers/aws/resource"
+	"go.uber.org/zap"
+)
+
+type Client struct {
+	session   *session.Session
+	db        *database.Database
+	log       *zap.Logger
+	accountID string
+	region    string
+	svc       *autoscaling.AutoScaling
+}
+
+func NewClient(session *session.Session, awsConfig *aws.Config, db *database.Database, log *zap.Logger,
+	accountID string, region string) resource.ClientInterface {
+	return &Client{
+		session:   session,
+		db:        db,
+		log:       log,
+		accountID: accountID,
+		region:    region,
+		svc:       autoscaling.New(session, awsConfig),
+	}
+}
+
+func (c *Client) CollectResource(resource string, config interface{}) error {
+	switch resource {
+	case "launch_configurations":
+		return c.launchConfigurations(config)
+	default:
+		return fmt.Errorf("unsupported resource autoscaling.%s", resource)
+	}
+}

autoscaling/launch_configurations.go

@@ -0,0 +1,179 @@
+package autoscaling
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/cloudquery/cloudquery/providers/common"
+	"github.com/mitchellh/mapstructure"
+	"go.uber.org/zap"
+	"time"
+)
+
+type LaunchConfiguration struct {
+	_  interface{} `neo:"raw:MERGE (a:AWSAccount {account_id: $account_id}) MERGE (a) - [:Resource] -> (n)"`
+	ID uint        `gorm:"primarykey"`
+
+	// AWS account id
+	AccountID string
+
+	Region                       string
+	AssociatePublicIpAddress     *bool
+	BlockDeviceMappings          []*LaunchConfigurationBlockDeviceMapping `gorm:"constraint:OnDelete:CASCADE;"`
+	ClassicLinkVPCId             *string
+	ClassicLinkVPCSecurityGroups *string
+	CreatedTime                  *time.Time
+	EbsOptimized                 *bool
+	IamInstanceProfile           *string
+	ImageId                      *string
+	InstanceMonitoringEnabled    *bool
+	InstanceType                 *string
+	KernelId                     *string
+	KeyName                      *string
+
+	LaunchConfigurationARN  *string `neo:"unique"`
+	LaunchConfigurationName *string
+
+	MetadataHttpEndpoint            *string
+	MetadataHttpPutResponseHopLimit *int64
+	MetadataHttpTokens              *string
+
+	PlacementTenancy *string
+	RamdiskId        *string
+	SecurityGroups   *string
+	SpotPrice        *string
+	UserData         *string
+}
+
+func (LaunchConfiguration) TableName() string {
+	return "aws_autoscaling_launch_configurations"
+}
+
+type LaunchConfigurationBlockDeviceMapping struct {
+	ID                    uint `gorm:"primarykey"`
+	LaunchConfigurationID uint `neo:"ignore"`
+
+	AccountID string `gorm:"-"`
+	Region    string `gorm:"-"`
+
+	// The device name exposed to the EC2 instance (for example, /dev/sdh or xvdh).
+	// For more information, see Device Naming on Linux Instances (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html)
+	// in the Amazon EC2 User Guide for Linux Instances.
+	DeviceName *string
+
+	EbsDeleteOnTermination *bool
+	EbsEncrypted           *bool
+	EbsIops                *int64
+	EbsSnapshotId          *string
+	EbsVolumeSize          *int64
+	EbsVolumeType          *string
+
+	// If NoDevice is true for the root device, instances might fail the EC2 health
+	// check. In that case, Amazon EC2 Auto Scaling launches replacement instances.
+	NoDevice *bool
+
+	// The name of the virtual device (for example, ephemeral0).
+	VirtualName *string
+}
+
+func (LaunchConfigurationBlockDeviceMapping) TableName() string {
+	return "aws_autoscaling_launch_configuration_block_device_mapping"
+}
+
+func (c *Client) transformLaunchConfigurationBlockDeviceMapping(value *autoscaling.BlockDeviceMapping) *LaunchConfigurationBlockDeviceMapping {
+	res := LaunchConfigurationBlockDeviceMapping{
+		AccountID:   c.accountID,
+		Region:      c.region,
+		DeviceName:  value.DeviceName,
+		NoDevice:    value.NoDevice,
+		VirtualName: value.VirtualName,
+	}
+	if value.Ebs != nil {
+		res.EbsDeleteOnTermination = value.Ebs.DeleteOnTermination
+		res.EbsEncrypted = value.Ebs.Encrypted
+		res.EbsIops = value.Ebs.Iops
+		res.EbsSnapshotId = value.Ebs.SnapshotId
+		res.EbsVolumeSize = value.Ebs.VolumeSize
+		res.EbsVolumeType = value.Ebs.VolumeType
+	}
+	return &res
+}
+
+func (c *Client) transformLaunchConfigurationBlockDeviceMappings(values []*autoscaling.BlockDeviceMapping) []*LaunchConfigurationBlockDeviceMapping {
+	var tValues []*LaunchConfigurationBlockDeviceMapping
+	for _, v := range values {
+		tValues = append(tValues, c.transformLaunchConfigurationBlockDeviceMapping(v))
+	}
+	return tValues
+}
+
+func (c *Client) transformLaunchConfiguration(value *autoscaling.LaunchConfiguration) *LaunchConfiguration {
+	res := LaunchConfiguration{
+		Region:                       c.region,
+		AccountID:                    c.accountID,
+		AssociatePublicIpAddress:     value.AssociatePublicIpAddress,
+		BlockDeviceMappings:          c.transformLaunchConfigurationBlockDeviceMappings(value.BlockDeviceMappings),
+		ClassicLinkVPCId:             value.ClassicLinkVPCId,
+		ClassicLinkVPCSecurityGroups: common.StringListToString(value.ClassicLinkVPCSecurityGroups),
+		CreatedTime:                  value.CreatedTime,
+		EbsOptimized:                 value.EbsOptimized,
+		IamInstanceProfile:           value.IamInstanceProfile,
+		ImageId:                      value.ImageId,
+		InstanceType:                 value.InstanceType,
+		KernelId:                     value.KernelId,
+		KeyName:                      value.KeyName,
+		LaunchConfigurationARN:       value.LaunchConfigurationARN,
+		LaunchConfigurationName:      value.LaunchConfigurationName,
+		PlacementTenancy:             value.PlacementTenancy,
+		RamdiskId:                    value.RamdiskId,
+		SecurityGroups:               common.StringListToString(value.SecurityGroups),
+		SpotPrice:                    value.SpotPrice,
+		UserData:                     value.UserData,
+	}
+
+	if value.MetadataOptions != nil {
+		res.MetadataHttpEndpoint = value.MetadataOptions.HttpEndpoint
+		res.MetadataHttpPutResponseHopLimit = value.MetadataOptions.HttpPutResponseHopLimit
+		res.MetadataHttpTokens = value.MetadataOptions.HttpTokens
+	}
+
+	if value.InstanceMonitoring != nil {
+		res.InstanceMonitoringEnabled = value.InstanceMonitoring.Enabled
+	}
+
+	return &res
+}
+
+func (c *Client) transformLaunchConfigurations(values []*autoscaling.LaunchConfiguration) []*LaunchConfiguration {
+	var tValues []*LaunchConfiguration
+	for _, v := range values {
+		tValues = append(tValues, c.transformLaunchConfiguration(v))
+	}
+	return tValues
+}
+
+var LaunchConfigurationTables = []interface{}{
+	&LaunchConfiguration{},
+	&LaunchConfigurationBlockDeviceMapping{},
+}
+
+func (c *Client) launchConfigurations(gConfig interface{}) error {
+	var config autoscaling.DescribeLaunchConfigurationsInput
+	err := mapstructure.Decode(gConfig, &config)
+	if err != nil {
+		return err
+	}
+	c.db.Where("region", c.region).Where("account_id", c.accountID).Delete(LaunchConfigurationTables...)
+	for {
+		output, err := c.svc.DescribeLaunchConfigurations(&config)
+		if err != nil {
+			return err
+		}
+		c.db.ChunkedCreate(c.transformLaunchConfigurations(output.LaunchConfigurations))
+		c.log.Info("Fetched resources", zap.String("resource", "auto_scaling.launch_configurations"), zap.Int("count", len(output.LaunchConfigurations)))
+		if aws.StringValue(output.NextToken) == "" {
+			break
+		}
+		config.NextToken = output.NextToken
+	}
+	return nil
+}