Skip to content

Commit

Permalink
Add support for Snowflake and BQ in global_connection
Browse files Browse the repository at this point in the history
  • Loading branch information
@b-per
b-per committed Aug 16, 2024
1 parent d60dfdb commit c6f6984
Show file tree
Hide file tree
Showing 7 changed files with 1,016 additions and 172 deletions.
32 changes: 32 additions & 0 deletions examples/resources/dbtcloud_global_connection/resource.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
resource "dbtcloud_global_connection" "snowflake" {
name = "My Snowflake connection"
// we can set Privatelink if needed
private_link_endpoint_id = data.dbtcloud_privatelink_endpoint.my_private_link.id
snowflake = {
account = "my-snowflake-account"
database = "MY_DATABASE"
warehouse = "MY_WAREHOUSE"
client_session_keep_alive = false
allow_sso = true
oauth_client_id = "yourclientid"
oauth_client_secret = "yourclientsecret"
}
}

resource "dbtcloud_global_connection" "bigquery" {
name = "My BigQuery connection"
bigquery = {
gcp_project_id = "my-gcp-project-id"
timeout_seconds = 1000
private_key_id = "my-private-key-id"
private_key = "ABCDEFGHIJKL"
client_email = "my_client_email"
client_id = "my_client_id"
auth_uri = "my_auth_uri"
token_uri = "my_token_uri"
auth_provider_x509_cert_url = "my_auth_provider_x509_cert_url"
client_x509_cert_url = "my_client_x509_cert_url"
application_id = "oauth_application_id"
application_secret = "oauth_secret_id"
}
}
101 changes: 49 additions & 52 deletions pkg/dbt_cloud/global_connection.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,20 @@ import (
"encoding/json"
"fmt"
"net/http"
)

// // TODO: Do we need this? Or do we just remove it?
// type GlobalConnectionType int

// // Declare enum values using iota
// const (
// Snowflake GlobalConnectionType = iota
// BigQuery
// )
"github.com/oapi-codegen/nullable"
)

type GlobalConnectionConfig interface {
AdapterVersion() string
}

type GlobalConnectionCommon struct {
ID *int64 `json:"id,omitempty"`
Name *string `json:"name"`
IsSshTunnelEnabled *bool `json:"is_ssh_tunnel_enabled"`
PrivateLinkEndpointId *int64 `json:"private_link_endpoint_id"`
OauthConfigurationId *int64 `json:"oauth_configuration_id"`
ID *int64 `json:"id,omitempty"`
Name *string `json:"name,omitempty"`
IsSshTunnelEnabled *bool `json:"is_ssh_tunnel_enabled,omitempty"`
PrivateLinkEndpointId nullable.Nullable[string] `json:"private_link_endpoint_id,omitempty"`
OauthConfigurationId *int64 `json:"oauth_configuration_id,omitempty"`
// OauthRedirectUri *string `json:"oauth_redirect_uri"` //those are read-only fields, we could maybe get them as Computed but never send them
// IsConfiguredForNativeOauth bool `json:"is_configured_for_native_oauth"`
}
Expand Down Expand Up @@ -81,7 +74,10 @@ func (c *GlobalConnectionClient[T]) Get(connectionID int64) (*GlobalConnectionCo
return &resp.Data.GlobalConnectionCommon, &resp.Data.Config, nil
}

func (c *GlobalConnectionClient[T]) Create(common GlobalConnectionCommon, config T) (*GlobalConnectionCommon, *T, error) {
func (c *GlobalConnectionClient[T]) Create(
common GlobalConnectionCommon,
config T,
) (*GlobalConnectionCommon, *T, error) {

buffer := new(bytes.Buffer)
enc := json.NewEncoder(buffer)
Expand Down Expand Up @@ -127,7 +123,12 @@ func (c *GlobalConnectionClient[T]) Create(common GlobalConnectionCommon, config
return &resp.Data.GlobalConnectionCommon, &resp.Data.Config, nil

}
func (c *GlobalConnectionClient[T]) Update(connectionID int64, common GlobalConnectionCommon, config T) (*GlobalConnectionCommon, *T, error) {

func (c *GlobalConnectionClient[T]) Update(
connectionID int64,
common GlobalConnectionCommon,
config T,
) (*GlobalConnectionCommon, *T, error) {

buffer := new(bytes.Buffer)
enc := json.NewEncoder(buffer)
Expand Down Expand Up @@ -194,50 +195,46 @@ func (c *Client) DeleteGlobalConnection(connectionID int64) (string, error) {
return "", nil
}

// I originally put pointers everywhere and used omitempty but it prevented us from sending null values
// and differentiating between null and not sending the field at all
// TODO: rename to something else than Pointers now that those fields are not Pointers :-)
// TODO: check if we could reuse the same structure as the other one, with the nullable fields and the omitempty
type SnowflakeConfig struct {
Account *string `json:"account,omitempty"`
Database *string `json:"database,omitempty"`
Warehouse *string `json:"warehouse,omitempty"`
ClientSessionKeepAlive *bool `json:"client_session_keep_alive,omitempty"`
Role *string `json:"role,omitempty"`
AllowSso *bool `json:"allow_sso,omitempty"`
OauthClientID *string `json:"oauth_client_id,omitempty"`
OauthClientSecret *string `json:"oauth_client_secret,omitempty"`
Account *string `json:"account,omitempty"`
Database *string `json:"database,omitempty"`
Warehouse *string `json:"warehouse,omitempty"`
ClientSessionKeepAlive *bool `json:"client_session_keep_alive,omitempty"`
Role nullable.Nullable[string] `json:"role,omitempty"`
AllowSso *bool `json:"allow_sso,omitempty"`
OauthClientID *string `json:"oauth_client_id,omitempty"`
OauthClientSecret *string `json:"oauth_client_secret,omitempty"`
}

func (SnowflakeConfig) AdapterVersion() string {
return "snowflake_v0"
}

type BigQueryConfig struct {
ProjectID *string `json:"project_id,omitempty"`
TimeoutSeconds *int64 `json:"timeout_seconds,omitempty"`
PrivateKeyID *string `json:"private_key_id,omitempty"`
PrivateKey *string `json:"private_key,omitempty"`
ClientEmail *string `json:"client_email,omitempty"`
ClientID *string `json:"client_id,omitempty"`
AuthURI *string `json:"auth_uri,omitempty"`
TokenURI *string `json:"token_uri,omitempty"`
AuthProviderX509CertURL *string `json:"auth_provider_x509_cert_url,omitempty"`
ClientX509CertURL *string `json:"client_x509_cert_url,omitempty"`
Priority *string `json:"priority,omitempty"`
Retries *int64 `json:"retries,omitempty"`
Location *string `json:"location,omitempty"`
MaximumBytesBilled *int64 `json:"maximum_bytes_billed,omitempty"`
ExecutionProject *string `json:"execution_project,omitempty"`
ImpersonateServiceAccount *string `json:"impersonate_service_account,omitempty"`
JobRetryDeadlineSeconds *int64 `json:"job_retry_deadline_seconds,omitempty"`
JobCreationTimeoutSeconds *int64 `json:"job_creation_timeout_seconds,omitempty"`
ApplicationID *string `json:"application_id,omitempty"`
ApplicationSecret *string `json:"application_secret,omitempty"`
GcsBucket *string `json:"gcs_bucket,omitempty"`
DataprocRegion *string `json:"dataproc_region,omitempty"`
DataprocClusterName *string `json:"dataproc_cluster_name,omitempty"`
Scopes []string `json:"scopes,omitempty"`
ProjectID *string `json:"project_id,omitempty"`
TimeoutSeconds *int64 `json:"timeout_seconds,omitempty"`
PrivateKeyID *string `json:"private_key_id,omitempty"`
PrivateKey *string `json:"private_key,omitempty"`
ClientEmail *string `json:"client_email,omitempty"`
ClientID *string `json:"client_id,omitempty"`
AuthURI *string `json:"auth_uri,omitempty"`
TokenURI *string `json:"token_uri,omitempty"`
AuthProviderX509CertURL *string `json:"auth_provider_x509_cert_url,omitempty"`
ClientX509CertURL *string `json:"client_x509_cert_url,omitempty"`
Priority nullable.Nullable[string] `json:"priority,omitempty"`
Retries *int64 `json:"retries,omitempty"` //not nullable because there is a default in the UI
Location nullable.Nullable[string] `json:"location,omitempty"`
MaximumBytesBilled nullable.Nullable[int64] `json:"maximum_bytes_billed,omitempty"`
ExecutionProject nullable.Nullable[string] `json:"execution_project,omitempty"`
ImpersonateServiceAccount nullable.Nullable[string] `json:"impersonate_service_account,omitempty"`
JobRetryDeadlineSeconds nullable.Nullable[int64] `json:"job_retry_deadline_seconds,omitempty"`
JobCreationTimeoutSeconds nullable.Nullable[int64] `json:"job_creation_timeout_seconds,omitempty"`
ApplicationID nullable.Nullable[string] `json:"application_id,omitempty"`
ApplicationSecret nullable.Nullable[string] `json:"application_secret,omitempty"`
GcsBucket nullable.Nullable[string] `json:"gcs_bucket,omitempty"`
DataprocRegion nullable.Nullable[string] `json:"dataproc_region,omitempty"`
DataprocClusterName nullable.Nullable[string] `json:"dataproc_cluster_name,omitempty"`
Scopes []string `json:"scopes,omitempty"` //not nullable because there is a default in the UI
}

func (BigQueryConfig) AdapterVersion() string {
Expand Down
84 changes: 46 additions & 38 deletions pkg/framework/objects/global_connection/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,65 +4,73 @@ import (
"github.com/hashicorp/terraform-plugin-framework/types"
)

type GlobalConnectionResourceModel struct {
CommonConfig
SnowflakeConfig *SnowflakeConfig `tfsdk:"snowflake"`
BigQueryConfig *BigQueryConfig `tfsdk:"bigquery"`
}
var supportedGlobalConfigTypes = []string{"bigquery", "snowflake"}

type CommonConfig struct {
ID types.Int64 `tfsdk:"id"`
// AdapterVersion types.String `tfsdk:"adapter_version"` // TODO(cwalden): needed?
Name types.String `tfsdk:"name"`
IsSshTunnelEnabled types.Bool `tfsdk:"is_ssh_tunnel_enabled"`
PrivateLinkEndpointId types.Int64 `tfsdk:"private_link_endpoint_id"`
OauthConfigurationId types.Int64 `tfsdk:"oauth_configuration_id"`
type GlobalConnectionResourceModel struct {
ID types.Int64 `tfsdk:"id"`
AdapterVersion types.String `tfsdk:"adapter_version"`
Name types.String `tfsdk:"name"`
IsSshTunnelEnabled types.Bool `tfsdk:"is_ssh_tunnel_enabled"`
PrivateLinkEndpointId types.String `tfsdk:"private_link_endpoint_id"`
OauthConfigurationId types.Int64 `tfsdk:"oauth_configuration_id"`
SnowflakeConfig *SnowflakeConfig `tfsdk:"snowflake"`
BigQueryConfig *BigQueryConfig `tfsdk:"bigquery"`
}

type BigQueryConfig struct {
GCPProjectID types.String `tfsdk:"gcp_project_id"`
TimeoutSeconds types.Int64 `tfsdk:"timeout_seconds"`
PrivateKeyID types.String `tfsdk:"private_key_id"`
PrivateKey types.String `tfsdk:"private_key"`
ClientEmail types.String `tfsdk:"client_email"`
ClientID types.String `tfsdk:"client_id"`
AuthURI types.String `tfsdk:"auth_uri"`
TokenURI types.String `tfsdk:"token_uri"`
AuthProviderX509CertURL types.String `tfsdk:"auth_provider_x509_cert_url"`
ClientX509CertURL types.String `tfsdk:"client_x509_cert_url"`
Priority types.String `tfsdk:"priority"`
Retries types.Int64 `tfsdk:"retries"`
Location types.String `tfsdk:"location"`
MaximumBytesBilled types.Int64 `tfsdk:"maximum_bytes_billed"`
ExecutionProject types.String `tfsdk:"execution_project"`
ImpersonateServiceAccount types.String `tfsdk:"impersonate_service_account"`
JobRetryDeadlineSeconds types.Int64 `tfsdk:"job_retry_deadline_seconds"`
JobCreationTimeoutSeconds types.Int64 `tfsdk:"job_creation_timeout_seconds"`
ApplicationID types.String `tfsdk:"application_id"`
ApplicationSecret types.String `tfsdk:"application_secret"`
GcsBucket types.String `tfsdk:"gcs_bucket"`
DataprocRegion types.String `tfsdk:"dataproc_region"`
DataprocClusterName types.String `tfsdk:"dataproc_cluster_name"`
Scopes []types.String `tfsdk:"scopes"`
GCPProjectID types.String `tfsdk:"gcp_project_id"`
TimeoutSeconds types.Int64 `tfsdk:"timeout_seconds"`
PrivateKeyID types.String `tfsdk:"private_key_id"`
PrivateKey types.String `tfsdk:"private_key"`
ClientEmail types.String `tfsdk:"client_email"`
ClientID types.String `tfsdk:"client_id"`
AuthURI types.String `tfsdk:"auth_uri"`
TokenURI types.String `tfsdk:"token_uri"`
AuthProviderX509CertURL types.String `tfsdk:"auth_provider_x509_cert_url"`
ClientX509CertURL types.String `tfsdk:"client_x509_cert_url"`
Retries types.Int64 `tfsdk:"retries"`
Scopes []types.String `tfsdk:"scopes"`
// nullable
Priority types.String `tfsdk:"priority"`
Location types.String `tfsdk:"location"`
MaximumBytesBilled types.Int64 `tfsdk:"maximum_bytes_billed"`
ExecutionProject types.String `tfsdk:"execution_project"`
ImpersonateServiceAccount types.String `tfsdk:"impersonate_service_account"`
JobRetryDeadlineSeconds types.Int64 `tfsdk:"job_retry_deadline_seconds"`
JobCreationTimeoutSeconds types.Int64 `tfsdk:"job_creation_timeout_seconds"`
ApplicationID types.String `tfsdk:"application_id"`
ApplicationSecret types.String `tfsdk:"application_secret"`
GcsBucket types.String `tfsdk:"gcs_bucket"`
DataprocRegion types.String `tfsdk:"dataproc_region"`
DataprocClusterName types.String `tfsdk:"dataproc_cluster_name"`
}

type SnowflakeConfig struct {
Account types.String `tfsdk:"account"`
Database types.String `tfsdk:"database"`
Warehouse types.String `tfsdk:"warehouse"`
ClientSessionKeepAlive types.Bool `tfsdk:"client_session_keep_alive"`
Role types.String `tfsdk:"role"`
AllowSso types.Bool `tfsdk:"allow_sso"`
OauthClientID types.String `tfsdk:"oauth_client_id"`
OauthClientSecret types.String `tfsdk:"oauth_client_secret"`
// nullable
Role types.String `tfsdk:"role"`
}

type DatabricksConfig struct{}

type FabricConfig struct{}
type RedshiftConfig struct{}

type PostgresConfig struct{}

type FabricConfig struct{}

type StarburstConfig struct{}

type SparkConfig struct{}

type SynapseConfig struct{}

type GlobalConnectionDataSourceModel struct {
// TBD, and do we use the same as the for the Resource model?
}
Expand Down
Loading

0 comments on commit c6f6984

Please sign in to comment.