diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 48a4805..d533763 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -32,19 +32,19 @@ jobs: run: python -m pip install --upgrade pip setuptools wheel - name: Install Python connector requirements - working-directory: examples/connector/python + working-directory: v1_examples/connector/python run: pip install -r requirements.txt - name: Run Python connector build script - working-directory: examples/connector/python + working-directory: v1_examples/connector/python run: ./build.sh - name: Install Python destination requirements - working-directory: examples/destination/python + working-directory: v1_examples/destination/python run: pip install -r requirements.txt - name: Run Python destination build script - working-directory: examples/destination/python + working-directory: v1_examples/destination/python run: ./build.sh - name: Set up Go @@ -58,45 +58,45 @@ jobs: version: "23.2" - name: Set up protobuf for Go - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 - name: Set up protoc-gen-go - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - name: Set up Go protoc path - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: export PATH="$PATH:$(go env GOPATH)/bin" - name: Verify dependencies - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: go mod verify - name: Build Go connector - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: scripts/build.sh - name: Run go vet - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: go vet ./... - name: Install staticcheck run: go install honnef.co/go/tools/cmd/staticcheck@latest - name: Run staticcheck - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: staticcheck ./... - name: Install golint run: go install golang.org/x/lint/golint@latest - name: Run golint - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: golint ./... - name: Run tests - working-directory: examples/connector/golang + working-directory: v1_examples/connector/golang run: go test -race -vet=off ./... - name: Setup Gradle @@ -105,17 +105,17 @@ jobs: gradle-version: 8.5 - name: Run Java connector copyProtos - working-directory: examples/connector/java + working-directory: v1_examples/connector/java run: gradle copyProtos - name: Run Java connector test with Gradle Wrapper - working-directory: examples/connector/java + working-directory: v1_examples/connector/java run: gradle build - name: Run Java destination copyProtos - working-directory: examples/destination/java + working-directory: v1_examples/destination/java run: gradle copyProtos - name: Run Java destination test with Gradle Wrapper - working-directory: examples/destination/java + working-directory: v1_examples/destination/java run: gradle build diff --git a/.gitignore b/.gitignore index 20377fe..53b3097 100644 --- a/.gitignore +++ b/.gitignore @@ -5,13 +5,13 @@ **/build/ **/golang/main bin/* -examples/**/*.proto -examples/**/*.pb.go -examples/**/*pb2.py -examples/**/*pb2.pyi -examples/**/*pb2_grpc.py +v1_examples/*/**/*.proto +v1_examples/**/*.pb.go +v1_examples/**/*pb2.py +v1_examples/**/*pb2.pyi +v1_examples/**/*pb2_grpc.py destination_run/ connector_run/ **/__pycache__/ -examples/**/node_modules/ -examples/**/package-lock.json \ No newline at end of file +v1_examples/**/node_modules/ +v1_examples/**/package-lock.json \ No newline at end of file diff --git a/README.md b/README.md index 1c01907..d9e0f2a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A way for partners to create connectors and destinations that run on [Fivetran ## Repo Structure This repo consists of example connectors and destinations along with a local testing environment. Connectors and destinations are written in a [gRPC supported language](https://grpc.io/docs/languages/) that can generate a statically linked binary. We recommend Java, Golang, or Rust. -* [Examples](examples/) +* [Examples](v1_examples/) * [Local Testing Tools](tools/) ## Development diff --git a/examples/connector/golang/scripts/copy_protos.sh b/examples/connector/golang/scripts/copy_protos.sh deleted file mode 100755 index a6f1a14..0000000 --- a/examples/connector/golang/scripts/copy_protos.sh +++ /dev/null @@ -1 +0,0 @@ -cp ../../../*.proto proto/ diff --git a/tools/connector-tester/README.md b/tools/connector-tester/README.md index e42673d..1fc8419 100644 --- a/tools/connector-tester/README.md +++ b/tools/connector-tester/README.md @@ -2,7 +2,7 @@ ## Pre-requisites - Docker Desktop >= 4.23.0 or [Rancher Desktop](https://rancherdesktop.io/) >= 1.12.1 -- gRPC server is running for the particular example (see [example readme's](/examples/connector/)) +- gRPC server is running for the particular example (see [example readme's](/v1_examples/connector/)) ## How To Run diff --git a/tools/destination-tester/README.md b/tools/destination-tester/README.md index be38de9..0d1302f 100644 --- a/tools/destination-tester/README.md +++ b/tools/destination-tester/README.md @@ -2,7 +2,7 @@ ## Pre-requisites - Docker Desktop >= 4.23.0 or [Rancher Desktop](https://rancherdesktop.io/) >= 1.12.1 -- gRPC server is running for the particular example (see [example readme's](/examples/destination/)) +- gRPC server is running for the particular example (see [example readme's](/v1_examples/destination/)) ## How To Run diff --git a/v1_examples/common.proto b/v1_examples/common.proto new file mode 100644 index 0000000..2f09d03 --- /dev/null +++ b/v1_examples/common.proto @@ -0,0 +1,133 @@ +syntax = "proto3"; +option optimize_for = SPEED; +option java_multiple_files = true; +option go_package = "fivetran.com/fivetran_sdk"; +package fivetran_sdk; + +import "google/protobuf/timestamp.proto"; + +message ConfigurationFormRequest {} + +message ConfigurationFormResponse { + bool schema_selection_supported = 1; + bool table_selection_supported = 2; + repeated FormField fields = 3; + repeated ConfigurationTest tests = 4; +} + +message FormField { + string name = 1; + string label = 2; + bool required = 3; + optional string description = 4; + oneof type { + TextField text_field = 5; + DropdownField dropdown_field = 6; + ToggleField toggle_field = 7; + } +} + +message DropdownField { + repeated string dropdown_field = 1; +} + +message ToggleField {} + +enum TextField { + PlainText = 0; + Password = 1; + Hidden = 2; +} + +message ConfigurationTest { + string name = 1; // unique identifier for the test + string label = 2; // A few words indicating what we are testing, e.g. 'Connecting to database' +} + +message TestRequest { + string name = 1; + map configuration = 2; +} + +message TestResponse { + oneof response { + bool success = 1; + string failure = 2; + // potential future warning + } +} + +message SchemaList { + repeated Schema schemas = 1; +} + +message TableList { + repeated Table tables = 1; +} + +message Schema { + string name = 1; + repeated Table tables = 2; +} + +enum DataType { + UNSPECIFIED = 0; + BOOLEAN = 1; + SHORT = 2; + INT = 3; + LONG = 4; + DECIMAL = 5; + FLOAT = 6; + DOUBLE = 7; + NAIVE_DATE = 8; + NAIVE_DATETIME = 9; + UTC_DATETIME = 10; + BINARY = 11; + XML = 12; + STRING = 13; + JSON = 14; +} + +message DecimalParams { + uint32 precision = 1; + uint32 scale = 2; +} + +enum OpType { + UPSERT = 0; + UPDATE = 1; + DELETE = 2; + TRUNCATE = 3; +} + +message ValueType { + oneof inner { + bool null = 1; + bool bool = 2; + int32 short = 3; + int32 int = 4; + int64 long = 5; + float float = 6; + double double = 7; + google.protobuf.Timestamp naive_date = 8; + google.protobuf.Timestamp naive_datetime = 9; + google.protobuf.Timestamp utc_datetime = 10; + string decimal = 11; + bytes binary = 12; + string string = 13; + string json = 14; + string xml = 15; + } +} + +message Table { + string name = 1; + repeated Column columns = 2; +} + +message Column { + string name = 1; + DataType type = 2; + bool primary_key = 3; + optional DecimalParams decimal = 4; +} diff --git a/examples/connector/golang/Dockerfile b/v1_examples/connector/golang/Dockerfile similarity index 100% rename from examples/connector/golang/Dockerfile rename to v1_examples/connector/golang/Dockerfile diff --git a/examples/connector/golang/README.md b/v1_examples/connector/golang/README.md similarity index 83% rename from examples/connector/golang/README.md rename to v1_examples/connector/golang/README.md index 4068114..9b8bbb3 100644 --- a/examples/connector/golang/README.md +++ b/v1_examples/connector/golang/README.md @@ -3,7 +3,7 @@ Run all commands from the golang folder root ## Steps ``` -> cd examples/connector/golang +> cd v1_examples/connector/golang > scripts/copy_protos.sh > scripts/compile_protos.sh > scripts/build.sh diff --git a/examples/connector/golang/go.mod b/v1_examples/connector/golang/go.mod similarity index 100% rename from examples/connector/golang/go.mod rename to v1_examples/connector/golang/go.mod diff --git a/examples/connector/golang/go.sum b/v1_examples/connector/golang/go.sum similarity index 100% rename from examples/connector/golang/go.sum rename to v1_examples/connector/golang/go.sum diff --git a/examples/connector/golang/golang_connector/main.go b/v1_examples/connector/golang/golang_connector/main.go similarity index 99% rename from examples/connector/golang/golang_connector/main.go rename to v1_examples/connector/golang/golang_connector/main.go index d900959..94f6916 100644 --- a/examples/connector/golang/golang_connector/main.go +++ b/v1_examples/connector/golang/golang_connector/main.go @@ -36,7 +36,7 @@ func (s *server) Update(in *pb.UpdateRequest, stream pb.Connector_UpdateServer) state := MyState{} json.Unmarshal([]byte(state_json), &state) - message := fmt.Sprintf("config: %s, selection: %s, state_json: %s, mystate: %s", config, selection, state_json, state) + message := fmt.Sprintf("config: %s, selection: %s, state_json: %s, mystate: %+v", config, selection, state_json, state) LogMessage(INFO, message) // -- Send a log message diff --git a/examples/connector/golang/scripts/build.sh b/v1_examples/connector/golang/scripts/build.sh similarity index 100% rename from examples/connector/golang/scripts/build.sh rename to v1_examples/connector/golang/scripts/build.sh diff --git a/examples/connector/golang/scripts/compile_protos.sh b/v1_examples/connector/golang/scripts/compile_protos.sh similarity index 100% rename from examples/connector/golang/scripts/compile_protos.sh rename to v1_examples/connector/golang/scripts/compile_protos.sh diff --git a/v1_examples/connector/golang/scripts/copy_protos.sh b/v1_examples/connector/golang/scripts/copy_protos.sh new file mode 100755 index 0000000..070905a --- /dev/null +++ b/v1_examples/connector/golang/scripts/copy_protos.sh @@ -0,0 +1 @@ +cp ../../*.proto proto/ diff --git a/examples/connector/golang/scripts/run.sh b/v1_examples/connector/golang/scripts/run.sh similarity index 100% rename from examples/connector/golang/scripts/run.sh rename to v1_examples/connector/golang/scripts/run.sh diff --git a/examples/connector/java/.gitattributes b/v1_examples/connector/java/.gitattributes similarity index 100% rename from examples/connector/java/.gitattributes rename to v1_examples/connector/java/.gitattributes diff --git a/examples/connector/java/Dockerfile b/v1_examples/connector/java/Dockerfile similarity index 100% rename from examples/connector/java/Dockerfile rename to v1_examples/connector/java/Dockerfile diff --git a/examples/connector/java/README.md b/v1_examples/connector/java/README.md similarity index 100% rename from examples/connector/java/README.md rename to v1_examples/connector/java/README.md diff --git a/examples/connector/java/build.gradle b/v1_examples/connector/java/build.gradle similarity index 98% rename from examples/connector/java/build.gradle rename to v1_examples/connector/java/build.gradle index a6b0153..1dd25d6 100644 --- a/examples/connector/java/build.gradle +++ b/v1_examples/connector/java/build.gradle @@ -62,7 +62,7 @@ application { } tasks.register('copyProtos', Copy) { - from file("$rootDir/../../..") + from file("$rootDir/../..") into file("src/main/proto/") include "*.proto" } diff --git a/examples/connector/java/gradle/wrapper/gradle-wrapper.jar b/v1_examples/connector/java/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from examples/connector/java/gradle/wrapper/gradle-wrapper.jar rename to v1_examples/connector/java/gradle/wrapper/gradle-wrapper.jar diff --git a/examples/connector/java/gradle/wrapper/gradle-wrapper.properties b/v1_examples/connector/java/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from examples/connector/java/gradle/wrapper/gradle-wrapper.properties rename to v1_examples/connector/java/gradle/wrapper/gradle-wrapper.properties diff --git a/examples/connector/java/gradlew b/v1_examples/connector/java/gradlew similarity index 100% rename from examples/connector/java/gradlew rename to v1_examples/connector/java/gradlew diff --git a/examples/connector/java/gradlew.bat b/v1_examples/connector/java/gradlew.bat similarity index 100% rename from examples/connector/java/gradlew.bat rename to v1_examples/connector/java/gradlew.bat diff --git a/examples/connector/java/settings.gradle b/v1_examples/connector/java/settings.gradle similarity index 100% rename from examples/connector/java/settings.gradle rename to v1_examples/connector/java/settings.gradle diff --git a/examples/connector/java/src/main/java/connector/ConnectorServiceImpl.java b/v1_examples/connector/java/src/main/java/connector/ConnectorServiceImpl.java similarity index 100% rename from examples/connector/java/src/main/java/connector/ConnectorServiceImpl.java rename to v1_examples/connector/java/src/main/java/connector/ConnectorServiceImpl.java diff --git a/examples/connector/java/src/main/java/connector/JavaConnector.java b/v1_examples/connector/java/src/main/java/connector/JavaConnector.java similarity index 100% rename from examples/connector/java/src/main/java/connector/JavaConnector.java rename to v1_examples/connector/java/src/main/java/connector/JavaConnector.java diff --git a/examples/connector/java/src/main/java/connector/State.java b/v1_examples/connector/java/src/main/java/connector/State.java similarity index 100% rename from examples/connector/java/src/main/java/connector/State.java rename to v1_examples/connector/java/src/main/java/connector/State.java diff --git a/examples/connector/nodejs/README.md b/v1_examples/connector/nodejs/README.md similarity index 100% rename from examples/connector/nodejs/README.md rename to v1_examples/connector/nodejs/README.md diff --git a/examples/connector/nodejs/build.sh b/v1_examples/connector/nodejs/build.sh similarity index 98% rename from examples/connector/nodejs/build.sh rename to v1_examples/connector/nodejs/build.sh index 73f6ec8..ff28626 100644 --- a/examples/connector/nodejs/build.sh +++ b/v1_examples/connector/nodejs/build.sh @@ -11,7 +11,7 @@ mkdir -p protos # Copy all .proto files from the parent directory (5 levels up) to the 'protos' directory. # These files are used for defining the gRPC services and messages. -cp ../../../../*.proto protos/ +cp ../../../*.proto protos/ # Return to the previous directory (the project root). cd .. diff --git a/examples/connector/nodejs/package.json b/v1_examples/connector/nodejs/package.json similarity index 100% rename from examples/connector/nodejs/package.json rename to v1_examples/connector/nodejs/package.json diff --git a/examples/connector/nodejs/src/index.js b/v1_examples/connector/nodejs/src/index.js similarity index 100% rename from examples/connector/nodejs/src/index.js rename to v1_examples/connector/nodejs/src/index.js diff --git a/examples/connector/python/README.md b/v1_examples/connector/python/README.md similarity index 100% rename from examples/connector/python/README.md rename to v1_examples/connector/python/README.md diff --git a/examples/connector/python/build.sh b/v1_examples/connector/python/build.sh similarity index 95% rename from examples/connector/python/build.sh rename to v1_examples/connector/python/build.sh index 471ab46..ef92235 100755 --- a/examples/connector/python/build.sh +++ b/v1_examples/connector/python/build.sh @@ -7,7 +7,7 @@ pip install -r requirements.txt # copying protos present in the root of directory to `protos` folder mkdir -p protos -cp ../../../*.proto protos/ +cp ../../*.proto protos/ # Generates the required gRPC Python files using protos into `sdk_pb2` folder mkdir -p sdk_pb2 python -m grpc_tools.protoc \ diff --git a/examples/connector/python/main.py b/v1_examples/connector/python/main.py similarity index 100% rename from examples/connector/python/main.py rename to v1_examples/connector/python/main.py diff --git a/examples/connector/python/requirements.txt b/v1_examples/connector/python/requirements.txt similarity index 100% rename from examples/connector/python/requirements.txt rename to v1_examples/connector/python/requirements.txt diff --git a/examples/connector/python/run.sh b/v1_examples/connector/python/run.sh similarity index 100% rename from examples/connector/python/run.sh rename to v1_examples/connector/python/run.sh diff --git a/v1_examples/connector_sdk.proto b/v1_examples/connector_sdk.proto new file mode 100644 index 0000000..4d906ee --- /dev/null +++ b/v1_examples/connector_sdk.proto @@ -0,0 +1,109 @@ +syntax = "proto3"; +option optimize_for = SPEED; +option java_multiple_files = true; +option go_package = "fivetran.com/fivetran_sdk"; +package fivetran_sdk; + +import "common.proto"; + +// Fivetran (grpc client) <> Connector (grpc server) +service Connector { + rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} + rpc Test (TestRequest) returns (TestResponse) {} + rpc Schema (SchemaRequest) returns (SchemaResponse) {} + rpc Update (UpdateRequest) returns (stream UpdateResponse) {} +} + +message SchemaRequest { + map configuration = 1; +} + +message SchemaResponse { + oneof response { + bool schema_response_not_supported = 1; + SchemaList with_schema = 2; + TableList without_schema = 3; + } + optional bool selection_not_supported = 4; +} + +message UpdateRequest { + map configuration = 1; + optional Selection selection = 2; + optional string state_json = 3; +} + +message Selection { + oneof selection { + TablesWithNoSchema without_schema = 1; + TablesWithSchema with_schema = 2; + } +} + +message TablesWithNoSchema { + repeated TableSelection tables = 1; + bool include_new_tables = 2; +} + +message TablesWithSchema { + repeated SchemaSelection schemas = 1; + bool include_new_schemas = 2; +} + +message SchemaSelection { + bool included = 1; + string schema_name = 2; + repeated TableSelection tables = 3; + bool include_new_tables = 4; +} + +message TableSelection { + bool included = 1; + string table_name = 2; + map columns = 3; + bool include_new_columns = 4; +} + +message UpdateResponse { + oneof response { + LogEntry log_entry = 1; + Operation operation = 2; + } +} + +enum LogLevel { + INFO = 0; + WARNING = 1; + SEVERE = 2; +} + +message LogEntry { + LogLevel level = 1; + string message = 2; +} + +message Operation { + oneof op { + Record record = 1; + SchemaChange schema_change = 2; + Checkpoint checkpoint = 3; + } +} + +message SchemaChange { + oneof change { + SchemaList with_schema = 1; + TableList without_schema = 2; + } +} + +message Record { + optional string schema_name = 1; + string table_name = 2; + OpType type = 3; + map data = 4; +} + +message Checkpoint { + string state_json = 1; +} \ No newline at end of file diff --git a/examples/destination/java/.gitattributes b/v1_examples/destination/java/.gitattributes similarity index 100% rename from examples/destination/java/.gitattributes rename to v1_examples/destination/java/.gitattributes diff --git a/examples/destination/java/Dockerfile b/v1_examples/destination/java/Dockerfile similarity index 100% rename from examples/destination/java/Dockerfile rename to v1_examples/destination/java/Dockerfile diff --git a/examples/destination/java/README.md b/v1_examples/destination/java/README.md similarity index 100% rename from examples/destination/java/README.md rename to v1_examples/destination/java/README.md diff --git a/examples/destination/java/build.gradle b/v1_examples/destination/java/build.gradle similarity index 98% rename from examples/destination/java/build.gradle rename to v1_examples/destination/java/build.gradle index a25f69f..1474354 100644 --- a/examples/destination/java/build.gradle +++ b/v1_examples/destination/java/build.gradle @@ -65,7 +65,7 @@ application { } tasks.register('copyProtos', Copy) { - from file("$rootDir/../../..") + from file("$rootDir/../..") into file("src/main/proto/") include "*.proto" } diff --git a/examples/destination/java/gradle/wrapper/gradle-wrapper.jar b/v1_examples/destination/java/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from examples/destination/java/gradle/wrapper/gradle-wrapper.jar rename to v1_examples/destination/java/gradle/wrapper/gradle-wrapper.jar diff --git a/examples/destination/java/gradle/wrapper/gradle-wrapper.properties b/v1_examples/destination/java/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from examples/destination/java/gradle/wrapper/gradle-wrapper.properties rename to v1_examples/destination/java/gradle/wrapper/gradle-wrapper.properties diff --git a/examples/destination/java/gradlew b/v1_examples/destination/java/gradlew similarity index 100% rename from examples/destination/java/gradlew rename to v1_examples/destination/java/gradlew diff --git a/examples/destination/java/gradlew.bat b/v1_examples/destination/java/gradlew.bat similarity index 100% rename from examples/destination/java/gradlew.bat rename to v1_examples/destination/java/gradlew.bat diff --git a/examples/destination/java/settings.gradle b/v1_examples/destination/java/settings.gradle similarity index 100% rename from examples/destination/java/settings.gradle rename to v1_examples/destination/java/settings.gradle diff --git a/examples/destination/java/src/main/java/destination/DestinationServiceImpl.java b/v1_examples/destination/java/src/main/java/destination/DestinationServiceImpl.java similarity index 100% rename from examples/destination/java/src/main/java/destination/DestinationServiceImpl.java rename to v1_examples/destination/java/src/main/java/destination/DestinationServiceImpl.java diff --git a/examples/destination/java/src/main/java/destination/JavaDestination.java b/v1_examples/destination/java/src/main/java/destination/JavaDestination.java similarity index 100% rename from examples/destination/java/src/main/java/destination/JavaDestination.java rename to v1_examples/destination/java/src/main/java/destination/JavaDestination.java diff --git a/examples/destination/python/README.md b/v1_examples/destination/python/README.md similarity index 100% rename from examples/destination/python/README.md rename to v1_examples/destination/python/README.md diff --git a/examples/destination/python/build.sh b/v1_examples/destination/python/build.sh similarity index 95% rename from examples/destination/python/build.sh rename to v1_examples/destination/python/build.sh index 77bf52c..d318efd 100755 --- a/examples/destination/python/build.sh +++ b/v1_examples/destination/python/build.sh @@ -10,7 +10,7 @@ source destination_run/bin/activate mkdir -p protos # Copy proto files t oprotos directory -cp ../../../*.proto protos/ +cp ../../*.proto protos/ # Install the required packages pip install -r requirements.txt diff --git a/examples/destination/python/main.py b/v1_examples/destination/python/main.py similarity index 100% rename from examples/destination/python/main.py rename to v1_examples/destination/python/main.py diff --git a/examples/destination/python/read_csv.py b/v1_examples/destination/python/read_csv.py similarity index 100% rename from examples/destination/python/read_csv.py rename to v1_examples/destination/python/read_csv.py diff --git a/examples/destination/python/requirements.txt b/v1_examples/destination/python/requirements.txt similarity index 100% rename from examples/destination/python/requirements.txt rename to v1_examples/destination/python/requirements.txt diff --git a/examples/destination/python/run.sh b/v1_examples/destination/python/run.sh similarity index 100% rename from examples/destination/python/run.sh rename to v1_examples/destination/python/run.sh diff --git a/v1_examples/destination_sdk.proto b/v1_examples/destination_sdk.proto new file mode 100644 index 0000000..8c5635b --- /dev/null +++ b/v1_examples/destination_sdk.proto @@ -0,0 +1,134 @@ +syntax = "proto3"; +option optimize_for = SPEED; +option java_multiple_files = true; +option go_package = "fivetran.com/fivetran_sdk"; +package fivetran_sdk; + +import "google/protobuf/timestamp.proto"; +import "common.proto"; + +// Fivetran (grpc client) <> Destination (grpc server) +service Destination { + rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} + rpc Capabilities (CapabilitiesRequest) returns (CapabilitiesResponse) {} + rpc Test (TestRequest) returns (TestResponse) {} + rpc DescribeTable (DescribeTableRequest) returns (DescribeTableResponse) {} + rpc CreateTable(CreateTableRequest) returns (CreateTableResponse) {} + rpc AlterTable(AlterTableRequest) returns (AlterTableResponse) {} + rpc Truncate(TruncateRequest) returns (TruncateResponse) {} + rpc WriteBatch (WriteBatchRequest) returns (WriteBatchResponse) {} +} + +message CapabilitiesRequest {} + +message CapabilitiesResponse { + BatchFileFormat batch_file_format = 1; +} + +message DescribeTableRequest { + map configuration = 1; + string schema_name = 2; + string table_name = 3; +} + +message DescribeTableResponse { + oneof response { + bool not_found = 1; + string failure = 2; + Table table = 3; + } +} + +message CreateTableRequest { + map configuration = 1; + string schema_name = 2; + Table table = 3; +} + +message CreateTableResponse { + oneof response { + bool success = 1; + string failure = 2; + } +} + +message AlterTableRequest { + map configuration = 1; + string schema_name = 2; + Table table = 3; +} + +message AlterTableResponse { + oneof response { + bool success = 1; + string failure = 2; + } +} + +message TruncateRequest { + map configuration = 1; + string schema_name = 2; + string table_name = 3; + string synced_column = 4; + google.protobuf.Timestamp utc_delete_before = 5; + optional SoftTruncate soft = 6; +} + +message SoftTruncate { + string deleted_column = 3; +} + +message TruncateResponse { + oneof response { + bool success = 1; + string failure = 2; + } +} + +message WriteBatchRequest { + map configuration = 1; + string schema_name = 2; + Table table = 3; + map keys = 4; + repeated string replace_files = 5; + repeated string update_files = 6; + repeated string delete_files = 7; + oneof file_params { + CsvFileParams csv = 8; + ParquetFileParams parquet = 9; + } +} + +message CsvFileParams { + Compression compression = 1; + Encryption encryption = 2; + string null_string = 3; + string unmodified_string = 4; +} + +message ParquetFileParams { + Encryption encryption = 1; +} + +enum Encryption { + NONE = 0; + AES = 1; +} + +enum BatchFileFormat { + CSV = 0; + PARQUET = 1; +} + +enum Compression { + OFF = 0; + ZSTD = 1; + GZIP = 2; +} + +message WriteBatchResponse { + oneof response { + bool success = 1; + string failure = 2; + } +} \ No newline at end of file