From 5e6aa18d43b48c4a8f24281f5e1a907d1d742d16 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Fri, 27 Dec 2024 16:46:05 -0800 Subject: [PATCH 1/4] Last one. --- lib/typing/converters/string_converter.go | 34 ++++++++++++++++++++--- lib/typing/values/string.go | 21 +------------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index 6c198ed16..82aaab2fe 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -3,21 +3,28 @@ package converters import ( "encoding/json" "fmt" + "log/slog" + "reflect" "strings" "time" + "github.com/artie-labs/transfer/lib/stringutil" + "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/decimal" "github.com/artie-labs/transfer/lib/typing/ext" ) -type StringConverter interface { +type Converter interface { Convert(value any) (string, error) } -func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { +func GetStringConverter(kd typing.KindDetails) (Converter, error) { switch kd.Kind { + // String + case typing.String.Kind: + return StringConverter{}, nil // Time case typing.Date.Kind: return DateConverter{}, nil @@ -38,10 +45,29 @@ func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { return FloatConverter{}, nil case typing.Struct.Kind: return StructConverter{}, nil + default: + slog.Warn("[GetStringConverter] - Unsupported type", slog.String("kind", kd.Kind)) + return nil, nil + } +} + +type StringConverter struct{} + +func (StringConverter) Convert(value any) (string, error) { + // TODO Simplify this function + isArray := reflect.ValueOf(value).Kind() == reflect.Slice + _, isMap := value.(map[string]any) + // If colVal is either an array or a JSON object, we should run JSON parse. + if isMap || isArray { + colValBytes, err := json.Marshal(value) + if err != nil { + return "", err + } + + return string(colValBytes), nil } - // TODO: Return an error when all the types are implemented. - return nil, nil + return stringutil.EscapeBackslashes(fmt.Sprint(value)), nil } type DateConverter struct{} diff --git a/lib/typing/values/string.go b/lib/typing/values/string.go index 73a3cde83..3414bf457 100644 --- a/lib/typing/values/string.go +++ b/lib/typing/values/string.go @@ -1,11 +1,8 @@ package values import ( - "encoding/json" "fmt" - "reflect" - "github.com/artie-labs/transfer/lib/stringutil" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/converters" ) @@ -20,26 +17,10 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return "", fmt.Errorf("failed to get string converter: %w", err) } + // TODO: Simplify this block if sv != nil { return sv.Convert(colVal) } - // TODO: Move all of this into converter function - switch colKind.Kind { - case typing.String.Kind: - isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice - _, isMap := colVal.(map[string]any) - // If colVal is either an array or a JSON object, we should run JSON parse. - if isMap || isArray { - colValBytes, err := json.Marshal(colVal) - if err != nil { - return "", err - } - - return string(colValBytes), nil - } - - return stringutil.EscapeBackslashes(fmt.Sprint(colVal)), nil - } return fmt.Sprint(colVal), nil } From edb0514904adce534080c40f09d229c2488171e4 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Fri, 27 Dec 2024 16:47:11 -0800 Subject: [PATCH 2/4] Imports. --- lib/typing/converters/string_converter.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index 82aaab2fe..64a470afc 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -8,9 +8,8 @@ import ( "strings" "time" - "github.com/artie-labs/transfer/lib/stringutil" - "github.com/artie-labs/transfer/lib/config/constants" + "github.com/artie-labs/transfer/lib/stringutil" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/decimal" "github.com/artie-labs/transfer/lib/typing/ext" From f24f3143fe68fcbe185fc49aefb26f82905d100b Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Fri, 27 Dec 2024 17:10:21 -0800 Subject: [PATCH 3/4] Clean up and adding Boolean Converter. --- lib/typing/converters/string_converter.go | 25 +++++++++++++++---- .../converters/string_converter_test.go | 20 +++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index 64a470afc..74a45fff8 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -21,10 +21,12 @@ type Converter interface { func GetStringConverter(kd typing.KindDetails) (Converter, error) { switch kd.Kind { - // String + // Base types + case typing.Boolean.Kind: + return BooleanConverter{}, nil case typing.String.Kind: return StringConverter{}, nil - // Time + // Time types case typing.Date.Kind: return DateConverter{}, nil case typing.Time.Kind: @@ -33,23 +35,36 @@ func GetStringConverter(kd typing.KindDetails) (Converter, error) { return TimestampNTZConverter{}, nil case typing.TimestampTZ.Kind: return TimestampTZConverter{}, nil + // Array and struct types case typing.Array.Kind: return ArrayConverter{}, nil - // Numbers + case typing.Struct.Kind: + return StructConverter{}, nil + // Numbers types case typing.EDecimal.Kind: return DecimalConverter{}, nil case typing.Integer.Kind: return IntegerConverter{}, nil case typing.Float.Kind: return FloatConverter{}, nil - case typing.Struct.Kind: - return StructConverter{}, nil + default: slog.Warn("[GetStringConverter] - Unsupported type", slog.String("kind", kd.Kind)) return nil, nil } } +type BooleanConverter struct{} + +func (BooleanConverter) Convert(value any) (string, error) { + booleanValue, isOk := value.(bool) + if !isOk { + return "", fmt.Errorf("failed to cast colVal as boolean, colVal: '%v', type: %T", value, value) + } + + return fmt.Sprint(booleanValue), nil +} + type StringConverter struct{} func (StringConverter) Convert(value any) (string, error) { diff --git a/lib/typing/converters/string_converter_test.go b/lib/typing/converters/string_converter_test.go index c52905bd3..7614fecdb 100644 --- a/lib/typing/converters/string_converter_test.go +++ b/lib/typing/converters/string_converter_test.go @@ -10,6 +10,26 @@ import ( "github.com/artie-labs/transfer/lib/typing/decimal" ) +func TestBooleanConverter_Convert(t *testing.T) { + { + // Not boolean + _, err := BooleanConverter{}.Convert("foo") + assert.ErrorContains(t, err, `failed to cast colVal as boolean, colVal: 'foo', type: string`) + } + { + // True + val, err := BooleanConverter{}.Convert(true) + assert.NoError(t, err) + assert.Equal(t, "true", val) + } + { + // False + val, err := BooleanConverter{}.Convert(false) + assert.NoError(t, err) + assert.Equal(t, "false", val) + } +} + func TestArrayConverter_Convert(t *testing.T) { // Array { From f34ecf91df57ca7550b7accdfe0e1bd975565cf7 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Fri, 27 Dec 2024 19:09:47 -0800 Subject: [PATCH 4/4] Add more tests. --- .../converters/string_converter_test.go | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/lib/typing/converters/string_converter_test.go b/lib/typing/converters/string_converter_test.go index 7614fecdb..ab15b037e 100644 --- a/lib/typing/converters/string_converter_test.go +++ b/lib/typing/converters/string_converter_test.go @@ -3,6 +3,8 @@ package converters import ( "testing" + "github.com/artie-labs/transfer/lib/typing" + "github.com/stretchr/testify/assert" "github.com/artie-labs/transfer/lib/config/constants" @@ -10,6 +12,81 @@ import ( "github.com/artie-labs/transfer/lib/typing/decimal" ) +func TestGetStringConverter(t *testing.T) { + { + // Boolean + converter, err := GetStringConverter(typing.Boolean) + assert.NoError(t, err) + assert.IsType(t, BooleanConverter{}, converter) + } + { + // String + converter, err := GetStringConverter(typing.String) + assert.NoError(t, err) + assert.IsType(t, StringConverter{}, converter) + } + { + // Date + converter, err := GetStringConverter(typing.Date) + assert.NoError(t, err) + assert.IsType(t, DateConverter{}, converter) + } + { + // Time + converter, err := GetStringConverter(typing.Time) + assert.NoError(t, err) + assert.IsType(t, TimeConverter{}, converter) + } + { + // TimestampNTZ + converter, err := GetStringConverter(typing.TimestampNTZ) + assert.NoError(t, err) + assert.IsType(t, TimestampNTZConverter{}, converter) + } + { + // TimestampTZ + converter, err := GetStringConverter(typing.TimestampTZ) + assert.NoError(t, err) + assert.IsType(t, TimestampTZConverter{}, converter) + } + { + // Array + converter, err := GetStringConverter(typing.Array) + assert.NoError(t, err) + assert.IsType(t, ArrayConverter{}, converter) + } + { + // Struct + converter, err := GetStringConverter(typing.Struct) + assert.NoError(t, err) + assert.IsType(t, StructConverter{}, converter) + } + { + // EDecimal + converter, err := GetStringConverter(typing.EDecimal) + assert.NoError(t, err) + assert.IsType(t, DecimalConverter{}, converter) + } + { + // Integer + converter, err := GetStringConverter(typing.Integer) + assert.NoError(t, err) + assert.IsType(t, IntegerConverter{}, converter) + } + { + // Float + converter, err := GetStringConverter(typing.Float) + assert.NoError(t, err) + assert.IsType(t, FloatConverter{}, converter) + } + { + // Invalid + converter, err := GetStringConverter(typing.Invalid) + assert.NoError(t, err) + assert.Nil(t, converter) + } +} + func TestBooleanConverter_Convert(t *testing.T) { { // Not boolean