From 451efb0e4842991905fc109c70f887b0803ebed9 Mon Sep 17 00:00:00 2001 From: chikamura Date: Fri, 5 Jan 2024 13:28:57 +0900 Subject: [PATCH 1/5] embulk v0.9 -> v0.10.0 --- build.gradle | 20 +++++++++++------ .../embulkPluginRuntime.lockfile | 22 +++++++++++++++++++ .../google_ads/GoogleAdsColumnVisitor.java | 16 ++++++-------- .../google_ads/GoogleAdsInputPlugin.java | 22 ++++++++++++------- .../input/google_ads/GoogleAdsReporter.java | 2 +- .../embulk/input/google_ads/PluginTask.java | 9 ++++---- .../google_ads/TestGoogleAdsAccessor.java | 5 ++++- .../TestGoogleAdsColumnVisitor.java | 5 +++++ .../google_ads/TestGoogleAdsReporter.java | 2 +- .../input/google_ads/TestGoogleAdsUtil.java | 6 ++--- .../embulk/input/google_ads/TestHelper.java | 9 ++++++++ 11 files changed, 84 insertions(+), 34 deletions(-) create mode 100644 gradle/dependency-locks/embulkPluginRuntime.lockfile create mode 100644 src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java diff --git a/build.gradle b/build.gradle index c81fccb..59abbf6 100644 --- a/build.gradle +++ b/build.gradle @@ -28,17 +28,23 @@ tasks.withType(JavaCompile) { options.compilerArgs << "-Xlint:deprecation" << "-Xlint:unchecked" } +def embulkVersion = '0.10.43' + dependencies { - compileOnly "org.embulk:embulk-core:0.9.23" - // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION" + compileOnly("org.embulk:embulk-api:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" } + compileOnly("org.embulk:embulk-spi:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" } + compile "org.embulk:embulk-util-timestamp:0.2.1" + compile "org.embulk:embulk-util-config:0.3.3" + compile "org.embulk:embulk-util-json:0.2.2" + compile "com.google.guava:guava:31.1-jre" + compile "org.slf4j:slf4j-log4j12:1.7.36" compile project(path: ":shadow-google-ads-helper", configuration: "shadow") + testCompile "junit:junit:4.13" - testCompile 'org.embulk:embulk-standards:0.9.23' - testCompile 'org.embulk:embulk-test:0.9.23' testCompile "org.mockito:mockito-core:1.10.19" - testCompile "org.embulk:embulk-core:0.9.23:tests" - testCompile "org.embulk:embulk-deps-buffer:0.9.23" - testCompile "org.embulk:embulk-deps-config:0.9.23" + testCompile("org.embulk:embulk-core:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" } + testCompile("org.embulk:embulk-deps:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" } + testCompile("org.embulk:embulk-junit4:$embulkVersion") { exclude group: "org.slf4j", module: "slf4j-api" } } embulkPlugin { diff --git a/gradle/dependency-locks/embulkPluginRuntime.lockfile b/gradle/dependency-locks/embulkPluginRuntime.lockfile new file mode 100644 index 0000000..4110d2e --- /dev/null +++ b/gradle/dependency-locks/embulkPluginRuntime.lockfile @@ -0,0 +1,22 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +ch.qos.reload4j:reload4j:1.2.19 +com.fasterxml.jackson.core:jackson-annotations:2.6.7 +com.fasterxml.jackson.core:jackson-core:2.6.7 +com.fasterxml.jackson.core:jackson-databind:2.6.7.5 +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7 +com.google.code.findbugs:jsr305:3.0.2 +com.google.errorprone:error_prone_annotations:2.11.0 +com.google.guava:failureaccess:1.0.1 +com.google.guava:guava:31.1-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.j2objc:j2objc-annotations:1.3 +javax.validation:validation-api:1.1.0.Final +org.checkerframework:checker-qual:3.12.0 +org.embulk:embulk-util-config:0.3.3 +org.embulk:embulk-util-json:0.2.2 +org.embulk:embulk-util-rubytime:0.3.2 +org.embulk:embulk-util-timestamp:0.2.1 +org.slf4j:slf4j-api:1.7.36 +org.slf4j:slf4j-reload4j:1.7.36 diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java index 2c19ab8..a993802 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java @@ -2,12 +2,13 @@ import com.google.gson.JsonElement; import org.embulk.spi.Column; -import org.embulk.spi.ColumnConfig; + import org.embulk.spi.ColumnVisitor; import org.embulk.spi.PageBuilder; -import org.embulk.spi.json.JsonParser; + import org.embulk.spi.time.Timestamp; -import org.embulk.spi.time.TimestampParser; +import org.embulk.util.config.units.ColumnConfig; +import org.embulk.util.json.JsonParser; import java.util.List; @@ -79,15 +80,12 @@ public void timestampColumn(Column column) String configColumnName = GoogleAdsUtil.escapeColumnName(config.getName(), task); if (configColumnName.equals(column.getName()) && config.getConfigSource() != null - && config.getConfigSource().getObjectNode() != null - && config.getConfigSource().getObjectNode().get("format") != null - && config.getConfigSource().getObjectNode().get("format").isTextual()) { - pattern = config.getConfigSource().getObjectNode().get("format").asText(); + && config.getConfigSource().get(String.class, "format", null) != null) { + pattern = config.getConfigSource().get(String.class, "format", null); break; } } - TimestampParser parser = TimestampParser.of(pattern, "UTC"); - Timestamp result = parser.parse(accessor.get(column.getName())); + Timestamp result = Timestamp.ofString(pattern); pageBuilder.setTimestamp(column, result); } catch (Exception e) { pageBuilder.setNull(column); diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java index e2da5a7..b397571 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java @@ -10,13 +10,15 @@ import org.embulk.config.TaskSource; import org.embulk.spi.Column; -import org.embulk.spi.ColumnConfig; import org.embulk.spi.Exec; import org.embulk.spi.InputPlugin; import org.embulk.spi.PageBuilder; import org.embulk.spi.PageOutput; import org.embulk.spi.Schema; - +import org.embulk.util.config.ConfigMapper; +import org.embulk.util.config.ConfigMapperFactory; +import org.embulk.util.config.TaskMapper; +import org.embulk.util.config.units.ColumnConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,18 +29,20 @@ public class GoogleAdsInputPlugin implements InputPlugin { + private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ConfigMapperFactory.builder().addDefaultModules().build(); private final Logger logger = LoggerFactory.getLogger(GoogleAdsInputPlugin.class); @Override public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control) { - PluginTask task = config.loadConfig(PluginTask.class); + final ConfigMapper configMapper = CONFIG_MAPPER_FACTORY.createConfigMapper(); + final PluginTask task = configMapper.map(config, PluginTask.class); Schema schema = buildSchema(task); int taskCount = 1; // number of run() method calls - return resume(task.dump(), schema, taskCount, control); + return resume(task.toTaskSource(), schema, taskCount, control); } @Override @@ -47,7 +51,7 @@ public ConfigDiff resume(TaskSource taskSource, InputPlugin.Control control) { control.run(taskSource, schema, taskCount); - return Exec.newConfigDiff(); + return CONFIG_MAPPER_FACTORY.newConfigDiff(); } @Override @@ -62,7 +66,8 @@ public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) { - PluginTask task = taskSource.loadTask(PluginTask.class); + final TaskMapper taskMapper = CONFIG_MAPPER_FACTORY.createTaskMapper(); + final PluginTask task = taskMapper.map(taskSource, PluginTask.class); Map result; GoogleAdsReporter reporter = new GoogleAdsReporter(task); @@ -87,13 +92,13 @@ public TaskReport run(TaskSource taskSource, throw e; } - return Exec.newTaskReport(); + return CONFIG_MAPPER_FACTORY.newTaskReport(); } @Override public ConfigDiff guess(ConfigSource config) { - return Exec.newConfigDiff(); + return CONFIG_MAPPER_FACTORY.newConfigDiff(); } public GoogleAdsReporter getClient(PluginTask task) @@ -114,6 +119,7 @@ public Schema buildSchema(PluginTask task) protected PageBuilder getPageBuilder(final Schema schema, final PageOutput output) { + // After　the end of embulk v0.9 support, use Exec.getPageBuilder return new PageBuilder(Exec.getBufferAllocator(), schema, output); } } diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java index bbc0e25..be3e9bb 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsReporter.java @@ -15,8 +15,8 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.GeneratedMessageV3; import com.google.protobuf.util.JsonFormat; -import org.embulk.spi.ColumnConfig; +import org.embulk.util.config.units.ColumnConfig; import org.msgpack.core.annotations.VisibleForTesting; import org.slf4j.Logger; diff --git a/src/main/java/org/embulk/input/google_ads/PluginTask.java b/src/main/java/org/embulk/input/google_ads/PluginTask.java index ced63a8..3e0adfa 100644 --- a/src/main/java/org/embulk/input/google_ads/PluginTask.java +++ b/src/main/java/org/embulk/input/google_ads/PluginTask.java @@ -1,9 +1,10 @@ package org.embulk.input.google_ads; -import org.embulk.config.Config; -import org.embulk.config.ConfigDefault; -import org.embulk.config.Task; -import org.embulk.spi.SchemaConfig; +import org.embulk.util.config.Config; +import org.embulk.util.config.ConfigDefault; + +import org.embulk.util.config.Task; +import org.embulk.util.config.units.SchemaConfig; import java.util.List; import java.util.Optional; diff --git a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java index 960a929..3202ecf 100644 --- a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java +++ b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java @@ -3,6 +3,9 @@ import org.embulk.config.ConfigSource; import org.embulk.spi.InputPlugin; import org.embulk.test.TestingEmbulk; +import org.embulk.util.config.ConfigMapper; +import org.embulk.util.config.ConfigMapperFactory; + import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -24,7 +27,7 @@ public void testAccessor() row.put("name2", "value2"); ConfigSource conf = TestHelper.getBaseConfig(embulk); - GoogleAdsAccessor accessor = new GoogleAdsAccessor(conf.loadConfig(PluginTask.class), row); + GoogleAdsAccessor accessor = new GoogleAdsAccessor(TestHelper.loadTask(conf), row); String result = accessor.get("name"); String notExist = accessor.get("not_exist"); diff --git a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java new file mode 100644 index 0000000..a8f0f14 --- /dev/null +++ b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java @@ -0,0 +1,5 @@ +package org.embulk.input.google_ads; + +public class TestGoogleAdsColumnVisitor { + +} diff --git a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsReporter.java b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsReporter.java index 5d2cc92..8127161 100644 --- a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsReporter.java +++ b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsReporter.java @@ -34,7 +34,7 @@ public void testTraverse() "}"; ObjectMapper mapper = new ObjectMapper(); ConfigSource conf = TestHelper.getBaseConfig(embulk); - GoogleAdsReporter reporter = new GoogleAdsReporter(conf.loadConfig(PluginTask.class)); + GoogleAdsReporter reporter = new GoogleAdsReporter(TestHelper.loadTask(conf)); try { JsonNode jsonNode = mapper.readTree(json); JsonNode resultJson = reporter.traverse(jsonNode); diff --git a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsUtil.java b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsUtil.java index 76c2ab8..ad4173e 100644 --- a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsUtil.java +++ b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsUtil.java @@ -18,7 +18,7 @@ public class TestGoogleAdsUtil public void testEscapeColumnName_not_escaped() { ConfigSource conf = TestHelper.getBaseConfig(embulk); - String escapedName = GoogleAdsUtil.escapeColumnName("ad.ad_group.ad", conf.loadConfig(PluginTask.class)); + String escapedName = GoogleAdsUtil.escapeColumnName("ad.ad_group.ad", TestHelper.loadTask(conf)); Assert.assertEquals("ad.ad_group.ad", escapedName); } @@ -27,7 +27,7 @@ public void testEscapeColumnName_escaped() { ConfigSource conf = TestHelper.getBaseConfig(embulk); conf.set("_replace_dot_in_column", "true"); - String escapedName = GoogleAdsUtil.escapeColumnName("ad.ad_group.ad", conf.loadConfig(PluginTask.class)); + String escapedName = GoogleAdsUtil.escapeColumnName("ad.ad_group.ad", TestHelper.loadTask(conf)); Assert.assertEquals("ad_ad_group_ad", escapedName); } @@ -37,7 +37,7 @@ public void testEscapeColumnName_escaped_with_slash() ConfigSource conf = TestHelper.getBaseConfig(embulk); conf.set("_replace_dot_in_column", "true"); conf.set("_replace_dot_in_column_with", "/"); - String escapedName = GoogleAdsUtil.escapeColumnName("ad.ad_group.ad", conf.loadConfig(PluginTask.class)); + String escapedName = GoogleAdsUtil.escapeColumnName("ad.ad_group.ad", TestHelper.loadTask(conf)); Assert.assertEquals("ad/ad_group/ad", escapedName); } } diff --git a/src/test/java/org/embulk/input/google_ads/TestHelper.java b/src/test/java/org/embulk/input/google_ads/TestHelper.java index 7b2ae7b..edffe2b 100644 --- a/src/test/java/org/embulk/input/google_ads/TestHelper.java +++ b/src/test/java/org/embulk/input/google_ads/TestHelper.java @@ -2,16 +2,25 @@ import org.embulk.config.ConfigSource; import org.embulk.test.TestingEmbulk; +import org.embulk.util.config.ConfigMapper; +import org.embulk.util.config.ConfigMapperFactory; import java.util.ArrayList; import java.util.List; public class TestHelper { + private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ConfigMapperFactory.builder().addDefaultModules().build(); + private TestHelper() { } + public static PluginTask loadTask(ConfigSource conf) { + final ConfigMapper configMapper = CONFIG_MAPPER_FACTORY.createConfigMapper(); + return configMapper.map(conf, PluginTask.class); + } + public static ConfigSource getBaseConfig(TestingEmbulk embulk) { ConfigSource configSource = embulk.newConfig(); From a7c75b635d0a38ec7b84d3e94816d936cda03b43 Mon Sep 17 00:00:00 2001 From: chikamura Date: Fri, 5 Jan 2024 22:51:42 +0900 Subject: [PATCH 2/5] add test --- .../TestGoogleAdsColumnVisitor.java | 137 +++++++++++++++++- .../embulk/input/google_ads/TestHelper.java | 12 +- 2 files changed, 141 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java index a8f0f14..29df99d 100644 --- a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java +++ b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java @@ -1,5 +1,140 @@ package org.embulk.input.google_ads; -public class TestGoogleAdsColumnVisitor { +import com.google.common.collect.ImmutableList; +import org.embulk.EmbulkTestRuntime; +import org.embulk.config.ConfigSource; +import org.embulk.spi.*; +import org.embulk.spi.json.JsonObject; +import org.embulk.spi.json.JsonValue; +import org.embulk.spi.time.Instants; +import org.embulk.spi.time.Timestamp; +import org.embulk.test.TestingEmbulk; +import org.embulk.util.config.units.ColumnConfig; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import java.time.Instant; +import java.util.HashMap; + +public class TestGoogleAdsColumnVisitor +{ + @Rule + public TestingEmbulk embulk = TestingEmbulk.builder() + .registerPlugin(InputPlugin.class, "google_ads", GoogleAdsInputPlugin.class) + .build(); + + @Rule public EmbulkTestRuntime runtime; + + @Before + public void setUp() { + runtime = new EmbulkTestRuntime(); + } + + @Test + public void testStringColumn() + { + PageReader pageReader = visitColumns("100", embulk.newConfig().set("name", "name").set("type", "string")); + Assert.assertTrue(pageReader.nextRecord()); + Assert.assertEquals("100", pageReader.getString(0)); + Assert.assertFalse(pageReader.nextRecord()); + } + + @Test + public void testLongColumn() + { + PageReader pageReader = visitColumns("100", embulk.newConfig().set("name", "name").set("type", "long")); + Assert.assertTrue(pageReader.nextRecord()); + Assert.assertEquals(100, pageReader.getLong(0)); + Assert.assertFalse(pageReader.nextRecord()); + } + + @Test + public void testBooleanColumnTrue() + { + PageReader pageReader = visitColumns("true", embulk.newConfig().set("name", "name").set("type", "boolean")); + Assert.assertTrue(pageReader.nextRecord()); + Assert.assertTrue(pageReader.getBoolean(0)); + Assert.assertFalse(pageReader.nextRecord()); + } + + @Test + public void testBooleanColumnFalse() + { + PageReader pageReader = visitColumns("false", embulk.newConfig().set("name", "name").set("type", "boolean")); + Assert.assertTrue(pageReader.nextRecord()); + Assert.assertFalse(pageReader.getBoolean(0)); + Assert.assertFalse(pageReader.nextRecord()); + } + + @Test + public void testDoubleColumn() + { + PageReader pageReader = visitColumns("1.23", embulk.newConfig().set("name", "name").set("type", "double")); + Assert.assertTrue(pageReader.nextRecord()); + Assert.assertEquals(1.23, pageReader.getDouble(0), 0.01); + Assert.assertFalse(pageReader.nextRecord()); + } + + @Test + public void testTimestampColumn() + { + PageReader pageReader = visitColumns("1970-01-01T09:00:10+09:00", embulk.newConfig().set("name", "name").set("type", "timestamp")); + Assert.assertTrue(pageReader.nextRecord()); + Assert.assertEquals(Instant.ofEpochSecond(10), pageReader.getTimestampInstant(0)); + Assert.assertFalse(pageReader.nextRecord()); + } + + @Test + public void testTimestampColumnWithFormat() + { + PageReader pageReader = visitColumns("1970/01/01 00:00:10", embulk.newConfig().set("name", "name").set("type", "timestamp").set("format", "%Y/%m/%d %H:%M:%S")); + Assert.assertTrue(pageReader.nextRecord()); + Assert.assertEquals(Instant.ofEpochSecond(10), pageReader.getTimestampInstant(0)); + Assert.assertFalse(pageReader.nextRecord()); + } + + @Test + public void testJsonColumn() + { + PageReader pageReader = visitColumns("{\"key0\":\"value0\"}", embulk.newConfig().set("name", "name").set("type", "json")); + Assert.assertTrue(pageReader.nextRecord()); + Assert.assertEquals("{\"key0\":\"value0\"}", pageReader.getJsonValue(0).toJson()); + Assert.assertFalse(pageReader.nextRecord()); + } + + private PageReader visitColumns(String value, ConfigSource configSource) { + ConfigSource conf = TestHelper.getBaseConfigWithFields(embulk, configSource); + + PluginTask task = TestHelper.loadTask(conf); + Schema schema = buildSchema(task); + HashMap row = new HashMap<>(); + row.put("name", value); + GoogleAdsAccessor accessor = new GoogleAdsAccessor(task, row); + final TestPageBuilderReader.MockPageOutput output = new TestPageBuilderReader.MockPageOutput(); + final PageBuilder pageBuilder = new PageBuilderImpl(runtime.getBufferAllocator(), schema, output); + GoogleAdsColumnVisitor columnVisitor = new GoogleAdsColumnVisitor(accessor, pageBuilder, task); + schema.visitColumns(columnVisitor); + pageBuilder.addRecord(); + pageBuilder.finish(); + + Assert.assertEquals(1, output.pages.size()); + + PageReader pageReader = new PageReader(schema); + pageReader.setPage(output.pages.get(0)); + + return pageReader; + } + + private Schema buildSchema(PluginTask task) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (int i = 0; i < task.getFields().size(); i++) { + ColumnConfig columnConfig = task.getFields().getColumn(i); + ColumnConfig escapedColumnConfig = new ColumnConfig(GoogleAdsUtil.escapeColumnName(columnConfig.getName(), task), columnConfig.getType(), columnConfig.getOption()); + builder.add(escapedColumnConfig.toColumn(i)); + } + return new Schema(builder.build()); + } } diff --git a/src/test/java/org/embulk/input/google_ads/TestHelper.java b/src/test/java/org/embulk/input/google_ads/TestHelper.java index edffe2b..65f460f 100644 --- a/src/test/java/org/embulk/input/google_ads/TestHelper.java +++ b/src/test/java/org/embulk/input/google_ads/TestHelper.java @@ -22,6 +22,11 @@ public static PluginTask loadTask(ConfigSource conf) { } public static ConfigSource getBaseConfig(TestingEmbulk embulk) + { + return getBaseConfigWithFields(embulk, embulk.newConfig().set("name", "name").set("type", "string")); + } + + public static ConfigSource getBaseConfigWithFields(TestingEmbulk embulk, ConfigSource... fields) { ConfigSource configSource = embulk.newConfig(); configSource.set("customer_id", "customer_id"); @@ -30,13 +35,6 @@ public static ConfigSource getBaseConfig(TestingEmbulk embulk) configSource.set("refresh_token", "refresh_token"); configSource.set("developer_token", "developer_token"); configSource.set("resource_type", "resource_type"); - ConfigSource field = embulk.newConfig(); - List fields = new ArrayList() - { - }; - field.set("name", "name"); - field.set("type", "string"); - fields.add(field); configSource.set("fields", fields); return configSource; From 9fa76e27783a0aa962dcce64d4e3fa0e01048e74 Mon Sep 17 00:00:00 2001 From: chikamura Date: Sun, 7 Jan 2024 11:59:02 +0900 Subject: [PATCH 3/5] fix timestamp bug --- .../org/embulk/input/google_ads/GoogleAdsColumnVisitor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java index a993802..cafcbd9 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java @@ -6,9 +6,9 @@ import org.embulk.spi.ColumnVisitor; import org.embulk.spi.PageBuilder; -import org.embulk.spi.time.Timestamp; import org.embulk.util.config.units.ColumnConfig; import org.embulk.util.json.JsonParser; +import org.embulk.util.timestamp.TimestampFormatter; import java.util.List; @@ -85,7 +85,8 @@ public void timestampColumn(Column column) break; } } - Timestamp result = Timestamp.ofString(pattern); + TimestampFormatter formatter = TimestampFormatter.builder(pattern, true).build(); + org.embulk.spi.time.Timestamp result = org.embulk.spi.time.Timestamp.ofInstant(formatter.parse(accessor.get(column.getName()))); pageBuilder.setTimestamp(column, result); } catch (Exception e) { pageBuilder.setNull(column); From 571241f2d00a01dc2da77b8f755f49c10814b206 Mon Sep 17 00:00:00 2001 From: chikamura Date: Sun, 7 Jan 2024 11:59:36 +0900 Subject: [PATCH 4/5] suppress deprecation warning --- .../org/embulk/input/google_ads/GoogleAdsColumnVisitor.java | 1 + .../org/embulk/input/google_ads/GoogleAdsInputPlugin.java | 2 +- .../embulk/input/google_ads/TestGoogleAdsColumnVisitor.java | 5 +---- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java index cafcbd9..9195c58 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsColumnVisitor.java @@ -70,6 +70,7 @@ public void doubleColumn(Column column) } } + @SuppressWarnings("deprecation") // After the end of embulk v0.9 support, replace Timestamp with Instant @Override public void timestampColumn(Column column) { diff --git a/src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java b/src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java index b397571..f547873 100644 --- a/src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java +++ b/src/main/java/org/embulk/input/google_ads/GoogleAdsInputPlugin.java @@ -117,9 +117,9 @@ public Schema buildSchema(PluginTask task) return new Schema(builder.build()); } + @SuppressWarnings("deprecation") // After　the end of embulk v0.9 support, use Exec.getPageBuilder protected PageBuilder getPageBuilder(final Schema schema, final PageOutput output) { - // After　the end of embulk v0.9 support, use Exec.getPageBuilder return new PageBuilder(Exec.getBufferAllocator(), schema, output); } } diff --git a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java index 29df99d..2989503 100644 --- a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java +++ b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java @@ -4,10 +4,6 @@ import org.embulk.EmbulkTestRuntime; import org.embulk.config.ConfigSource; import org.embulk.spi.*; -import org.embulk.spi.json.JsonObject; -import org.embulk.spi.json.JsonValue; -import org.embulk.spi.time.Instants; -import org.embulk.spi.time.Timestamp; import org.embulk.test.TestingEmbulk; import org.embulk.util.config.units.ColumnConfig; import org.junit.Assert; @@ -121,6 +117,7 @@ private PageReader visitColumns(String value, ConfigSource configSource) { Assert.assertEquals(1, output.pages.size()); + @SuppressWarnings("deprecation") // embulk v0.9-compatible PageReader pageReader = new PageReader(schema); pageReader.setPage(output.pages.get(0)); From ff4219dbbd47e60c79d9974a64cec4e70bcffe9d Mon Sep 17 00:00:00 2001 From: chikamura Date: Sun, 7 Jan 2024 12:36:21 +0900 Subject: [PATCH 5/5] refactoring --- .../input/google_ads/TestGoogleAdsAccessor.java | 2 -- .../google_ads/TestGoogleAdsColumnVisitor.java | 14 +++++++++++--- .../org/embulk/input/google_ads/TestHelper.java | 6 ++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java index 3202ecf..1379602 100644 --- a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java +++ b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsAccessor.java @@ -3,8 +3,6 @@ import org.embulk.config.ConfigSource; import org.embulk.spi.InputPlugin; import org.embulk.test.TestingEmbulk; -import org.embulk.util.config.ConfigMapper; -import org.embulk.util.config.ConfigMapperFactory; import org.junit.Assert; import org.junit.Rule; diff --git a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java index 2989503..a1ec51e 100644 --- a/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java +++ b/src/test/java/org/embulk/input/google_ads/TestGoogleAdsColumnVisitor.java @@ -3,7 +3,13 @@ import com.google.common.collect.ImmutableList; import org.embulk.EmbulkTestRuntime; import org.embulk.config.ConfigSource; -import org.embulk.spi.*; +import org.embulk.spi.Column; +import org.embulk.spi.InputPlugin; +import org.embulk.spi.PageBuilder; +import org.embulk.spi.PageBuilderImpl; +import org.embulk.spi.PageReader; +import org.embulk.spi.Schema; +import org.embulk.spi.TestPageBuilderReader; import org.embulk.test.TestingEmbulk; import org.embulk.util.config.units.ColumnConfig; import org.junit.Assert; @@ -24,7 +30,8 @@ public class TestGoogleAdsColumnVisitor @Rule public EmbulkTestRuntime runtime; @Before - public void setUp() { + public void setUp() + { runtime = new EmbulkTestRuntime(); } @@ -100,7 +107,8 @@ public void testJsonColumn() Assert.assertFalse(pageReader.nextRecord()); } - private PageReader visitColumns(String value, ConfigSource configSource) { + private PageReader visitColumns(String value, ConfigSource configSource) + { ConfigSource conf = TestHelper.getBaseConfigWithFields(embulk, configSource); PluginTask task = TestHelper.loadTask(conf); diff --git a/src/test/java/org/embulk/input/google_ads/TestHelper.java b/src/test/java/org/embulk/input/google_ads/TestHelper.java index 65f460f..92dd8d3 100644 --- a/src/test/java/org/embulk/input/google_ads/TestHelper.java +++ b/src/test/java/org/embulk/input/google_ads/TestHelper.java @@ -5,9 +5,6 @@ import org.embulk.util.config.ConfigMapper; import org.embulk.util.config.ConfigMapperFactory; -import java.util.ArrayList; -import java.util.List; - public class TestHelper { private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ConfigMapperFactory.builder().addDefaultModules().build(); @@ -16,7 +13,8 @@ private TestHelper() { } - public static PluginTask loadTask(ConfigSource conf) { + public static PluginTask loadTask(ConfigSource conf) + { final ConfigMapper configMapper = CONFIG_MAPPER_FACTORY.createConfigMapper(); return configMapper.map(conf, PluginTask.class); }