Support for Embulk v0.1x

.github/workflows/main.yml

@@ -10,15 +10,20 @@ on:
     branches:
       - 'master'
     types: [opened, synchronize]
-  workflow_dispatch:
+  pull_request_target:
+    branches:
+      - 'master'
+    types: [labeled]
 
 jobs:
-  main:
+  test:
     runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
+    if: >
+      ${{
+        github.event_name == 'pull_request' ||
+        (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) ||
+        startsWith(github.ref, 'refs/tags/')
+      }}
     steps:
       - uses: actions/checkout@v1
       - name: Set up JDK 1.8
@@ -29,24 +34,24 @@ jobs:
         # Run ./gradlew spotlessApply when failing on this step.
         run: ./gradlew spotlessCheck
       - run: ./gradlew test
-      - name: Show Current Version
-        run: ./gradlew printVersion
-      - name: Build Gem & Put it into './pkg/' Directory
-        run: ./gradlew gem
 
-      - name: Set up JRuby
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    needs: [ test ]
+    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.ref, 'refs/tags/') }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: jruby
-
-      - name: Publish to GPR
-        if: startsWith( github.ref, 'refs/tags/' ) || github.event_name == 'workflow_dispatch'
-        run: |
-          mkdir -p $HOME/.gem
-          touch $HOME/.gem/credentials
-          chmod 0600 $HOME/.gem/credentials
-          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
-          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} ./pkg/*.gem
-        env:
-          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
-          OWNER: ${{ github.repository_owner }}
+          ruby-version: 2.7
+      - name: push gem
+        uses: trocco-io/push-gem-to-gpr-action@v1
+        with:
+          language: java
+          gem-path: "./build/gems/*.gem"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"

build.gradle

@@ -1,19 +1,18 @@
 plugins {
-    id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "1.5.0"
     id "java"
+    id "org.embulk.embulk-plugins" version "0.5.5"
     id "com.diffplug.spotless" version "5.15.0"
     id "com.palantir.git-version" version "0.13.0"
+    id "com.adarshr.test-logger" version "3.0.0"
 }
-import com.github.jrubygradle.JRubyExec
+
 repositories {
     mavenCentral()
-    jcenter()
-}
-configurations {
-    provided
 }
 
+group = "io.trocco"
+description = "Stores files on Sf Bulk Api."
+
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
@@ -28,81 +27,60 @@ version = {
 
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.9.17"
-    provided "org.embulk:embulk-core:0.9.17"
+    def embulkVersion = "0.10.42"
+    compileOnly("org.embulk:embulk-api:${embulkVersion}")
+    compileOnly("org.embulk:embulk-spi:${embulkVersion}")
+
+    compile "org.embulk:embulk-util-config:0.3.4"
+
     // https://mvnrepository.com/artifact/com.force.api/force-wsc
     compile group: 'com.force.api', name: 'force-wsc', version: '47.0.0'
     // https://mvnrepository.com/artifact/com.force.api/force-partner-api
     compile group: 'com.force.api', name: 'force-partner-api', version: '47.0.0'
     compile group: 'com.google.code.gson', name: 'gson', version: '2.8.5'
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    testCompile "junit:junit:4.+"
-    testImplementation "org.mockito:mockito-inline:4.11.0"
-}
 
-task classpath(type: Copy, dependsOn: ["jar"]) {
-    doFirst { file("classpath").deleteDir() }
-    from (configurations.runtime - configurations.provided + files(jar.archivePath))
-    into "classpath"
-}
-clean { delete "classpath" }
+    testImplementation("org.embulk:embulk-api:${embulkVersion}")
+    testImplementation("org.embulk:embulk-spi:${embulkVersion}")
+    testImplementation "org.embulk:embulk-core:${embulkVersion}"
+    testImplementation "org.embulk:embulk-deps:${embulkVersion}"
+    testImplementation "org.embulk:embulk-junit4:${embulkVersion}"
+    testImplementation "org.embulk:embulk-input-file:${embulkVersion}"
+    testImplementation "org.embulk:embulk-parser-csv:${embulkVersion}"
 
-spotless {
-    java {
-        importOrder()
-        removeUnusedImports()
-        googleJavaFormat()
-    }
-}
-task checkstyle(type: Checkstyle) {
-    classpath = sourceSets.main.output + sourceSets.test.output
-    source = sourceSets.main.allJava + sourceSets.test.allJava
+    testImplementation "junit:junit:4.+"
+    testImplementation "org.mockito:mockito-inline:4.11.0"
+    testImplementation "com.squareup.okhttp3:mockwebserver:4.11.0"
 }
 
-task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
-    jrubyArgs "-S"
-    script "gem"
-    scriptArgs "build", "${project.name}.gemspec"
-    doLast { ant.move(file: "${project.name}-${version}.gem", todir: "pkg") }
+embulkPlugin {
+    mainClass = "org.embulk.output.sf_bulk_api.SfBulkApiOutputPlugin"
+    category = "output"
+    type = "sf_bulk_api"
 }
 
-task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
-    jrubyArgs "-S"
-    script "gem"
-    scriptArgs "push", "pkg/${project.name}-${version}.gem"
+tasks.withType(JavaCompile) {
+    options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation"
+    options.encoding = "UTF-8"
 }
 
-task "package"(dependsOn: ["gemspec", "classpath"]) {
-    doLast {
-        println "> Build succeeded."
-        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
-    }
+test {
+    forkEvery = 1
+    maxParallelForks = 1
 }
 
-task gemspec {
-    ext.gemspecFile = file("${project.name}.gemspec")
-    inputs.file "build.gradle"
-    outputs.file gemspecFile
-    doLast { gemspecFile.write($/
-Gem::Specification.new do |spec|
-  spec.name          = "${project.name}"
-  spec.version       = "${version}"
-  spec.authors       = ["dododo8m"]
-  spec.summary       = %[Sf Bulk Api file output plugin for Embulk]
-  spec.description   = %[Stores files on Sf Bulk Api.]
-  spec.email         = ["[email protected]"]
-  spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = "https://github.com/taichiddt+8m/embulk-output-sf_bulk_api"
-
-  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
-  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
-  spec.require_paths = ["lib"]
+gem {
+    from("LICENSE.txt")
+    authors = ["dododo8m"]
+    email = ["[email protected]"]
+    summary = "Sf Bulk Api file output plugin for Embulk"
+    homepage = "https://github.com/trocco-io/embulk-output-sf_bulk_api"
+    licenses = [ "MIT" ]
+}
 
-  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
-  spec.add_development_dependency 'bundler', ['~> 1.0']
-  spec.add_development_dependency 'rake', ['~> 12.0']
-end
-/$)
+spotless {
+    java {
+        importOrder()
+        removeUnusedImports()
+        googleJavaFormat()
     }
 }
-clean { delete "${project.name}.gemspec" }

gradle/dependency-locks/compileClasspath.lockfile

@@ -0,0 +1,25 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+antlr:antlr:2.7.7
+com.fasterxml.jackson.core:jackson-annotations:2.6.7
+com.fasterxml.jackson.core:jackson-core:2.6.7
+com.fasterxml.jackson.core:jackson-databind:2.6.7.5
+com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
+com.force.api:force-partner-api:47.0.0
+com.force.api:force-wsc:47.0.0
+com.google.code.gson:gson:2.8.5
+commons-beanutils:commons-beanutils:1.9.3
+commons-collections:commons-collections:3.2.2
+commons-logging:commons-logging:1.2
+javax.validation:validation-api:1.1.0.Final
+org.antlr:ST4:4.0.7
+org.antlr:antlr-runtime:3.5
+org.antlr:stringtemplate:3.2.1
+org.codehaus.jackson:jackson-core-asl:1.9.13
+org.codehaus.jackson:jackson-mapper-asl:1.9.13
+org.embulk:embulk-api:0.10.42
+org.embulk:embulk-spi:0.10.42
+org.embulk:embulk-util-config:0.3.4
+org.msgpack:msgpack-core:0.8.24
+org.slf4j:slf4j-api:1.7.30

gradle/dependency-locks/runtimeClasspath.lockfile

@@ -0,0 +1,21 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+antlr:antlr:2.7.7
+com.fasterxml.jackson.core:jackson-annotations:2.6.7
+com.fasterxml.jackson.core:jackson-core:2.6.7
+com.fasterxml.jackson.core:jackson-databind:2.6.7.5
+com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
+com.force.api:force-partner-api:47.0.0
+com.force.api:force-wsc:47.0.0
+com.google.code.gson:gson:2.8.5
+commons-beanutils:commons-beanutils:1.9.3
+commons-collections:commons-collections:3.2.2
+commons-logging:commons-logging:1.2
+javax.validation:validation-api:1.1.0.Final
+org.antlr:ST4:4.0.7
+org.antlr:antlr-runtime:3.5
+org.antlr:stringtemplate:3.2.1
+org.codehaus.jackson:jackson-core-asl:1.9.13
+org.codehaus.jackson:jackson-mapper-asl:1.9.13
+org.embulk:embulk-util-config:0.3.4

src/main/java/org/embulk/output/sf_bulk_api/ErrorHandler.java

@@ -17,7 +17,6 @@
 import java.util.stream.IntStream;
 import org.embulk.spi.Column;
 import org.embulk.spi.Schema;
-import org.embulk.spi.time.Timestamp;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -136,14 +135,15 @@ private Map<String, Object> getObject(final SObject sObject) {
     return map;
   }
 
+  @SuppressWarnings("deprecation") // For the use of org.embulk.spi.time.Timestamp.
   private Object getField(final SObject sObject, final Column column) {
     final Object field = sObject.getField(column.getName());
     if (field == null) {
       return null;
     }
     final String type = column.getType().getName();
     if ("timestamp".equals(type)) {
-      return Timestamp.ofInstant(((Calendar) field).toInstant()).toString();
+      return org.embulk.spi.time.Timestamp.ofInstant(((Calendar) field).toInstant()).toString();
     } else if ("boolean".equals(type)) {
       return Boolean.valueOf(field.toString());
     } else if ("double".equals(type) || "long".equals(type)) {

src/main/java/org/embulk/output/sf_bulk_api/PluginTask.java

@@ -1,8 +1,8 @@
 package org.embulk.output.sf_bulk_api;
 
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.Task;
+import org.embulk.util.config.Config;
+import org.embulk.util.config.ConfigDefault;
+import org.embulk.util.config.Task;
 
 public interface PluginTask extends Task {
   @Config("username")

src/main/java/org/embulk/output/sf_bulk_api/SForceColumnVisitor.java

@@ -2,14 +2,12 @@
 
 import com.sforce.soap.partner.sobject.SObject;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.List;
 import java.util.Locale;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
-import org.embulk.spi.time.Timestamp;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
 
 public class SForceColumnVisitor implements ColumnVisitor {
   private final List<String> fieldsToNull = new ArrayList<>();
@@ -67,23 +65,27 @@ public void stringColumn(Column column) {
     }
   }
 
+  // For the use of org.embulk.spi.time.Timestamp and pageReader.getTimestamp
+  @SuppressWarnings("deprecation")
   @Override
   public void timestampColumn(Column column) {
     if (pageReader.isNull(column)) {
       addFieldsToNull(column);
     } else {
-      Timestamp timestamp = pageReader.getTimestamp(column);
-      DateTime dateTime = new DateTime(timestamp.getEpochSecond() * MILLISECOND, DateTimeZone.UTC);
-      record.addField(column.getName(), dateTime.toCalendar(Locale.ENGLISH));
+      org.embulk.spi.time.Timestamp timestamp = pageReader.getTimestamp(column);
+      Calendar calendar = Calendar.getInstance(Locale.ENGLISH);
+      calendar.setTimeInMillis(timestamp.getInstant().toEpochMilli());
+      record.addField(column.getName(), calendar);
     }
   }
 
+  @SuppressWarnings("deprecation") // For the use of pageReader.getJson
   @Override
   public void jsonColumn(Column column) {
     if (pageReader.isNull(column)) {
       addFieldsToNull(column);
     } else {
-      record.addField(column.getName(), pageReader.getString(column));
+      record.addField(column.getName(), pageReader.getJson(column).toJson());
     }
   }
 

src/main/java/org/embulk/output/sf_bulk_api/SForceTransactionalPageOutput.java

@@ -1,5 +1,7 @@
 package org.embulk.output.sf_bulk_api;
 
+import static org.embulk.output.sf_bulk_api.SfBulkApiOutputPlugin.CONFIG_MAPPER_FACTORY;
+
 import com.sforce.soap.partner.fault.ApiFault;
 import com.sforce.soap.partner.fault.ExceptionCode;
 import com.sforce.soap.partner.fault.UnexpectedErrorFault;
@@ -9,7 +11,6 @@
 import java.util.List;
 import org.apache.commons.collections.CollectionUtils;
 import org.embulk.config.TaskReport;
-import org.embulk.spi.Exec;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.TransactionalPageOutput;
@@ -106,7 +107,7 @@ public void abort() {}
 
   @Override
   public TaskReport commit() {
-    final TaskReport taskReport = Exec.newTaskReport();
+    final TaskReport taskReport = CONFIG_MAPPER_FACTORY.newTaskReport();
     taskReport.set("failed", failed);
     taskReport.set("failures", failures);
     return taskReport;