From 6678687a690c596a1de6ee34726d2e048afabff6 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Wed, 30 Mar 2016 17:49:59 +0900 Subject: [PATCH 1/3] Implement ClientConfigurationConfigurable --- .../input/s3/AbstractS3FileInputPlugin.java | 19 +- .../s3/ClientConfigurationConfigurable.java | 421 ++++++++++++++++++ .../TestClientConfigurationConfigurable.java | 145 ++++++ 3 files changed, 571 insertions(+), 14 deletions(-) create mode 100644 embulk-input-s3/src/main/java/org/embulk/input/s3/ClientConfigurationConfigurable.java create mode 100644 embulk-input-s3/src/test/java/org/embulk/input/s3/TestClientConfigurationConfigurable.java diff --git a/embulk-input-s3/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java b/embulk-input-s3/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java index 3f50eae..ddc9cb0 100644 --- a/embulk-input-s3/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java +++ b/embulk-input-s3/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java @@ -1,19 +1,15 @@ package org.embulk.input.s3; import java.util.List; -import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; import java.io.IOException; import java.io.InterruptedIOException; import java.io.InputStream; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; import com.google.common.base.Optional; import com.google.common.base.Throwables; import org.slf4j.Logger; -import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.ListObjectsRequest; @@ -23,7 +19,6 @@ import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.ClientConfiguration; import com.amazonaws.AmazonServiceException; -import com.amazonaws.Protocol; import org.embulk.config.Config; import org.embulk.config.ConfigInject; import org.embulk.config.ConfigDefault; @@ -65,7 +60,9 @@ public interface PluginTask @ConfigDefault("null") public Optional getAccessKeyId(); - // TODO timeout, ssl, etc + @Config("client_config") + @ConfigDefault("{}") + public ClientConfigurationConfigurable.Task getClientConfigurationConfigurableTask(); public FileList getFiles(); public void setFiles(FileList files); @@ -129,14 +126,8 @@ protected AWSCredentialsProvider getCredentialsProvider(PluginTask task) protected ClientConfiguration getClientConfiguration(PluginTask task) { - ClientConfiguration clientConfig = new ClientConfiguration(); - - //clientConfig.setProtocol(Protocol.HTTP); - clientConfig.setMaxConnections(50); // SDK default: 50 - clientConfig.setMaxErrorRetry(3); // SDK default: 3 - clientConfig.setSocketTimeout(8*60*1000); // SDK default: 50*1000 - - return clientConfig; + ClientConfigurationConfigurable.Task configurableTask = task.getClientConfigurationConfigurableTask(); + return ClientConfigurationConfigurable.getClientConfiguration(configurableTask); } private FileList listFiles(PluginTask task) diff --git a/embulk-input-s3/src/main/java/org/embulk/input/s3/ClientConfigurationConfigurable.java b/embulk-input-s3/src/main/java/org/embulk/input/s3/ClientConfigurationConfigurable.java new file mode 100644 index 0000000..6e41c04 --- /dev/null +++ b/embulk-input-s3/src/main/java/org/embulk/input/s3/ClientConfigurationConfigurable.java @@ -0,0 +1,421 @@ +package org.embulk.input.s3; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.google.common.base.Optional; +import org.embulk.config.Config; +import org.embulk.config.ConfigDefault; +import org.embulk.config.ConfigException; +import org.embulk.config.Task; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; + +/** + * Created by takahiro.nakayama on 3/25/16. + * This class is utility class for ClientConfiguration + */ +public abstract class ClientConfigurationConfigurable +{ + public interface Task + extends org.embulk.config.Task + { + @Config("protocol") + @ConfigDefault("null") + Optional getProtocol(); + + @Config("max_connections") + @ConfigDefault("null") // default: 50 + Optional getMaxConnections(); + + @Config("user_agent") + @ConfigDefault("null") + Optional getUserAgent(); + + @Config("local_address") + @ConfigDefault("null") + Optional getLocalAddress(); + + @Config("proxy_host") + @ConfigDefault("null") + Optional getProxyHost(); + + @Config("proxy_port") + @ConfigDefault("null") + Optional getProxyPort(); + + @Config("proxy_username") + @ConfigDefault("null") + Optional getProxyUsername(); + + @Config("proxy_password") + @ConfigDefault("null") + Optional getProxyPassword(); + + @Config("proxy_domain") + @ConfigDefault("null") + Optional getProxyDomain(); + + @Config("proxy_workstation") + @ConfigDefault("null") + Optional getProxyWorkstation(); + + // NOTE: RetryPolicy is a interface + // @Config("retry_policy") + // @ConfigDefault("null") + // Optional getRetryPolicy(); + + @Config("max_error_retry") + @ConfigDefault("null") // default: 3 + Optional getMaxErrorRetry(); + + @Config("socket_timeout") + @ConfigDefault("null") // default: 8*60*1000 + Optional getSocketTimeout(); + + @Config("connection_timeout") + @ConfigDefault("null") + Optional getConnectionTimeout(); + + @Config("request_timeout") + @ConfigDefault("null") + Optional getRequestTimeout(); + + // NOTE: Can use `client_execution_timeout` from v1.10.65 + // @Config("client_execution_timeout") + // @ConfigDefault("null") + // Optional getClientExecutionTimeout(); + + @Config("use_reaper") + @ConfigDefault("null") + Optional getUseReaper(); + + // NOTE: Can use `use_throttle_retries` from v1.10.65 + // @Config("use_throttle_retries") + // @ConfigDefault("null") + // Optional getUseThrottleRetries(); + + @Config("use_gzip") + @ConfigDefault("null") + Optional getUseGzip(); + + @Config("socket_send_buffer_size_hints") // used by SocketBufferSizeHints + @ConfigDefault("null") + Optional getSocketSendBufferSizeHint(); + + @Config("socket_receive_buffer_size_hints") // used by SocketBufferSizeHints + @ConfigDefault("null") + Optional getSocketReceiveBufferSizeHints(); + + @Config("signer_override") + @ConfigDefault("null") + Optional getSignerOverride(); + + @Config("preemptive_basic_proxy_auth") + @ConfigDefault("null") + Optional getPreemptiveBasicProxyAuth(); + + @Config("connection_ttl") + @ConfigDefault("null") + Optional getConnectionTTL(); + + @Config("connection_max_idle_millis") + @ConfigDefault("null") + Optional getConnectionMaxIdleMillis(); + + @Config("use_tcp_keep_alive") + @ConfigDefault("null") + Optional getUseTcpKeepAlive(); + + // NOTE: DnsResolver is a interface + // @Config("dns_resolver") + // @ConfigDefault("null") + // Optional getDnsResolver(); + + @Config("response_metadata_cache_size") + @ConfigDefault("null") + Optional getResponseMetadataCacheSize(); + + @Config("secure_random") + @ConfigDefault("null") + Optional getSecureRandom(); + + @Config("use_expect_continue") + @ConfigDefault("null") + Optional getUseExpectContinue(); + } + + public interface SecureRandomTask + extends org.embulk.config.Task + { + @Config("algorithm") + String getAlgorithm(); + + @Config("provider") + @ConfigDefault("null") + Optional getProvider(); + } + + protected ClientConfigurationConfigurable() + { + } + + // For backward compatibility + public static final int DEFAULT_MAX_CONNECTIONS = 50; // SDK default: 50 + public static final int DEFAULT_MAX_ERROR_RETRY = 3; // SDK default: 3 + public static final int DEFAULT_SOCKET_TIMEOUT = 8*60*1000; // SDK default: 50*1000 + + public static ClientConfiguration getClientConfiguration(Task task) + { + ClientConfiguration clientConfiguration = new ClientConfiguration(); + + setProtocolOrDoNothing(clientConfiguration, task.getProtocol()); + setMaxConnectionsOrDoNothing(clientConfiguration, task.getMaxConnections()); + setUserAgentOrDoNothing(clientConfiguration, task.getUserAgent()); + + setLocalAddressOrDoNothing(clientConfiguration, task.getLocalAddress()); + setProxyHostOrDoNothing(clientConfiguration, task.getProxyHost()); + setProxyPortOrDoNothing(clientConfiguration, task.getProxyPort()); + + setProxyUsernameOrDoNothing(clientConfiguration, task.getProxyUsername()); + setProxyPasswordOrDoNothing(clientConfiguration, task.getProxyPassword()); + setProxyDomainOrDoNothing(clientConfiguration, task.getProxyDomain()); + + setProxyWorkstationOrDoNothing(clientConfiguration, task.getProxyWorkstation()); + setMaxErrorRetryOrDoNothing(clientConfiguration, task.getMaxErrorRetry()); + setSocketTimeoutOrDoNothing(clientConfiguration, task.getSocketTimeout()); + + setConnectionTimeoutOrDoNothing(clientConfiguration, task.getConnectionTimeout()); + setRequestTimeoutOrDoNothing(clientConfiguration, task.getRequestTimeout()); + setUseReaperOrDoNothing(clientConfiguration, task.getUseReaper()); + + setUseGzipOrDoNothing(clientConfiguration, task.getUseGzip()); + setSocketBufferSizeHintsOrDoNothing(clientConfiguration, task.getSocketSendBufferSizeHint(), + task.getSocketReceiveBufferSizeHints()); + + setSignerOverrideOrDoNothing(clientConfiguration, task.getSignerOverride()); + setPreemptiveBasicProxyAuthOrDoNothing(clientConfiguration, task.getPreemptiveBasicProxyAuth()); + setConnectionTTLOrDoNothing(clientConfiguration, task.getConnectionTTL()); + + setConnectionMaxIdleMillisOrDoNothing(clientConfiguration, task.getConnectionMaxIdleMillis()); + setUseTcpKeepAliveOrDoNothing(clientConfiguration, task.getUseTcpKeepAlive()); + setResponseMetadataCacheSizeOrDoNothing(clientConfiguration, task.getResponseMetadataCacheSize()); + + setSecureRandomOrDoNothing(clientConfiguration, task.getSecureRandom()); + setUseExpectContinueOrDoNothing(clientConfiguration, task.getUseExpectContinue()); + + setRetryPolicy(clientConfiguration); + setDnsResolver(clientConfiguration); + + return clientConfiguration; + } + + protected static void setProtocolOrDoNothing(ClientConfiguration clientConfiguration, Optional protocol) + { + if (protocol.isPresent()) { + clientConfiguration.setProtocol(protocol.get()); + } + } + + protected static void setMaxConnectionsOrDoNothing(ClientConfiguration clientConfiguration, Optional maxConnections) + { + clientConfiguration.setMaxConnections(maxConnections.or(DEFAULT_MAX_CONNECTIONS)); + } + + protected static void setUserAgentOrDoNothing(ClientConfiguration clientConfiguration, Optional userAgent) + { + if (userAgent.isPresent()) { + clientConfiguration.setUserAgent(userAgent.get()); + } + } + + protected static void setLocalAddressOrDoNothing(ClientConfiguration clientConfiguration, Optional localAddress) + { + if (localAddress.isPresent()) { + InetAddress inetAddress = getInetAddress(localAddress.get()); + clientConfiguration.setLocalAddress(inetAddress); + } + } + + protected static void setProxyHostOrDoNothing(ClientConfiguration clientConfiguration, Optional proxyHost) + { + if (proxyHost.isPresent()) { + clientConfiguration.setProxyHost(proxyHost.get()); + } + } + + protected static void setProxyPortOrDoNothing(ClientConfiguration clientConfiguration, Optional proxyPort) + { + if (proxyPort.isPresent()) { + clientConfiguration.setProxyPort(proxyPort.get()); + } + } + + protected static void setProxyUsernameOrDoNothing(ClientConfiguration clientConfiguration, Optional proxyUsername) + { + if (proxyUsername.isPresent()) { + clientConfiguration.setProxyUsername(proxyUsername.get()); + } + } + + protected static void setProxyPasswordOrDoNothing(ClientConfiguration clientConfiguration, Optional proxyPassword) + { + if (proxyPassword.isPresent()) { + clientConfiguration.setProxyPassword(proxyPassword.get()); + } + } + + protected static void setProxyDomainOrDoNothing(ClientConfiguration clientConfiguration, Optional proxyDomain) + { + if (proxyDomain.isPresent()) { + clientConfiguration.setProxyDomain(proxyDomain.get()); + } + } + + protected static void setProxyWorkstationOrDoNothing(ClientConfiguration clientConfiguration, Optional proxyWorkstation) + { + if (proxyWorkstation.isPresent()) { + clientConfiguration.setProxyWorkstation(proxyWorkstation.get()); + } + } + + protected static void setMaxErrorRetryOrDoNothing(ClientConfiguration clientConfiguration, Optional maxErrorRetry) + { + clientConfiguration.setMaxErrorRetry(maxErrorRetry.or(DEFAULT_MAX_ERROR_RETRY)); + } + + protected static void setSocketTimeoutOrDoNothing(ClientConfiguration clientConfiguration, Optional socketTimeout) + { + clientConfiguration.setSocketTimeout(socketTimeout.or(DEFAULT_SOCKET_TIMEOUT)); + } + + protected static void setConnectionTimeoutOrDoNothing(ClientConfiguration clientConfiguration, Optional connectionTimeout) + { + if (connectionTimeout.isPresent()) { + clientConfiguration.setConnectionTimeout(connectionTimeout.get()); + } + } + + protected static void setRequestTimeoutOrDoNothing(ClientConfiguration clientConfiguration, Optional requestTimeout) + { + if (requestTimeout.isPresent()) { + clientConfiguration.setRequestTimeout(requestTimeout.get()); + } + } + + protected static void setUseReaperOrDoNothing(ClientConfiguration clientConfiguration, Optional useReaper) + { + if (useReaper.isPresent()) { + clientConfiguration.setUseReaper(useReaper.get()); + } + } + + protected static void setUseGzipOrDoNothing(ClientConfiguration clientConfiguration, Optional useGzip) + { + if (useGzip.isPresent()) { + clientConfiguration.setUseGzip(useGzip.get()); + } + } + + protected static void setSocketBufferSizeHintsOrDoNothing(ClientConfiguration clientConfiguration, + Optional socketSendBufferSizeHint, Optional socketReceiveBufferSizeHint) + { + if (socketSendBufferSizeHint.isPresent() && socketReceiveBufferSizeHint.isPresent()) { + clientConfiguration.setSocketBufferSizeHints(socketSendBufferSizeHint.get(), socketReceiveBufferSizeHint.get()); + } + } + + protected static void setSignerOverrideOrDoNothing(ClientConfiguration clientConfiguration, Optional value) + { + if (value.isPresent()) { + clientConfiguration.setSignerOverride(value.get()); + } + } + + protected static void setPreemptiveBasicProxyAuthOrDoNothing(ClientConfiguration clientConfiguration, Optional preemptiveBasicProxyAuth) + { + if (preemptiveBasicProxyAuth.isPresent()) { + clientConfiguration.setPreemptiveBasicProxyAuth(preemptiveBasicProxyAuth.get()); + } + } + + protected static void setConnectionTTLOrDoNothing(ClientConfiguration clientConfiguration, Optional connectionTTL) + { + if (connectionTTL.isPresent()) { + clientConfiguration.setConnectionTTL(connectionTTL.get()); + } + } + + protected static void setConnectionMaxIdleMillisOrDoNothing(ClientConfiguration clientConfiguration, Optional connectionMaxIdleMillis) + { + if (connectionMaxIdleMillis.isPresent()) { + clientConfiguration.setConnectionMaxIdleMillis(connectionMaxIdleMillis.get()); + } + } + + protected static void setUseTcpKeepAliveOrDoNothing(ClientConfiguration clientConfiguration, Optional useTcpKeepAlive) + { + if (useTcpKeepAlive.isPresent()) { + clientConfiguration.setUseTcpKeepAlive(useTcpKeepAlive.get()); + } + } + + protected static void setResponseMetadataCacheSizeOrDoNothing(ClientConfiguration clientConfiguration, Optional responseMetadataCacheSize) + { + if (responseMetadataCacheSize.isPresent()) { + clientConfiguration.setResponseMetadataCacheSize(responseMetadataCacheSize.get()); + } + } + + protected static void setSecureRandomOrDoNothing(ClientConfiguration clientConfiguration, Optional secureRandomTask) + { + if (secureRandomTask.isPresent()) { + SecureRandom secureRandom = getSecureRandom(secureRandomTask.get()); + clientConfiguration.setSecureRandom(secureRandom); + } + } + + protected static void setUseExpectContinueOrDoNothing(ClientConfiguration clientConfiguration, Optional useExpectContinue) + { + if (useExpectContinue.isPresent()) { + clientConfiguration.setUseExpectContinue(useExpectContinue.get()); + } + } + + protected static void setRetryPolicy(ClientConfiguration clientConfiguration) + { + } + + protected static void setDnsResolver(ClientConfiguration clientConfiguration) + { + } + + + private static InetAddress getInetAddress(String host) + { + try { + return InetAddress.getByName(host); + } + catch (UnknownHostException e) { + throw new ConfigException(e); + } + } + + private static SecureRandom getSecureRandom(SecureRandomTask secureRandomTask) + { + try { + if (secureRandomTask.getProvider().isPresent()) { + return SecureRandom.getInstance(secureRandomTask.getAlgorithm(), secureRandomTask.getProvider().get()); + } + else { + return SecureRandom.getInstance(secureRandomTask.getAlgorithm()); + } + } + catch (NoSuchAlgorithmException | NoSuchProviderException e) { + throw new ConfigException(e); + } + } +} diff --git a/embulk-input-s3/src/test/java/org/embulk/input/s3/TestClientConfigurationConfigurable.java b/embulk-input-s3/src/test/java/org/embulk/input/s3/TestClientConfigurationConfigurable.java new file mode 100644 index 0000000..f4fac0f --- /dev/null +++ b/embulk-input-s3/src/test/java/org/embulk/input/s3/TestClientConfigurationConfigurable.java @@ -0,0 +1,145 @@ +package org.embulk.input.s3; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import org.embulk.EmbulkTestRuntime; +import org.embulk.config.ConfigException; +import org.embulk.config.ConfigSource; +import org.embulk.config.TaskReport; +import org.embulk.config.TaskSource; +import org.embulk.spi.Exec; +import org.embulk.spi.FileInputRunner; +import org.embulk.spi.InputPlugin; +import org.embulk.spi.Schema; +import org.embulk.spi.TestPageBuilderReader; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.List; + +import static org.embulk.input.s3.TestS3FileInputPlugin.parserConfig; +import static org.embulk.input.s3.TestS3FileInputPlugin.schemaConfig; +import static org.junit.Assert.*; +import static org.junit.Assume.assumeNotNull; + +/** + * Created by takahiro.nakayama on 3/30/16. + */ +public class TestClientConfigurationConfigurable +{ + private static String EMBULK_S3_TEST_BUCKET; + private static String EMBULK_S3_TEST_ACCESS_KEY_ID; + private static String EMBULK_S3_TEST_SECRET_ACCESS_KEY; + private static final String EMBULK_S3_TEST_PATH_PREFIX = "embulk_input_s3_test"; + + /* + * This test case requires environment variables: + * EMBULK_S3_TEST_BUCKET + * EMBULK_S3_TEST_ACCESS_KEY_ID + * EMBULK_S3_TEST_SECRET_ACCESS_KEY + * If the variables not set, the test case is skipped. + */ + @BeforeClass + public static void initializeConstantVariables() + { + EMBULK_S3_TEST_BUCKET = System.getenv("EMBULK_S3_TEST_BUCKET"); + EMBULK_S3_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_S3_TEST_ACCESS_KEY_ID"); + EMBULK_S3_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_S3_TEST_SECRET_ACCESS_KEY"); + assumeNotNull(EMBULK_S3_TEST_BUCKET, EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY); + } + + @Rule + public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); + + private S3FileInputPlugin plugin; + private ConfigSource config; + private FileInputRunner runner; + private TestPageBuilderReader.MockPageOutput output; + + @Before + public void createResources() + { + plugin = new S3FileInputPlugin(); + config = runtime.getExec().newConfigSource() + .set("type", "s3") + .set("bucket", EMBULK_S3_TEST_BUCKET) + .set("path_prefix", EMBULK_S3_TEST_PATH_PREFIX) + .set("parser", parserConfig(schemaConfig())); + runner = new FileInputRunner(runtime.getInstance(S3FileInputPlugin.class)); + output = new TestPageBuilderReader.MockPageOutput(); + } + + @Test + public void setOneParam() + { + config.setNested("client_config", Exec.newConfigSource() + .set("protocol", "HTTP") + .set("user_agent", "test_embulk_input_s3")); + + ClientConfiguration clientConfiguration = getClientConfiguration(); + + assertEquals(Protocol.HTTP, clientConfiguration.getProtocol()); + assertEquals("test_embulk_input_s3", clientConfiguration.getUserAgent()); + } + + @Test + public void setTwoParam() + { + config.setNested("client_config", Exec.newConfigSource() + .set("socket_send_buffer_size_hints", 4) + .set("socket_receive_buffer_size_hints", 8)); + + ClientConfiguration clientConfiguration = getClientConfiguration(); + + int[] socketBufferSizeHints = clientConfiguration.getSocketBufferSizeHints(); + assertEquals(4, socketBufferSizeHints[0]); + assertEquals(8, socketBufferSizeHints[1]); + } + + @Test + public void defaultValue() + { + ClientConfiguration clientConfiguration = getClientConfiguration(); + + assertEquals(3, clientConfiguration.getMaxErrorRetry()); + assertEquals(50, clientConfiguration.getMaxConnections()); + assertEquals(8*60*1000, clientConfiguration.getSocketTimeout()); + } + + @Test + public void secureRandom() + throws NoSuchAlgorithmException + { + config.setNested("client_config", Exec.newConfigSource() + .setNested("secure_random", Exec.newConfigSource() + .set("algorithm", "SHA1PRNG") + ) + ); + + ClientConfiguration clientConfiguration = getClientConfiguration(); + + assertEquals(SecureRandom.getInstance("SHA1PRNG").getAlgorithm(), clientConfiguration.getSecureRandom().getAlgorithm()); + } + + @Test(expected = ConfigException.class) + public void secureRandomNoSuchAlgorithmException() + { + config.setNested("client_config", Exec.newConfigSource() + .setNested("secure_random", Exec.newConfigSource() + .set("algorithm", "FOOOOOOO") + ) + ); + + ClientConfiguration clientConfiguration = getClientConfiguration(); + } + + private ClientConfiguration getClientConfiguration() + { + S3FileInputPlugin.S3PluginTask task = config.loadConfig(S3FileInputPlugin.S3PluginTask.class); + return plugin.getClientConfiguration(task); + } +} \ No newline at end of file From 1ec66184205a55a189e06ac17e2499ec6ec0fcb2 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Wed, 30 Mar 2016 18:03:15 +0900 Subject: [PATCH 2/3] Write README about client_config --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index 356003f..a1e5be1 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,60 @@ - **session_token**: session token (string, required) +- **client_config**: configure S3 client config (optional) + + - **protocol**: (enum, `HTTP` or `HTTPS`, optional) + + - **max_connections**: (int, optional) + + - **user_agent** (string, optional) + + - **local_address**: name of a hostname (string, optional) + + - **proxy_host**: name of a hostname (string, optional) + + - **proxy_port**: (int, optional) + + - **proxy_username**: (string, optional) + + - **proxy_password**: (string, optional) + + - **proxy_domain**: (string, optional) + + - **proxy_workstation**: (string, optional) + + - **max_error_retry**: (int, optional) + + - **socket_timeout**: (int, optional) + + - **connection_timeout**: (int, optional) + + - **request_timeout**: (int, optional) + + - **use_reaper**: (boolean, optional) + + - **use_gzip**: (boolean, optional) + + - **signer_override**: (string, optional) + + - **preemptive_basic_proxy_auth**: (boolean, optional) + + - **connection_ttl**: (long, optional) + + - **connection_max_idle_millis**: (long, optional) + + - **use_tcp_keep_alive**: (boolean, optional) + + - **response_metadata_cache_size**: (int, optional) + + - **use_expect_continue**: (boolean, optional) + + - **secure_random**: (optional) + + - **algorithm**: (string, required) + + - **provider**: (string, optional) + * **path_match_pattern**: regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional) * **total_file_count_limit**: maximum number of files to read (integer, optional) From 76140c0421808d13420232c632817bf08fdb708d Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Wed, 30 Mar 2016 18:12:17 +0900 Subject: [PATCH 3/3] Remove unuse code from TestClientConfigurationConfigurable.java --- .../TestClientConfigurationConfigurable.java | 40 ++----------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/embulk-input-s3/src/test/java/org/embulk/input/s3/TestClientConfigurationConfigurable.java b/embulk-input-s3/src/test/java/org/embulk/input/s3/TestClientConfigurationConfigurable.java index f4fac0f..7279ec2 100644 --- a/embulk-input-s3/src/test/java/org/embulk/input/s3/TestClientConfigurationConfigurable.java +++ b/embulk-input-s3/src/test/java/org/embulk/input/s3/TestClientConfigurationConfigurable.java @@ -5,60 +5,28 @@ import org.embulk.EmbulkTestRuntime; import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; -import org.embulk.config.TaskReport; -import org.embulk.config.TaskSource; import org.embulk.spi.Exec; -import org.embulk.spi.FileInputRunner; -import org.embulk.spi.InputPlugin; -import org.embulk.spi.Schema; -import org.embulk.spi.TestPageBuilderReader; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; -import java.util.List; import static org.embulk.input.s3.TestS3FileInputPlugin.parserConfig; import static org.embulk.input.s3.TestS3FileInputPlugin.schemaConfig; -import static org.junit.Assert.*; -import static org.junit.Assume.assumeNotNull; +import static org.junit.Assert.assertEquals; /** * Created by takahiro.nakayama on 3/30/16. */ public class TestClientConfigurationConfigurable { - private static String EMBULK_S3_TEST_BUCKET; - private static String EMBULK_S3_TEST_ACCESS_KEY_ID; - private static String EMBULK_S3_TEST_SECRET_ACCESS_KEY; - private static final String EMBULK_S3_TEST_PATH_PREFIX = "embulk_input_s3_test"; - - /* - * This test case requires environment variables: - * EMBULK_S3_TEST_BUCKET - * EMBULK_S3_TEST_ACCESS_KEY_ID - * EMBULK_S3_TEST_SECRET_ACCESS_KEY - * If the variables not set, the test case is skipped. - */ - @BeforeClass - public static void initializeConstantVariables() - { - EMBULK_S3_TEST_BUCKET = System.getenv("EMBULK_S3_TEST_BUCKET"); - EMBULK_S3_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_S3_TEST_ACCESS_KEY_ID"); - EMBULK_S3_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_S3_TEST_SECRET_ACCESS_KEY"); - assumeNotNull(EMBULK_S3_TEST_BUCKET, EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY); - } - @Rule public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); private S3FileInputPlugin plugin; private ConfigSource config; - private FileInputRunner runner; - private TestPageBuilderReader.MockPageOutput output; @Before public void createResources() @@ -66,11 +34,9 @@ public void createResources() plugin = new S3FileInputPlugin(); config = runtime.getExec().newConfigSource() .set("type", "s3") - .set("bucket", EMBULK_S3_TEST_BUCKET) - .set("path_prefix", EMBULK_S3_TEST_PATH_PREFIX) + .set("bucket", "dummy") + .set("path_prefix", "dummy") .set("parser", parserConfig(schemaConfig())); - runner = new FileInputRunner(runtime.getInstance(S3FileInputPlugin.class)); - output = new TestPageBuilderReader.MockPageOutput(); } @Test