diff --git a/data-layers/data-layer-api-caller/pom.xml b/data-layers/data-layer-api-caller/pom.xml new file mode 100644 index 0000000..b1a81ad --- /dev/null +++ b/data-layers/data-layer-api-caller/pom.xml @@ -0,0 +1,65 @@ + + 4.0.0 + + aesop + com.flipkart.aesop + 1.2.1-SNAPSHOT + ../../pom.xml + + data-layer-api-caller + + + org.springframework + spring-jdbc + ${org.springframework.version} + + + + com.flipkart.aesop + runtime-client-cluster + 1.2.1-SNAPSHOT + + + com.flipkart.aesop + client-event-consumer + 1.2.1-SNAPSHOT + + + com.google.code.gson + gson + 2.2.2 + + + flipkart.scm + java-cas-client + 0.0.17.4-auth + + + com.ning + async-http-client + 1.8.15 + + + com.flipkart.kloud.authn + client + 15.07.17.1 + + + com.flipkart.kloud.authn + dropwizard8-relying-party + 15.07.17.1 + + + javax.ws.rs + javax.ws.rs-api + 2.0 + + + flipkart.platform + prometheus + 2.3.2 + + + + + \ No newline at end of file diff --git a/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/ApiCallerDataLayer.java b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/ApiCallerDataLayer.java new file mode 100644 index 0000000..87c6a2a --- /dev/null +++ b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/ApiCallerDataLayer.java @@ -0,0 +1,73 @@ +package com.flipkart.aesop.apicallerdatalayer; + +import com.flipkart.aesop.apicallerdatalayer.client.HttpPostClient; +import com.flipkart.aesop.apicallerdatalayer.headers.ContextualHeaderProvider; +import com.flipkart.aesop.apicallerdatalayer.headers.StaticHeaderProvider; +import com.flipkart.aesop.event.AbstractEvent; +import com.flipkart.aesop.processor.DestinationEventProcessor; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.linkedin.databus.client.pub.ConsumerCallbackResult; +import org.trpr.platform.core.impl.logging.LogFactory; +import org.trpr.platform.core.spi.logging.Logger; + +import javax.naming.OperationNotSupportedException; +import javax.ws.rs.core.Response; +import java.util.HashMap; +import java.util.Map; + +/** + * Created by aman.gupta on 08/12/15. + */ +public class ApiCallerDataLayer implements DestinationEventProcessor{ + private String url; + private StaticHeaderProvider staticHeaderProvider; + private ContextualHeaderProvider contextualHeaderProvider; + private HttpPostClient httpPostClient; + private static final Logger LOGGER = LogFactory.getLogger(ApiCallerDataLayer.class); + + public void setStaticHeaderProvider(StaticHeaderProvider staticHeaderProvider) { + this.staticHeaderProvider = staticHeaderProvider; + } + + public void setContextualHeaderProvider(ContextualHeaderProvider contextualHeaderProvider) { + this.contextualHeaderProvider = contextualHeaderProvider; + } + + public void setHttpPostClient(HttpPostClient httpPostClient) { + this.httpPostClient = httpPostClient; + } + + public void setUrl(String url) { + this.url = url; + } + + + @Override + public ConsumerCallbackResult processDestinationEvent(AbstractEvent destinationEvent) throws OperationNotSupportedException { + try { + Map eventMap = destinationEvent.getFieldMapPair(); + Gson gson = new GsonBuilder().serializeNulls().create(); + String payload = gson.toJson(eventMap); + Map headers = new HashMap(); + headers.putAll(staticHeaderProvider.getHeaders()); + headers.putAll(contextualHeaderProvider.getHeaders(eventMap)); + headers.put("Content-Type", "application/json"); + LOGGER.info("Making a post call to url: " + url + " with payload as: " + payload + " and headers as: " + headers); + Response response = httpPostClient.post(url, payload, headers); + int responseCode = response.getStatus(); + if (responseCode >= 200 && responseCode < 300) { + LOGGER.info("Call successful with response code as " + responseCode + " for payload: " + payload); + return ConsumerCallbackResult.SUCCESS; + } else { + LOGGER.info("Call unsuccessful with response code as " + responseCode + " and message as " + response.readEntity(String.class) + " for payload: " + payload); + return ConsumerCallbackResult.ERROR; + } + }catch(Exception e){ + LOGGER.error("Call unsuccessful with error: ",e); + return ConsumerCallbackResult.ERROR; + } + } + + +} diff --git a/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/HttpPostClient.java b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/HttpPostClient.java new file mode 100644 index 0000000..ab62fbc --- /dev/null +++ b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/HttpPostClient.java @@ -0,0 +1,15 @@ +package com.flipkart.aesop.apicallerdatalayer.client; + + +import javax.ws.rs.core.Response; +import java.util.Map; + +/** + * Created by aman.gupta on 09/12/15. + */ +public interface HttpPostClient { + //Request request = new Request(url,payload,headers); + //Response response = client.executePost(request); + + public Response post(String url, String payload, Map headers); +} diff --git a/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/implementation/HttpCASPostClientImpl.java b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/implementation/HttpCASPostClientImpl.java new file mode 100644 index 0000000..5d61f40 --- /dev/null +++ b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/implementation/HttpCASPostClientImpl.java @@ -0,0 +1,37 @@ +package com.flipkart.aesop.apicallerdatalayer.client.implementation; + +import com.flipkart.aesop.apicallerdatalayer.client.HttpPostClient; +import com.flipkart.casclient.client.HttpAuthClient; +import com.flipkart.casclient.entity.Request; +import flipkart.platform.cachefarm.Cache; +import org.trpr.platform.core.impl.logging.LogFactory; +import org.trpr.platform.core.spi.logging.Logger; + +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.Map; + +/** + * Created by aman.gupta on 09/12/15. + */ +public class HttpCASPostClientImpl implements HttpPostClient { + private HttpAuthClient httpAuthClient; + private static final Logger LOGGER = LogFactory.getLogger(HttpCASPostClientImpl.class); + public HttpCASPostClientImpl(String casUrl, String user, String password, boolean enableAuth, Cache cache) { + this.httpAuthClient = new HttpAuthClient(casUrl,user,password,enableAuth,cache); + } + + @Override + public Response post(String url, String payload, Map headers) { + Request request = new Request(url,payload,headers); + com.ning.http.client.Response response = httpAuthClient.executePost(request); + String responseBody; + try { + responseBody = response.getResponseBody(); + } catch (IOException e) { + LOGGER.error("Unable to get response body for payload"+payload+" Keeping it blank.",e); + responseBody = ""; + } + return Response.status(response.getStatusCode()).entity(responseBody).build(); + } +} diff --git a/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/implementation/HttpJavaxPostClientImpl.java b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/implementation/HttpJavaxPostClientImpl.java new file mode 100644 index 0000000..236cb7a --- /dev/null +++ b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/client/implementation/HttpJavaxPostClientImpl.java @@ -0,0 +1,28 @@ +package com.flipkart.aesop.apicallerdatalayer.client.implementation; + + +import com.flipkart.aesop.apicallerdatalayer.client.HttpPostClient; +import org.trpr.platform.core.impl.logging.LogFactory; +import org.trpr.platform.core.spi.logging.Logger; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Map; + +/** + * Created by aman.gupta on 10/12/15. + */ +public class HttpJavaxPostClientImpl implements HttpPostClient { + Client client = ClientBuilder.newClient(); + private static final Logger LOGGER = LogFactory.getLogger(HttpJavaxPostClientImpl.class); + @Override + public Response post(String url, String payload, Map headers) { + + Response response = client.target(url).request().post(Entity.entity(payload, MediaType.APPLICATION_JSON_TYPE)); + LOGGER.info(response.readEntity(String.class)); + return response; + } +} diff --git a/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/ContextualHeaderProvider.java b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/ContextualHeaderProvider.java new file mode 100644 index 0000000..574d52b --- /dev/null +++ b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/ContextualHeaderProvider.java @@ -0,0 +1,10 @@ +package com.flipkart.aesop.apicallerdatalayer.headers; + +import java.util.Map; + +/** + * Created by aman.gupta on 14/12/15. + */ +public interface ContextualHeaderProvider { + public Map getHeaders(Map event); +} diff --git a/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/StaticHeaderProvider.java b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/StaticHeaderProvider.java new file mode 100644 index 0000000..0cc2c68 --- /dev/null +++ b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/StaticHeaderProvider.java @@ -0,0 +1,10 @@ +package com.flipkart.aesop.apicallerdatalayer.headers; + +import java.util.Map; + +/** + * Created by aman.gupta on 14/12/15. + */ +public interface StaticHeaderProvider { + public Map getHeaders(); +} diff --git a/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/impl/StringBasedContextualHeaderProviderImpl.java b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/impl/StringBasedContextualHeaderProviderImpl.java new file mode 100644 index 0000000..12f8888 --- /dev/null +++ b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/impl/StringBasedContextualHeaderProviderImpl.java @@ -0,0 +1,27 @@ +package com.flipkart.aesop.apicallerdatalayer.headers.impl; + +import com.flipkart.aesop.apicallerdatalayer.headers.ContextualHeaderProvider; +import com.google.common.base.Splitter; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by aman.gupta on 14/12/15. + */ +public class StringBasedContextualHeaderProviderImpl implements ContextualHeaderProvider { + Map headers; + + public StringBasedContextualHeaderProviderImpl(String headers) { + this.headers = Splitter.on(",").withKeyValueSeparator("=").split(headers); + } + + @Override + public Map getHeaders(Map event) { + Map map = new HashMap(); + for(Map.Entry entry : headers.entrySet()){ + map.put(entry.getKey(),(String)event.get(entry.getValue())); + } + return map; + } +} diff --git a/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/impl/StringBasedStaticHeaderProviderImpl.java b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/impl/StringBasedStaticHeaderProviderImpl.java new file mode 100644 index 0000000..2b21e7b --- /dev/null +++ b/data-layers/data-layer-api-caller/src/main/java/com/flipkart/aesop/apicallerdatalayer/headers/impl/StringBasedStaticHeaderProviderImpl.java @@ -0,0 +1,22 @@ +package com.flipkart.aesop.apicallerdatalayer.headers.impl; + +import com.flipkart.aesop.apicallerdatalayer.headers.StaticHeaderProvider; +import com.google.common.base.Splitter; + +import java.util.Map; + +/** + * Created by aman.gupta on 14/12/15. + */ +public class StringBasedStaticHeaderProviderImpl implements StaticHeaderProvider { + Map headers; + + public StringBasedStaticHeaderProviderImpl(String headers) { + this.headers = Splitter.on(",").withKeyValueSeparator("=").split(headers); + } + + @Override + public Map getHeaders() { + return headers; + } +} diff --git a/pom.xml b/pom.xml index 3120078..937e6ef 100755 --- a/pom.xml +++ b/pom.xml @@ -90,6 +90,7 @@ data-layers/data-layer-console-appender data-layers/data-layer-elastic-search data-layers/data-layer-kafka + data-layers/data-layer-api-caller samples/sample-memory-relay samples/sample-hbase-relay samples/sample-mysql-mysql-blocking-bootstrap @@ -100,6 +101,7 @@ samples/sample-snapshot-serializer samples/sample-mysql-relay samples/sample-client-common + samples/sample-api-caller-client-cluster-consumer samples/sample-console-appender-client-cluster-consumer samples/sample-elastic-search-client-cluster-consumer samples/sample-hbase-client-cluster-consumer diff --git a/samples/sample-api-caller-client-cluster-consumer/pom.xml b/samples/sample-api-caller-client-cluster-consumer/pom.xml new file mode 100644 index 0000000..42b4707 --- /dev/null +++ b/samples/sample-api-caller-client-cluster-consumer/pom.xml @@ -0,0 +1,186 @@ + + + + aesop + com.flipkart.aesop + 1.2.1-SNAPSHOT + ../../pom.xml + + 4.0.0 + + com.flipkart.aesop + sample-api-caller-client-cluster-consumer + + + + com.flipkart.aesop + runtime-client-cluster + + + + com.flipkart.aesop + client-event-consumer + ${project.version} + + + + com.flipkart.aesop + data-layer-api-caller + 1.2.1-SNAPSHOT + + + org.apache.httpcomponents + httpclient + 4.5 + + + ch.qos.logback + logback-core + 1.0.5 + + + ch.qos.logback + logback-classic + 1.0.5 + + + + + + + + + src/main/resources + true + + + + + + maven-clean-plugin + 2.5 + + + + distribution + + **/* + **/* + + false + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + true + + + + packaged/**/* + com/**/* + project.properties + + + + + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${jdk.version} + ${jdk.version} + + + + maven-dependency-plugin + + + install + + copy-dependencies + + + distribution/lib + + + + + + maven-antrun-plugin + + + CopyResources + install + + + + + + + + + + + + + + + run + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + + + maven-dependency-plugin + + + [2.1,) + + + + copy-dependencies + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/sample-api-caller-client-cluster-consumer/src/main/resources/application.conf b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/application.conf new file mode 100644 index 0000000..1c6ed9a --- /dev/null +++ b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/application.conf @@ -0,0 +1,11 @@ +MAPPER_CONFIG + { + or_test: + { + "Person"=[ + { + "mapAll":"true", + "groupNo":"1" + }] + } + } diff --git a/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/bootstrap.xml b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/bootstrap.xml new file mode 100644 index 0000000..4d4cb66 --- /dev/null +++ b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/bootstrap.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/logback.xml b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/logback.xml new file mode 100644 index 0000000..e491b2c --- /dev/null +++ b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/logback.xml @@ -0,0 +1,23 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %class{36}.%M %L %logger{36} - %msg%n + + + + + /tmp/sample-api-caller-client-cluster-consumer/sample-mysql-relay.log + true + + %d{HH:mm:ss.SSS} [%thread] %-5level %class{36}.%M %L %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file diff --git a/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/runtime-config.properties b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/runtime-config.properties new file mode 100644 index 0000000..091f49a --- /dev/null +++ b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/runtime-config.properties @@ -0,0 +1,4 @@ +# Start Jetty configuration values +runtime.server.jetty.admin.port=8082 +runtime.server.jetty.maxFormContentSize=60000000 +# End Jetty configuration values \ No newline at end of file diff --git a/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/spring-client-config.xml b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/spring-client-config.xml new file mode 100644 index 0000000..e254ee7 --- /dev/null +++ b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/external/spring-client-config.xml @@ -0,0 +1,187 @@ + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + com.flipkart.aesop.events.ortest.Person + + + + + + + + + + + + + + 11125 + false + 1 + -1 + false + false + + + + + + + + + + + + + com.flipkart.aesop.events.ortest.Person + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/sample-api-caller-client-cluster-consumer/src/main/resources/project.properties b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/project.properties new file mode 100644 index 0000000..437d9a7 --- /dev/null +++ b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/project.properties @@ -0,0 +1,9 @@ +CHECKPOINT_DIR_LOCATION=${checkpoint_dir_location} +ZK_ADDR=${zk_addr} +MAPPER_CONFIG_FILE_PATH=${mapper_config_file_path} +RELAY_HOST=${relay_host} +DENORMALIZED_DB=${denormalized_db} +DENORMALIZED_DB_USER=${denormalized_db_user} +DENORMALIZED_DB_PASSWORD=${denormalized_db_password} +DENORMALIZED_DB_JDBC_URL=${denormalized_db_jdbc_url} +NUM_PARTITIONS=${num_partitions} \ No newline at end of file diff --git a/samples/sample-api-caller-client-cluster-consumer/src/main/resources/scripts/start.sh b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/scripts/start.sh new file mode 100644 index 0000000..826e433 --- /dev/null +++ b/samples/sample-api-caller-client-cluster-consumer/src/main/resources/scripts/start.sh @@ -0,0 +1 @@ +java -cp "lib/*" org.trpr.platform.runtime.impl.bootstrap.BootstrapLauncher resources/external/bootstrap.xml \ No newline at end of file