add dragons graph content generation zip

YANG-DB · Mar 29, 2022 · 7990709 · 7990709
1 parent 6fe8a18
commit 7990709
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 18 deletions.
diff --git a/...bly/resources/assembly/Dragons/config/application.test.engine3.m1.dfs.dragons.public.conf b/...bly/resources/assembly/Dragons/config/application.test.engine3.m1.dfs.dragons.public.conf
@@ -9,7 +9,7 @@ server.http.ResponseBufferSize = 10m
 # Max request body size to keep in memory
 server.http.RequestBufferSize = 10m
 # Max request size total (body + header)
-server.http.MaxRequestSize = 10m
+server.http.MaxRequestSize = 100m
 
 modules.activeProfile = [
   "com.yangdb.fuse.services.modules.ServiceModule",

diff --git a/...atagen/src/main/java/com/yangdb/fuse/generator/data/generation/DragonsGraphGenerator.java b/...atagen/src/main/java/com/yangdb/fuse/generator/data/generation/DragonsGraphGenerator.java
@@ -61,6 +61,7 @@ public class DragonsGraphGenerator extends GraphGeneratorBase<DragonConfiguratio
     public static final String[] DRAGON_HEADER = {"id", "name", "birthDate", "power", "gender", "color"};
     public static final String[] DRAGONS_FIRE_HEADER = {"id", "entityA.id", "entityA.type", "entityB.id", "entityB.type", "date", "temperature"};
     public static final String[] DRAGON_FREEZE_HEADER = {"id", "entityA.id", "entityA.type", "entityB.id", "entityB.type", "date", "temperature"};
+
     private final Logger logger = LoggerFactory.getLogger(DragonsGraphGenerator.class);
 
     //region Ctrs

diff --git a/...sembly/Dragons/config/application.test.engine3.count.redundant.m1.dfs.dragons.public.conf b/...sembly/Dragons/config/application.test.engine3.count.redundant.m1.dfs.dragons.public.conf
@@ -9,7 +9,7 @@ server.http.ResponseBufferSize = 10m
 # Max request body size to keep in memory
 server.http.RequestBufferSize = 10m
 # Max request size total (body + header)
-server.http.MaxRequestSize = 10m
+server.http.MaxRequestSize = 100m
 
 modules.activeProfile = [
   "com.yangdb.fuse.services.modules.ServiceModule",

diff --git a/...src/resources/assembly/Dragons/config/application.test.engine3.m1.dfs.dragons.public.conf b/...src/resources/assembly/Dragons/config/application.test.engine3.m1.dfs.dragons.public.conf
@@ -9,7 +9,7 @@ server.http.ResponseBufferSize = 10m
 # Max request body size to keep in memory
 server.http.RequestBufferSize = 10m
 # Max request size total (body + header)
-server.http.MaxRequestSize = 10m
+server.http.MaxRequestSize = 100m
 
 modules.activeProfile = [
   "com.yangdb.fuse.services.modules.ServiceModule",

diff --git a/...v/fuse-dv-core/src/main/java/com/yangdb/fuse/executor/ontology/schema/load/LoadUtils.java b/...v/fuse-dv-core/src/main/java/com/yangdb/fuse/executor/ontology/schema/load/LoadUtils.java
@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,53 +37,54 @@
 import org.opensearch.action.index.IndexRequestBuilder;
 import org.opensearch.client.Client;
 import org.opensearch.common.xcontent.XContentType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Optional;
 import java.util.TimeZone;
-import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * documents bulk loading utilities
  */
 public class LoadUtils {
+    private static final Logger logger = LoggerFactory.getLogger(LoadUtils.class);
+
     private static final SimpleDateFormat sdf = new SimpleDateFormat(GlobalConstants.DEFAULT_DATE_FORMAT);
     private static ObjectMapper mapper = new ObjectMapper();
 
     static {
         sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
     }
 
-    public static Tuple2<Response,BulkRequestBuilder> load(PartitionResolver schema, Client client, DataTransformerContext<Object> context) {
+    public static Tuple2<Response, BulkRequestBuilder> load(PartitionResolver schema, Client client, DataTransformerContext<Object> context) {
         Response upload = new Response("upload");
         BulkRequestBuilder bulk = client.prepareBulk();
         //populate bulk entities documents index requests
         for (DocumentBuilder documentBuilder : context.getEntities()) {
             try {
-                if(documentBuilder.isSuccess())
-                    buildIndexRequest(schema,client,bulk, documentBuilder);
+                if (documentBuilder.isSuccess())
+                    buildIndexRequest(schema, client, bulk, documentBuilder);
             } catch (FuseError.FuseErrorException e) {
                 upload.failure(e.getError());
             }
         }
         //populate bulk relations document index requests
         for (DocumentBuilder e : context.getRelations()) {
             try {
-                if(e.isSuccess())
-                    buildIndexRequest(schema,client,bulk, e);
+                if (e.isSuccess())
+                    buildIndexRequest(schema, client, bulk, e);
             } catch (FuseError.FuseErrorException err) {
                 upload.failure(err.getError());
             }
         }
-        return new Tuple2<>(upload,bulk);
+        return new Tuple2<>(upload, bulk);
     }
 
     public static IndexRequestBuilder buildIndexRequest(PartitionResolver schema, Client client, BulkRequestBuilder bulk, DocumentBuilder node) {
         try {
-            String index = resolveIndex(schema,node);
+            String index = resolveIndex(schema, node);
             IndexRequestBuilder request = client.prepareIndex()
                     .setIndex(index.toLowerCase())
                     .setType(node.getType())
@@ -119,6 +120,7 @@ public static String resolveIndex(PartitionResolver schema, DocumentBuilder node
 
     /**
      * submit bulk request
+     *
      * @param bulk
      * @param upload
      * @return
@@ -136,7 +138,9 @@ public static void submit(BulkRequestBuilder bulk, Response upload) {
                     BulkItemResponse.Failure failure = item.getFailure();
                     DocWriteRequest<?> request = bulk.request().requests().get(item.getItemId());
                     //todo - get additional information from request
-                    upload.failure(new FuseError("commit failed", failure.toString()));
+                    FuseError commitFailed = new FuseError("commit failed", failure.toString());
+                    upload.failure(commitFailed);
+                    logger.error(commitFailed.toString());
                 }
             }
         } catch (Exception err) {

diff --git a/fuse-dv/fuse-dv-core/src/test/resources/conf/application.engine2.dev.conf b/fuse-dv/fuse-dv-core/src/test/resources/conf/application.engine2.dev.conf
@@ -9,7 +9,7 @@ server.http.ResponseBufferSize = 10m
 # Max request body size to keep in memory
 server.http.RequestBufferSize = 10m
 # Max request size total (body + header)
-server.http.MaxRequestSize = 10m
+server.http.MaxRequestSize = 100m
 
 modules.activeProfile = [
   "com.yangdb.fuse.services.modules.ServiceModule",

diff --git a/fuse-service/src/test/resources/config/application.conf b/fuse-service/src/test/resources/config/application.conf
@@ -9,7 +9,7 @@ server.http.ResponseBufferSize = 10m
 # Max request body size to keep in memory
 server.http.RequestBufferSize = 10m
 # Max request size total (body + header)
-server.http.MaxRequestSize = 10m
+server.http.MaxRequestSize = 100m
 
 modules.activeProfile = [
   "com.yangdb.fuse.services.modules.ServiceModule",