Skip to content

Commit

Permalink
add dragons graph content generation zip
Browse files Browse the repository at this point in the history
  • Loading branch information
@YANG-DB
YANG-DB committed Mar 29, 2022
1 parent c1bb5ca commit 6fe8a18
Show file tree
Hide file tree
Showing 11 changed files with 219 additions and 77 deletions.
8 changes: 8 additions & 0 deletions fuse-domain/fuse-domain-dragons/fuse-domain-dragons-datagen/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,14 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>11</source>
<target>11</target>
</configuration>
</plugin>
</plugins>

<resources>
Expand Down
5 changes: 4 additions & 1 deletion ...ns/fuse-domain-dragons-datagen/src/main/java/com/yangdb/fuse/generator/DataGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,11 @@ public static void main(String[] args) {
System.out.println("************GENERATING DRAGONS GRAPH *************************************************");
DragonsGraphGenerator dgg = new DragonsGraphGenerator(new DragonConfiguration(configuration));
List<String> dragonsIds = dgg.generateMassiveDragonsGraph();
dgg.Cleanup();

System.out.println("************GENERATING HORSES GRAPH *************************************************");
HorsesGraphGenerator hgg = new HorsesGraphGenerator(new HorseConfiguration(configuration));
hgg.Cleanup();
List<String> horsesIds = hgg.generateHorsesGraph();

System.out.println("************GENERATING PERSON GRAPH *************************************************");
Expand All @@ -95,12 +97,13 @@ public static void main(String[] args) {
List<String> personsIds = pgg.generatePersonsGraph();
pgg.attachDragonsToPerson(dragonsIds, personsIds, personConf.getMeanDragonsPerPerson(), personConf.getSdDragonsPerPerson());
pgg.attachHorsesToPerson(horsesIds, personsIds, personConf.getMeanHorsesPerPerson(), personConf.getSdHorsesPerPerson());

pgg.Cleanup();

System.out.println("************GENERATING GUILDS GRAPH *************************************************");
GuildsGraphGenerator ggg = new GuildsGraphGenerator(configuration);
List<String> guildsIds = ggg.generateGuildsGraph();
ggg.attachPersonsToGuilds(guildsIds, personsIds);
ggg.Cleanup();

System.out.println("************GENERATING KINGDOM GRAPH *************************************************");
KingdomsGraphGenerator kgg = new KingdomsGraphGenerator(new KingdomConfiguration(configuration));
Expand Down
60 changes: 39 additions & 21 deletions ...atagen/src/main/java/com/yangdb/fuse/generator/data/generation/DragonsGraphGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand All @@ -21,7 +21,6 @@
*/



import com.google.common.base.Stopwatch;
import com.yangdb.fuse.generator.configuration.DragonConfiguration;
import com.yangdb.fuse.generator.data.generation.entity.DragonGenerator;
Expand All @@ -35,6 +34,7 @@
import com.yangdb.fuse.generator.model.relation.Freezes;
import com.yangdb.fuse.generator.model.relation.RelationBase;
import com.yangdb.fuse.generator.util.DateUtil;
import com.yangdb.fuse.generator.util.FileUtils;
import com.yangdb.fuse.generator.util.RandomUtil;
import javaslang.Tuple2;
import javaslang.collection.Stream;
Expand All @@ -44,6 +44,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
Expand Down Expand Up @@ -126,7 +127,11 @@ protected Graph generateGraph() {
List<Tuple2> edgesList = graph.getEdgeSet().stream().map(edge ->
new Tuple2<>(edge.getSourceNode().getId(), edge.getTargetNode().getId())).collect(Collectors.toList());

writeGraph(nodesList, edgesList);
try {
writeGraph(nodesList, edgesList);
} catch (Throwable err) {
logger.warn(err.getMessage(), err);
}
return graph;
}

Expand All @@ -137,7 +142,7 @@ protected List<String> generateMassiveGraph() {
model.getNumOfNodes(),
((ScaleFreeModel) model).getEdgesPerNode(),
BAGraphGenerator.SamplingMode.ROLL_TREE,
configuration.getRelationsFilePath());
personConf.getRelationsFilePath());


Set<Long> tempSet = new LinkedHashSet<>(Stream.ofAll(edgesList).map(tuple2 -> (Long) tuple2._1).toJavaList());
Expand All @@ -146,7 +151,11 @@ protected List<String> generateMassiveGraph() {
Collections.sort(nodeNumericIds);
List<String> nodesList = nodeNumericIds.stream().map(Object::toString).collect(Collectors.toList());

writeGraph(nodesList, edgesList);
try {
writeGraph(nodesList, edgesList);
} catch (Throwable err) {
logger.warn(err.getMessage(), err);
}

return nodesList;
}
Expand All @@ -161,27 +170,27 @@ protected Dragon buildEntityNode(String id) {
@Override
protected RelationBase buildEntityRelation(String sourceId, String targetId, String edgeId) {
ArrayList<Pair<RelationType, Double>> probs = new ArrayList<>(
Arrays.asList(new Pair<>(RelationType.FIRES,configuration.getFireProbability()),
new Pair<>(RelationType.FREEZES, configuration.getFreezProbability())
));
Arrays.asList(new Pair<>(RelationType.FIRES, personConf.getFireProbability()),
new Pair<>(RelationType.FREEZES, personConf.getFreezProbability())
));
RelationType relationType = RandomUtil.enumeratedDistribution(probs);
Date date = RandomUtil.randomDate(configuration.getStartDateOfStory(), configuration.getEndDateOfStory());
int temperature = RandomUtil.randomInt(configuration.getFireMinTemperature(), configuration.getFireMaxTemperature());
Date date = RandomUtil.randomDate(personConf.getStartDateOfStory(), personConf.getEndDateOfStory());
int temperature = RandomUtil.randomInt(personConf.getFireMinTemperature(), personConf.getFireMaxTemperature());
RelationBase relationBase = null;

if (relationType == RelationType.FIRES) {
relationBase = new Fires(edgeId, sourceId, targetId, date, temperature);
}

if (relationType == RelationType.FREEZES) {
Date dateTill = DateUtil.addMinutesToDate(date, RandomUtil.randomInt(1, configuration.getFreezMaxDuraution()));
relationBase = new Freezes(edgeId, sourceId, targetId, date, dateTill,temperature);
Date dateTill = DateUtil.addMinutesToDate(date, RandomUtil.randomInt(1, personConf.getFreezMaxDuraution()));
relationBase = new Freezes(edgeId, sourceId, targetId, date, dateTill, temperature);
}
return relationBase;
}

@Override
protected void writeGraph(List<String> nodesList, List<Tuple2> edgesList) {
protected void writeGraph(List<String> nodesList, List<Tuple2> edgesList) throws IOException {
List<String[]> peopleRecords = new ArrayList<>();
List<String[]> dragonsRecords = new ArrayList<>();

Expand All @@ -190,25 +199,25 @@ protected void writeGraph(List<String> nodesList, List<Tuple2> edgesList) {

//add headers
peopleRecords.add(0, DRAGON_HEADER);
dragonsRecords.add(0,DRAGON_HEADER);
dragonsRecords.add(0, DRAGON_HEADER);
dragonsFiresRecords.add(0, DRAGONS_FIRE_HEADER);
dragonsFreezeRecords.add(0, DRAGON_FREEZE_HEADER);

String fireRelationsFile = configuration.getRelationsFilePath().replace(".csv", "") + "_" + RelationType.FIRES + ".csv";
String freezeRelationsFile = configuration.getRelationsFilePath().replace(".csv", "") + "_" + RelationType.FREEZES + ".csv";
String entitiesFile = configuration.getEntitiesFilePath();
String fireRelationsFile = personConf.getRelationsFilePath().replace(".csv", "") + "_" + RelationType.FIRES + ".csv";
String freezeRelationsFile = personConf.getRelationsFilePath().replace(".csv", "") + "_" + RelationType.FREEZES + ".csv";
String entitiesFile = personConf.getEntitiesFilePath();

for (String nodeId : nodesList) {
dragonsRecords.add(buildEntityNode(nodeId).getRecord());
if (dragonsRecords.size() % BUFFER == 0) { //BUFFER
logger.info("writing to file ... "+ BUFFER +" elements");
logger.info("writing to file ... " + BUFFER + " elements");
appendResults(dragonsRecords, entitiesFile);
dragonsRecords.clear();
}
}

for (Tuple2 edge : edgesList) {
int numOfInteractions = RandomUtil.randomInt(configuration.getMinUniqueInteractions(), configuration.getMaxUniqueInteractions());
int numOfInteractions = RandomUtil.randomInt(personConf.getMinUniqueInteractions(), personConf.getMaxUniqueInteractions());
for (int i = 0; i < numOfInteractions; i++) {
String sourceId = edge._1.toString();
String targetId = edge._2.toString();
Expand All @@ -224,7 +233,7 @@ protected void writeGraph(List<String> nodesList, List<Tuple2> edgesList) {
if ((dragonsFiresRecords.size() + dragonsFreezeRecords.size()) % BUFFER == 0) { //BUFFER
appendResults(dragonsFiresRecords, fireRelationsFile);
appendResults(dragonsFreezeRecords, freezeRelationsFile);
logger.info("writing to file ... "+ BUFFER +" elements");
logger.info("writing to file ... " + BUFFER + " elements");
dragonsFreezeRecords.clear();
dragonsFiresRecords.clear();
}
Expand All @@ -235,12 +244,21 @@ protected void writeGraph(List<String> nodesList, List<Tuple2> edgesList) {
appendResults(dragonsRecords, entitiesFile);
appendResults(dragonsFreezeRecords, freezeRelationsFile);

//zip files
FileUtils.zip(new File(entitiesFile), entitiesFile + "_zipped");
FileUtils.zip(new File(fireRelationsFile), fireRelationsFile + "_zipped");
FileUtils.zip(new File(freezeRelationsFile), freezeRelationsFile + "_zipped");
}

@Override
protected void writeCSVs(List<Dragon> elements) {
//todo
}

@Override
public void Cleanup() {
new File(dragonConf.getRelationsFilePath()).delete();
}
//endregion

//region Private Methods
Expand Down
27 changes: 17 additions & 10 deletions ...datagen/src/main/java/com/yangdb/fuse/generator/data/generation/GuildsGraphGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand All @@ -21,7 +21,6 @@
*/



import com.yangdb.fuse.generator.configuration.GuildConfiguration;
import com.yangdb.fuse.generator.configuration.PersonConfiguration;
import com.yangdb.fuse.generator.data.generation.entity.GuildGenerator;
Expand All @@ -38,6 +37,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
Expand All @@ -50,7 +50,7 @@
public class GuildsGraphGenerator {

public static final String[] GUILDS_HEADER = {"id", "name", "description", "iconId", "url", "establishDate"};
public static final String[] PERSON_TO_GUILD_HEADER = {"id", "entityA.id", "entityA.type", "entityB.id", "entityB.type", "startDate","endDate"};
public static final String[] PERSON_TO_GUILD_HEADER = {"id", "entityA.id", "entityA.type", "entityB.id", "entityB.type", "startDate", "endDate"};
private final Logger logger = LoggerFactory.getLogger(GuildsGraphGenerator.class);
//Not all of the population is member of guild
private final double NOT_ASSIGNED_TO_GUILD_RATIO = 0.025;
Expand Down Expand Up @@ -98,7 +98,7 @@ public List<Guild> generateGuilds() {
* @return Map <Guild Id, List of Persons Ids>
*/
public Map<String, List<String>> attachPersonsToGuilds(List<String> guildsIdList,
List<String> personsIdList) {
List<String> personsIdList) {
Map<String, List<String>> guildToPersonsSet = new HashMap<>();

int maxGuildMembership = personConf.getMaxGuildMembership();
Expand All @@ -115,13 +115,13 @@ public Map<String, List<String>> attachPersonsToGuilds(List<String> guildsIdList
for (int k = 0; k < maxGuildMembership; k++) {
int startIndex = 0;
for (int i = 0; i < guildsMembersDist.size(); i++) {
String guildId = guildsIdList.get(i);
String guildId = guildsIdList.get(i);
int guildMembersSize = Math.toIntExact(Math.round(guildsMembersDist.get(i) * membersPopulationSize));
for (int j = startIndex; j < membersPopulationSize; j++) {
String personId = shuffledPersonsIds.get(j);

if(guildToPersonsSet.size() % BUFFER == 0)
logger.info("Collecting to generate ... "+ BUFFER +" elements");
if (guildToPersonsSet.size() % BUFFER == 0)
logger.info("Collecting to generate ... " + BUFFER + " elements");

if (guildToPersonsSet.get(guildId) == null) {
guildToPersonsSet.put(guildId, new ArrayList<>(Arrays.asList(personId)));
Expand Down Expand Up @@ -155,8 +155,8 @@ private void printPersonsToGuild(Map<String, List<String>> personsToGuild, Entit
Date till = RandomUtil.randomDate(since, guildConf.getEndDateOfStory());
RelationBase personMemberOfGuildsRel = new MemberOf(edgeId, guildId, personId, since, till);
p2gRecords.add(personMemberOfGuildsRel.getRecord());
if(p2gRecords.size() % BUFFER == 0)
logger.info("Collecting to generate ... "+ BUFFER +" elements");
if (p2gRecords.size() % BUFFER == 0)
logger.info("Collecting to generate ... " + BUFFER + " elements");

}
}
Expand All @@ -171,6 +171,13 @@ private void printPersonsToGuild(Map<String, List<String>> personsToGuild, Entit
//region Fields
private final GuildConfiguration guildConf;
private final PersonConfiguration personConf;

/**
* cleanup intermediate files
*/
public void Cleanup() {
new File(guildConf.getRelationsFilePath()).delete();
}
//endregion

}
5 changes: 5 additions & 0 deletions ...datagen/src/main/java/com/yangdb/fuse/generator/data/generation/HorsesGraphGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ public List<Horse> generateHorses() {

//region Fields
private final HorseConfiguration horseConf;

/**
* cleanup intermediate files
*/
public void Cleanup() {}
//endregion

}
Loading

0 comments on commit 6fe8a18

Please sign in to comment.