Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/new/usagi_csv_mapping'
Browse files Browse the repository at this point in the history
  • Loading branch information
@joaorafaelalmeida
joaorafaelalmeida committed Oct 28, 2020
2 parents 5993d89 + 702c37a commit f6ecdd5
Show file tree
Hide file tree
Showing 8 changed files with 456 additions and 78 deletions.
34 changes: 16 additions & 18 deletions app/assets/javascripts/step/stepController.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ define('StepController', ['Controller', 'StepView', 'Institution', 'Step', 'Rout
_super_.initialize.call(this, $container);

this.data = {};
if(this.stepId){
if (this.stepId) {
this.getStep(this.stepId);
}
};
Expand All @@ -37,15 +37,14 @@ define('StepController', ['Controller', 'StepView', 'Institution', 'Step', 'Rout
},
function (callback) {
Step.inputFieldsName(context.institutionId, stepId, function (inputFields) {
if(_.keys(inputFields).length == 1){
_.each(inputFields, function(field){
if (_.keys(inputFields).length == 1) {
_.each(inputFields, function (field) {
context.inputFields = field.valueMetaList;
});
}
else{
} else {
context.streamFields = {};
_.each(_.keys(inputFields), function(key) {
context.streamFields[key] = inputFields[key].valueMetaList;
_.each(_.keys(inputFields), function (key) {
context.streamFields[key] = inputFields[key].valueMetaList;
});
}

Expand All @@ -60,7 +59,7 @@ define('StepController', ['Controller', 'StepView', 'Institution', 'Step', 'Rout
});
},
function (callback) {
Step.getInstitution(context.institutionId, stepId, function(institutionId) {
Step.getInstitution(context.institutionId, stepId, function (institutionId) {
Institution.getDataSources(institutionId, function (dataSources) {
context.dataSources = dataSources;
callback();
Expand All @@ -80,22 +79,20 @@ define('StepController', ['Controller', 'StepView', 'Institution', 'Step', 'Rout
/**
* Returns to the pipeline view.
*/
StepController.prototype.cancelClick = function(){
StepController.prototype.cancelClick = function () {
Router.navigatePrevious();
}

StepController.prototype.updateColumn = function (tableId, name, component, $element) {
var table = this.view.dataTables[tableId].clear().draw();

var selectedVal;
try{
try {
selectedVal = this.view.$elements[component].val();
}
catch(err) {
try{
} catch (err) {
try {
selectedVal = component.val();
}
catch(err) {
} catch (err) {
selectedVal = $element.val();
}
}
Expand Down Expand Up @@ -140,9 +137,10 @@ define('StepController', ['Controller', 'StepView', 'Institution', 'Step', 'Rout
}

// Get files
_.each($form[0], function(element){
if(element.type === "file" && element.files.length > 0){
formData = new FormData();
_.each($form[0], function (element) {
if (element.type === "file" && element.files.length > 0) {
if (formData === null)
formData = new FormData();

var fileObject = element.files[0];
formData.append(element.name, fileObject);
Expand Down
12 changes: 12 additions & 0 deletions app/assets/templates/step.handlebars
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,18 @@
{{/../dataSources}}
{{/ifCond}}

{{#ifCond source '==' 'inputFields'}}
<option></option>
{{#../inputFields}}
{{#ifCond ../stepProperty.value '==' name}}
<option value="{{name}}" selected>{{name}}</option>
{{/ifCond}}
{{#ifCond ../stepProperty.value '!=' name}}
<option value="{{name}}">{{name}}</option>
{{/ifCond}}
{{/../inputFields}}
{{/ifCond}}

{{#ifCond source '==' 'inputSteps'}}
<option></option>
{{#../inputSteps}}
Expand Down
35 changes: 18 additions & 17 deletions app/controllers/StepController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,24 +353,25 @@ public Result uploadFile(long institutionId, long stepId) {
boolean useNumbers = false;
String randomName = RandomStringUtils.random(length, useLetters, useNumbers);
*/

File file = (File) filePart.get(0).getFile();

try {
long componentPropertyId = Long.parseLong(filePart.get(0).getKey());
StepProperty stepProperty = stepPropertyRepository.getByStepAndComponentProperty(stepId, componentPropertyId);

ComponentProperty componentProperty = componentPropertyRepository.get(componentPropertyId);
if (stepProperty == null) {
stepProperty = new StepProperty(file.getPath());
stepProperty.setComponentProperty(componentProperty);
stepProperty.setStep(stepRepository.get(stepId));
stepPropertyRepository.add(stepProperty);
} else {
stepProperty.setValue(file.getPath());
stepPropertyRepository.add(stepProperty);
for (int i = 0; i < filePart.size(); i++) {
File file = (File) filePart.get(i).getFile();

try {
long componentPropertyId = Long.parseLong(filePart.get(i).getKey());
StepProperty stepProperty = stepPropertyRepository.getByStepAndComponentProperty(stepId, componentPropertyId);

ComponentProperty componentProperty = componentPropertyRepository.get(componentPropertyId);
if (stepProperty == null) {
stepProperty = new StepProperty(file.getPath());
stepProperty.setComponentProperty(componentProperty);
stepProperty.setStep(stepRepository.get(stepId));
stepPropertyRepository.add(stepProperty);
} else {
stepProperty.setValue(file.getPath());
stepPropertyRepository.add(stepProperty);
}
} catch (Exception e) {
}
} catch (Exception e) {
}
}

Expand Down
93 changes: 84 additions & 9 deletions app/diSdk/step/AbstractStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@ public StepMeta decodeStep(Step step, List<DatabaseMeta> databases) throws Excep
String fileName = null;
String delimiter = null;

// USAGI Value Mapper
List<String> usagiSourceValues = null;
List<String> usagiTargetValues = null;
String usagiExportFilename = null;


System.out.println(stepname);

if (stepMetaInterface != null) {
Expand All @@ -68,6 +74,56 @@ public StepMeta decodeStep(Step step, List<DatabaseMeta> databases) throws Excep
.filter(method -> method.getParameterCount() == 1 || method.getName().equals("getStepIOMeta"))
.toArray(Method[]::new);

// USAGI PREPROCESSING //
Optional<StepProperty> usagiStepProperty = stepProperties.stream()
.filter(stepProperty -> stepProperty.getComponentProperty().getShortName().equalsIgnoreCase("usagiFile"))
.findFirst();

if (usagiStepProperty.isPresent()) {
usagiExportFilename = usagiStepProperty.get().getValue();

int inputColumn;
Optional<StepProperty> inputColumnStep = stepProperties.stream()
.filter(stepProperty -> stepProperty.getComponentProperty().getShortName().equalsIgnoreCase("inputColumn"))
.findFirst();

if (!inputColumnStep.isPresent())
inputColumn = 1;
else
inputColumn = Integer.parseInt(inputColumnStep.get().getValue());

int outputColumn;
Optional<StepProperty> outputColumnStep = stepProperties.stream()
.filter(stepProperty -> stepProperty.getComponentProperty().getShortName().equalsIgnoreCase("outputColumn"))
.findFirst();

if (!outputColumnStep.isPresent())
outputColumn = 5;
else
outputColumn = Integer.parseInt(outputColumnStep.get().getValue());

try {
BufferedReader br = new BufferedReader(new FileReader(usagiExportFilename));
br.readLine(); //Skip Header

usagiSourceValues = new ArrayList<>();
usagiTargetValues = new ArrayList<>();

String row;
while ((row = br.readLine()) != null) {
String[] data = row.split(",");
usagiSourceValues.add(data[inputColumn]);
usagiTargetValues.add(data[outputColumn]);
}

} catch (FileNotFoundException e) {

}
}
System.out.println(usagiSourceValues);
System.out.println(usagiTargetValues);
/////////////////////////

for (Method method : methods) {
if (method.getName().equals("getStepIOMeta")) {
List<StreamInterface> targetStreams = stepMetaInterface.getStepIOMeta().getTargetStreams();
Expand Down Expand Up @@ -105,7 +161,7 @@ public StepMeta decodeStep(Step step, List<DatabaseMeta> databases) throws Excep

// If dealing with CSVFileInput get the input fields and define them
if (shortName.equals("InputFields")) {
if(fileName == null){
if (fileName == null) {
Optional<StepProperty> fileNameStepProperty = stepProperties.stream()
.filter(stepProperty -> stepProperty.getComponentProperty().getShortName().equalsIgnoreCase("Filename"))
.findFirst();
Expand All @@ -115,7 +171,7 @@ public StepMeta decodeStep(Step step, List<DatabaseMeta> databases) throws Excep
fileName = fileNameStepProperty.get().getValue();
}

if(delimiter == null){
if (delimiter == null) {
Optional<StepProperty> delimiterStepProperty = stepProperties.stream()
.filter(stepProperty -> stepProperty.getComponentProperty().getShortName().equalsIgnoreCase("Delimiter"))
.findFirst();
Expand All @@ -125,7 +181,7 @@ public StepMeta decodeStep(Step step, List<DatabaseMeta> databases) throws Excep
delimiter = delimiterStepProperty.get().getValue();
}

try{
try {
BufferedReader br = new BufferedReader(new FileReader(fileName));
String header = br.readLine();

Expand All @@ -135,7 +191,7 @@ public StepMeta decodeStep(Step step, List<DatabaseMeta> databases) throws Excep
}

TextFileInputField[] value = new TextFileInputField[fields.length];
for(int i = 0 ; i < fields.length ; i++){
for (int i = 0; i < fields.length; i++) {
String field = fields[i];
value[i] = new TextFileInputField();
value[i].setName(field);
Expand All @@ -145,10 +201,25 @@ public StepMeta decodeStep(Step step, List<DatabaseMeta> databases) throws Excep
// Invoke the current method with the StepProperty value.
invokeMethod(stepMetaInterface, method, value, databases);

}catch (FileNotFoundException e){
} catch (FileNotFoundException e) {

}

} else if (shortName.equals("BufferSize")) { //If BufferSize isn't defined, use the default value
// Invoke the current method with the StepProperty value.
System.out.println("oof");
invokeMethod(stepMetaInterface, method, "50000", databases);

} else if (usagiSourceValues != null && shortName.equals("SourceValue")) {
System.out.println("VALUE - " + usagiSourceValues.toString() + "\n");

// Invoke the current method with the StepProperty value.
invokeMethod(stepMetaInterface, method, usagiSourceValues, databases);
} else if (usagiTargetValues != null && shortName.equals("TargetValue")) {
System.out.println("VALUE - " + usagiTargetValues.toString() + "\n");

// Invoke the current method with the StepProperty value.
invokeMethod(stepMetaInterface, method, usagiTargetValues, databases);
}

continue;
Expand Down Expand Up @@ -176,16 +247,20 @@ public StepMeta decodeStep(Step step, List<DatabaseMeta> databases) throws Excep
.collect(Collectors.toList());
}

System.out.println("VALUE - " + value.toString() + "\n");
System.out.println("VALUE - " + value.toString() +"\n");

if(shortName.equals("Filename")){
if (shortName.equals("Filename")) {
fileName = value.toString();
}
if(shortName.equals("Delimiter")){
if (shortName.equals("Delimiter")) {
delimiter = value.toString();
}

// Invoke the current method with the StepProperty value.
if (shortName.equals("BufferSize") && value.toString().length()==0) {
value="50000";
}

// Invoke the current method with the StepProperty value.
invokeMethod(stepMetaInterface, method, value, databases);
stepProperties.remove(optStepProperty.get());

Expand Down
18 changes: 18 additions & 0 deletions app/diSdk/step/parser/ValueMapper.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package diSdk.step.parser;

import diSdk.step.AbstractStep;
import models.Step;
import org.pentaho.di.trans.step.StepMetaInterface;
import org.w3c.dom.Element;

public class ValueMapper extends AbstractStep {
@Override
public void decode(StepMetaInterface stepMetaInterface, Step step) throws Exception {

}

@Override
public Element encode(StepMetaInterface stepMetaInterface) throws Exception {
return null;
}
}
2 changes: 1 addition & 1 deletion app/kettleExt/trans/steps/FilterRows.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public ValueMetaAndData decodeValueMetaAndData(JSONObject jsonObject) throws Ket
ValueMetaAndData valueMetaAndData = new ValueMetaAndData();

String valname = jsonObject.optString("name");
int valtype = ValueMetaBase.getType(jsonObject.optString("type"));
int valtype = ValueMetaBase.getType(jsonObject.optString("tconditionype"));
String text = jsonObject.optString("text");
boolean isnull = jsonObject.optBoolean("isnull");
int len = jsonObject.optInt("length", -1);
Expand Down
Loading

0 comments on commit f6ecdd5

Please sign in to comment.