Skip to content

Commit

Permalink
#117 Implemented mapping ODML objects into ElasticSearch. Finally.
Browse files Browse the repository at this point in the history
  • Loading branch information
rinkesj committed Jul 2, 2015
1 parent b898269 commit e0d65f0
Show file tree
Hide file tree
Showing 11 changed files with 248 additions and 42 deletions.
7 changes: 7 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -839,6 +839,13 @@
<!--<scope>test</scope>-->
</dependency>

<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20131018</version>
</dependency>


</dependencies>

<build>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,52 +27,40 @@
*/
package cz.zcu.kiv.eegdatabase.data.dao;

import cz.zcu.kiv.eegdatabase.data.nosql.entities.ExperimentElastic;
import cz.zcu.kiv.eegdatabase.data.nosql.entities.GenericParameter;
import cz.zcu.kiv.eegdatabase.data.pojo.DataFile;
import cz.zcu.kiv.eegdatabase.data.pojo.Experiment;
import cz.zcu.kiv.eegdatabase.data.pojo.Person;
import cz.zcu.kiv.eegdatabase.logic.controller.search.SearchRequest;
import cz.zcu.kiv.eegdatabase.logic.util.ControllerUtils;

import org.hibernate.Query;
import org.hibernate.Session;
import org.springframework.transaction.annotation.Transactional;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import javax.annotation.Resource;

import org.apache.commons.lang.NotImplementedException;
import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.BoolFilterBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;

import static org.elasticsearch.index.query.FilterBuilders.termFilter;

import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;

import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;

import org.elasticsearch.index.query.NestedFilterBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.OrFilterBuilder;
import org.hibernate.Query;
import org.hibernate.Session;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SearchQuery;
import org.springframework.transaction.annotation.Transactional;

import cz.zcu.kiv.eegdatabase.data.nosql.entities.ExperimentElastic;
import cz.zcu.kiv.eegdatabase.data.nosql.entities.GenericParameter;
import cz.zcu.kiv.eegdatabase.data.pojo.DataFile;
import cz.zcu.kiv.eegdatabase.data.pojo.Experiment;
import cz.zcu.kiv.eegdatabase.data.pojo.Person;
import cz.zcu.kiv.eegdatabase.logic.controller.search.SearchRequest;
import cz.zcu.kiv.eegdatabase.logic.util.ControllerUtils;

/**
* This class extends powers (extend from) class SimpleGenericDao. Class is
Expand All @@ -87,6 +75,7 @@ public class SimpleExperimentDao extends SimpleGenericDao<Experiment, Integer> i

public SimpleExperimentDao() {
super(Experiment.class);

}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,14 @@
*/
package cz.zcu.kiv.eegdatabase.data.nosql;

import cz.zcu.kiv.eegdatabase.data.nosql.entities.ExperimentElastic;
import cz.zcu.kiv.eegdatabase.data.nosql.entities.GenericParameter;
import cz.zcu.kiv.eegdatabase.data.nosql.entities.ParameterAttribute;
import cz.zcu.kiv.eegdatabase.data.pojo.Digitization;
import cz.zcu.kiv.eegdatabase.data.pojo.Disease;
import cz.zcu.kiv.eegdatabase.data.pojo.Experiment;
import cz.zcu.kiv.eegdatabase.data.pojo.Hardware;
import cz.zcu.kiv.eegdatabase.data.pojo.Pharmaceutical;
import cz.zcu.kiv.eegdatabase.data.pojo.ProjectType;
import cz.zcu.kiv.eegdatabase.data.pojo.Software;
import cz.zcu.kiv.eegdatabase.data.pojo.Weather;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

import javax.annotation.Resource;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.elasticsearch.index.query.IdsQueryBuilder;
import org.hibernate.EmptyInterceptor;
import org.hibernate.type.Type;
Expand All @@ -51,8 +44,20 @@
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SearchQuery;

import cz.zcu.kiv.eegdatabase.data.nosql.entities.ExperimentElastic;
import cz.zcu.kiv.eegdatabase.data.nosql.entities.GenericParameter;
import cz.zcu.kiv.eegdatabase.data.nosql.entities.ParameterAttribute;
import cz.zcu.kiv.eegdatabase.data.pojo.Digitization;
import cz.zcu.kiv.eegdatabase.data.pojo.Disease;
import cz.zcu.kiv.eegdatabase.data.pojo.Experiment;
import cz.zcu.kiv.eegdatabase.data.pojo.Hardware;
import cz.zcu.kiv.eegdatabase.data.pojo.Pharmaceutical;
import cz.zcu.kiv.eegdatabase.data.pojo.ProjectType;
import cz.zcu.kiv.eegdatabase.data.pojo.Software;
import cz.zcu.kiv.eegdatabase.data.pojo.Weather;

/**
*
*
* @author bydga
*/
public class ElasticSynchronizationInterceptor extends EmptyInterceptor {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

import odml.core.Section;

import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
Expand All @@ -50,7 +53,10 @@ public class ExperimentElastic implements Serializable {
private int userId;
@Field(type = FieldType.Integer)
private int groupId;


@Field(type = FieldType.Object)
private Section metadata;

public ExperimentElastic() {
}

Expand Down Expand Up @@ -85,4 +91,12 @@ public String getExperimentId() {
public void setExperimentId(String experimentId) {
this.experimentId = experimentId;
}

public Section getMetadata() {
return metadata;
}

public void setMetadata(Section metadata) {
this.metadata = metadata;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,8 @@ public void update(Experiment experiment) {
updated.setWeather(experiment.getWeather());
updated.setEnvironmentNote(experiment.getEnvironmentNote());

updated.setElasticExperiment(experiment.getElasticExperiment());

experimentDao.update(updated);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package cz.zcu.kiv.eegdatabase.wui.core.experiments.metadata;

import java.io.IOException;

import odml.core.Section;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codehaus.jackson.Version;
import org.codehaus.jackson.map.DeserializationConfig.Feature;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.module.SimpleModule;
import org.springframework.data.elasticsearch.core.EntityMapper;

public class CustomEntityMapper implements EntityMapper {

protected Log log = LogFactory.getLog(getClass());

private ObjectMapper objectMapper;

public CustomEntityMapper() {

objectMapper = new ObjectMapper();
objectMapper.configure(Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
// objectMapper.configure(Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); need it ??? don't know now.

SimpleModule module = new SimpleModule("section", Version.unknownVersion());
module.addSerializer(Section.class, new ODMLSectionSerializer());
module.addDeserializer(Section.class, new ODMLSectionDeserializer());

objectMapper.registerModule(module);
}

@Override
public String mapToString(Object object) throws IOException {
return objectMapper.writeValueAsString(object);
}

@Override
public <T> T mapToObject(String source, Class<T> clazz) throws IOException {
return objectMapper.readValue(source, clazz);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package cz.zcu.kiv.eegdatabase.wui.core.experiments.metadata;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import odml.core.Reader;
import odml.core.Section;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.wicket.ajax.json.JSONException;
import org.apache.wicket.ajax.json.JSONObject;
import org.apache.wicket.ajax.json.XML;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonProcessingException;
import org.codehaus.jackson.map.DeserializationContext;
import org.codehaus.jackson.map.JsonDeserializer;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

public class ODMLSectionDeserializer extends JsonDeserializer<Section> {

protected Log log = LogFactory.getLog(getClass());

@Override
public Section deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {
long start = System.currentTimeMillis();

try {

String jsonString = jp.getCodec().readTree(jp).toString();
JSONObject jsonObject = new JSONObject(jsonString);
String xmlString = XML.toString(jsonObject);
Reader reader = new Reader();

ByteArrayInputStream stream = new ByteArrayInputStream(xmlString.getBytes());
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder dbuilder = dbf.newDocumentBuilder();
Document dom = dbuilder.parse(stream);
Element rootElement = dom.getDocumentElement();
rootElement.setAttribute("version", "1.0");

stream = new ByteArrayInputStream(getStringFromDocument(dom).getBytes());
return reader.load(stream);
} catch (JSONException e) {
log.warn(e.getMessage(), e);
} catch (Exception e) {
log.warn(e.getMessage(), e);
} finally {
long end = System.currentTimeMillis();
log.warn("Deserialize time - " + (end - start) + " ms.");
}
return null;
}

public String getStringFromDocument(Document doc) {
try {
DOMSource domSource = new DOMSource(doc);
StringWriter writer = new StringWriter();
StreamResult result = new StreamResult(writer);
TransformerFactory tf = TransformerFactory.newInstance();
Transformer transformer = tf.newTransformer();
transformer.transform(domSource, result);
return writer.toString();
} catch (TransformerException e) {
log.warn(e.getMessage(), e);
return "";
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package cz.zcu.kiv.eegdatabase.wui.core.experiments.metadata;

import java.io.ByteArrayOutputStream;
import java.io.IOException;

import odml.core.Section;
import odml.core.Writer;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.wicket.ajax.json.JSONException;
import org.apache.wicket.ajax.json.JSONObject;
import org.apache.wicket.ajax.json.XML;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.JsonProcessingException;
import org.codehaus.jackson.map.JsonSerializer;
import org.codehaus.jackson.map.SerializerProvider;

public class ODMLSectionSerializer extends JsonSerializer<Section> {

protected Log log = LogFactory.getLog(getClass());

@Override
public void serialize(Section value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException {
long start = System.currentTimeMillis();
try {

Writer wr = new Writer(value, true, false);
ByteArrayOutputStream stream = new ByteArrayOutputStream();
wr.write(stream, false, false);

String xmlString = stream.toString("UTF-8");
JSONObject jsonObject = XML.toJSONObject(xmlString);
String jsonString = jsonObject.toString();

jgen.writeRawValue(jsonString);

} catch (JSONException e) {
log.error(e.getMessage(), e);
} finally {
long end = System.currentTimeMillis();
log.error("Serialize time - " + (end - start) + " ms.");
}

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ public List<Section> getListOfAvailableODMLSections() {
try {
paths = new ClassPathResource(odmlSectionsPath).getFile().listFiles();
for (File path : paths) {
log.error(path.getName());
if (path.getName().endsWith(".xml")) {
filteredFiles.add(path);
}
Expand Down
Loading

0 comments on commit e0d65f0

Please sign in to comment.