Updating JPA storage provider to use the JPA2 criteria api for searches

to improve performance rather than a simpler (slower) model of pulling all results back and filtering in appsensor.
jtmelton · Sep 5, 2015 · 4ac6b9b · 4ac6b9b
1 parent 1e17389
commit 4ac6b9b
Show file tree

Hide file tree

Showing 6 changed files with 229 additions and 34 deletions.
diff --git a/...ppsensor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/Jpa2AttackStore.java b/...ppsensor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/Jpa2AttackStore.java
@@ -51,8 +51,10 @@ public void addAttack(Attack attack) {
 	 */
 	@Override
 	public Collection<Attack> findAttacks(SearchCriteria criteria) {
-		// TODO: instead of findAll every time (inefficient), update this to do actual query
-		return findAttacks(criteria, attackRepository.findAll());
+		Collection<Attack> attacksAllTimestamps = attackRepository.find(criteria);
+
+		// timestamp stored as string not queryable in DB, all timestamps come back, still need to filter this subset		
+		return findAttacks(criteria, attacksAllTimestamps);
 	}
 
 }
diff --git a/...appsensor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/Jpa2EventStore.java b/...appsensor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/Jpa2EventStore.java
@@ -50,8 +50,10 @@ public void addEvent(Event event) {
 	 */
 	@Override
 	public Collection<Event> findEvents(SearchCriteria criteria) {
-		// TODO: instead of findAll every time (inefficient), update this to do actual query
-		return findEvents(criteria, eventRepository.findAll());
+		Collection<Event> eventsAllTimestamps = eventRepository.find(criteria);
+
+		// timestamp stored as string not queryable in DB, all timestamps come back, still need to filter this subset		
+		return findEvents(criteria, eventsAllTimestamps);
 	}
 
 }
diff --git a/...sensor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/Jpa2ResponseStore.java b/...sensor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/Jpa2ResponseStore.java
@@ -1,20 +1,17 @@
 package org.owasp.appsensor.storage.jpa2;
 
-import java.util.ArrayList;
 import java.util.Collection;
 
 import javax.inject.Inject;
 import javax.inject.Named;
 
-import org.joda.time.DateTime;
+import org.owasp.appsensor.core.Attack;
 import org.owasp.appsensor.core.Event;
 import org.owasp.appsensor.core.Response;
-import org.owasp.appsensor.core.User;
 import org.owasp.appsensor.core.criteria.SearchCriteria;
 import org.owasp.appsensor.core.listener.ResponseListener;
 import org.owasp.appsensor.core.logging.Loggable;
 import org.owasp.appsensor.core.storage.ResponseStore;
-import org.owasp.appsensor.core.util.DateUtils;
 import org.owasp.appsensor.storage.jpa2.dao.ResponseRepository;
 import org.slf4j.Logger;
 
@@ -55,33 +52,10 @@ public void addResponse(Response response) {
 	 */
 	@Override
 	public Collection<Response> findResponses(SearchCriteria criteria) {
-		if (criteria == null) {
-			throw new IllegalArgumentException("criteria must be non-null");
-		}
+		Collection<Response> responsesAllTimestamps = responseRepository.find(criteria);
 
-		Collection<Response> matches = new ArrayList<Response>();
-
-		User user = criteria.getUser();
-		Collection<String> detectionSystemIds = criteria.getDetectionSystemIds(); 
-		DateTime earliest = DateUtils.fromString(criteria.getEarliest());
-
-		// TODO: instead of findAll every time (inefficient), update this to do actual query
-		for (Response response : responseRepository.findAll()) {
-			//check user match if user specified
-			boolean userMatch = (user != null) ? user.equals(response.getUser()) : true;
-
-			//check detection system match if detection systems specified
-			boolean detectionSystemMatch = (detectionSystemIds != null && detectionSystemIds.size() > 0) ? 
-					detectionSystemIds.contains(response.getDetectionSystem().getDetectionSystemId()) : true;
-
-			boolean earliestMatch = (earliest != null) ? earliest.isBefore(DateUtils.fromString(response.getTimestamp())) : true;
-
-			if (userMatch && detectionSystemMatch && earliestMatch) {
-				matches.add(response);
-			}
-		}
-
-		return matches;
+		// timestamp stored as string not queryable in DB, all timestamps come back, still need to filter this subset		
+		return findResponses(criteria, responsesAllTimestamps);
 	}
 
 }
diff --git a/...sor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/dao/AttackRepository.java b/...sor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/dao/AttackRepository.java
@@ -1,11 +1,19 @@
 package org.owasp.appsensor.storage.jpa2.dao;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.CriteriaQuery;
+import javax.persistence.criteria.Predicate;
+import javax.persistence.criteria.Root;
 
 import org.owasp.appsensor.core.Attack;
+import org.owasp.appsensor.core.criteria.SearchCriteria;
 import org.springframework.stereotype.Repository;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -58,4 +66,79 @@ public Collection<Attack> findAll() {
 		return em.createQuery("FROM Attack", Attack.class).getResultList();
 	}
 
+	/**
+	 * Retrive all {@link Attack}s from the DB matching criteria
+	 * 
+	 * @return {@link Collection} of {@link Attack}s from the DB
+	 */
+	@Transactional(readOnly = true)
+	public Collection<Attack> find(SearchCriteria searchCriteria) {
+		CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+		CriteriaQuery<Attack> criteriaQuery = criteriaBuilder.createQuery(Attack.class);
+		Root<Attack> root = criteriaQuery.from(Attack.class);
+
+		Collection<Predicate> conditions = new ArrayList<>();
+
+		if (searchCriteria.getUser() != null) {
+			Predicate userCondition = criteriaBuilder.equal(root.get("user").get("username"), searchCriteria.getUser().getUsername());
+			conditions.add(userCondition);
+		}
+
+		if (searchCriteria.getDetectionPoint() != null) {
+
+			if (searchCriteria.getDetectionPoint().getCategory() != null) {
+				Predicate categoryCondition = criteriaBuilder.equal(root.get("detectionPoint").get("category"), 
+						searchCriteria.getDetectionPoint().getCategory());
+				conditions.add(categoryCondition);
+			}
+
+			if (searchCriteria.getDetectionPoint().getLabel() != null) {
+				Predicate labelCondition = criteriaBuilder.equal(root.get("detectionPoint").get("label"), 
+						searchCriteria.getDetectionPoint().getLabel());
+				conditions.add(labelCondition);
+			}
+
+			if (searchCriteria.getDetectionPoint().getThreshold() != null) {
+
+				if (searchCriteria.getDetectionPoint().getThreshold().getCount() > 0) {
+					Predicate countCondition = criteriaBuilder.equal(root.get("detectionPoint").get("threshold").get("count"), 
+							searchCriteria.getDetectionPoint().getThreshold().getCount());
+					conditions.add(countCondition);
+				}
+
+				if (searchCriteria.getDetectionPoint().getThreshold().getInterval() != null) {
+					if (searchCriteria.getDetectionPoint().getThreshold().getInterval().getUnit() != null) {
+						Predicate durationCondition = criteriaBuilder.equal(root.get("detectionPoint").get("threshold").get("interval").get("duration"), 
+								searchCriteria.getDetectionPoint().getThreshold().getInterval().getDuration());
+						conditions.add(durationCondition);
+					}
+
+					if (searchCriteria.getDetectionPoint().getThreshold().getInterval().getDuration() > 0) {
+						Predicate unitCondition = criteriaBuilder.equal(root.get("detectionPoint").get("threshold").get("interval").get("unit"), 
+								searchCriteria.getDetectionPoint().getThreshold().getInterval().getUnit());
+						conditions.add(unitCondition);
+					}
+				}
+
+			}
+
+		}
+
+		if (searchCriteria.getDetectionSystemIds() != null) {
+			Predicate detectionSystemCondition = root.get("detectionSystem").get("detectionSystemId").in(searchCriteria.getDetectionSystemIds());
+			conditions.add(detectionSystemCondition);
+		}
+
+		if (conditions.size() > 0) {
+			criteriaQuery.where(criteriaBuilder.and(conditions.toArray(new Predicate[0])));
+		}
+
+		criteriaQuery.orderBy(criteriaBuilder.asc(root.get("timestamp")));
+
+		TypedQuery<Attack> query = em.createQuery(criteriaQuery); 
+		List<Attack> result = query.getResultList();
+
+		return result;
+	}
+
 }
diff --git a/...nsor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/dao/EventRepository.java b/...nsor-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/dao/EventRepository.java
@@ -1,11 +1,19 @@
 package org.owasp.appsensor.storage.jpa2.dao;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.CriteriaQuery;
+import javax.persistence.criteria.Predicate;
+import javax.persistence.criteria.Root;
 
 import org.owasp.appsensor.core.Event;
+import org.owasp.appsensor.core.criteria.SearchCriteria;
 import org.springframework.stereotype.Repository;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -58,4 +66,85 @@ public Collection<Event> findAll() {
 		return em.createQuery("FROM Event", Event.class).getResultList();
 	}
 
+	/**
+	 * Retrive all {@link Event}s from the DB
+	 * 
+	 * @return {@link Collection} of {@link Event}s from the DB
+	 */
+	@Transactional(readOnly = true)
+	public Collection<Event> find(SearchCriteria searchCriteria) {
+
+//		if user exists, compare on [username]
+//		if detection point exists, compare on [category, label, threshold count, threshold interval]
+//		if detection system ids exist, do an "in"
+//		if earliest exists, compare on [event date > earliest]
+
+		CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+		CriteriaQuery<Event> criteriaQuery = criteriaBuilder.createQuery(Event.class);
+		Root<Event> root = criteriaQuery.from(Event.class);
+
+		Collection<Predicate> conditions = new ArrayList<>();
+
+		if (searchCriteria.getUser() != null) {
+			Predicate userCondition = criteriaBuilder.equal(root.get("user").get("username"), searchCriteria.getUser().getUsername());
+			conditions.add(userCondition);
+		}
+
+		if (searchCriteria.getDetectionPoint() != null) {
+
+			if (searchCriteria.getDetectionPoint().getCategory() != null) {
+				Predicate categoryCondition = criteriaBuilder.equal(root.get("detectionPoint").get("category"), 
+						searchCriteria.getDetectionPoint().getCategory());
+				conditions.add(categoryCondition);
+			}
+
+			if (searchCriteria.getDetectionPoint().getLabel() != null) {
+				Predicate labelCondition = criteriaBuilder.equal(root.get("detectionPoint").get("label"), 
+						searchCriteria.getDetectionPoint().getLabel());
+				conditions.add(labelCondition);
+			}
+
+			if (searchCriteria.getDetectionPoint().getThreshold() != null) {
+
+				if (searchCriteria.getDetectionPoint().getThreshold().getCount() > 0) {
+					Predicate countCondition = criteriaBuilder.equal(root.get("detectionPoint").get("threshold").get("count"), 
+							searchCriteria.getDetectionPoint().getThreshold().getCount());
+					conditions.add(countCondition);
+				}
+
+				if (searchCriteria.getDetectionPoint().getThreshold().getInterval() != null) {
+					if (searchCriteria.getDetectionPoint().getThreshold().getInterval().getUnit() != null) {
+						Predicate durationCondition = criteriaBuilder.equal(root.get("detectionPoint").get("threshold").get("interval").get("duration"), 
+								searchCriteria.getDetectionPoint().getThreshold().getInterval().getDuration());
+						conditions.add(durationCondition);
+					}
+
+					if (searchCriteria.getDetectionPoint().getThreshold().getInterval().getDuration() > 0) {
+						Predicate unitCondition = criteriaBuilder.equal(root.get("detectionPoint").get("threshold").get("interval").get("unit"), 
+								searchCriteria.getDetectionPoint().getThreshold().getInterval().getUnit());
+						conditions.add(unitCondition);
+					}
+				}
+
+			}
+
+		}
+
+		if (searchCriteria.getDetectionSystemIds() != null) {
+			Predicate detectionSystemCondition = root.get("detectionSystem").get("detectionSystemId").in(searchCriteria.getDetectionSystemIds());
+			conditions.add(detectionSystemCondition);
+		}
+
+		if (conditions.size() > 0) {
+			criteriaQuery.where(criteriaBuilder.and(conditions.toArray(new Predicate[0])));
+		}
+
+		criteriaQuery.orderBy(criteriaBuilder.asc(root.get("timestamp")));
+
+		TypedQuery<Event> query = em.createQuery(criteriaQuery); 
+		List<Event> result = query.getResultList();
+
+		return result;
+	}
+
 }
diff --git a/...r-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/dao/ResponseRepository.java b/...r-storage-jpa2/src/main/java/org/owasp/appsensor/storage/jpa2/dao/ResponseRepository.java
@@ -1,11 +1,21 @@
 package org.owasp.appsensor.storage.jpa2.dao;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.CriteriaQuery;
+import javax.persistence.criteria.Predicate;
+import javax.persistence.criteria.Root;
 
+import org.owasp.appsensor.core.Attack;
 import org.owasp.appsensor.core.Response;
+import org.owasp.appsensor.core.criteria.SearchCriteria;
+import org.owasp.appsensor.core.util.DateUtils;
 import org.springframework.stereotype.Repository;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -58,4 +68,39 @@ public Collection<Response> findAll() {
 		return em.createQuery("FROM Response", Response.class).getResultList();
 	}
 
+	/**
+	 * Retrive all {@link Response}s from the DB matching criteria
+	 * 
+	 * @return {@link Collection} of {@link Response}s from the DB
+	 */
+	@Transactional(readOnly = true)
+	public Collection<Response> find(SearchCriteria searchCriteria) {
+		CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+		CriteriaQuery<Response> criteriaQuery = criteriaBuilder.createQuery(Response.class);
+		Root<Response> root = criteriaQuery.from(Response.class);
+
+		Collection<Predicate> conditions = new ArrayList<>();
+
+		if (searchCriteria.getUser() != null) {
+			Predicate userCondition = criteriaBuilder.equal(root.get("user").get("username"), searchCriteria.getUser().getUsername());
+			conditions.add(userCondition);
+		}
+
+		if (searchCriteria.getDetectionSystemIds() != null) {
+			Predicate detectionSystemCondition = root.get("detectionSystem").get("detectionSystemId").in(searchCriteria.getDetectionSystemIds());
+			conditions.add(detectionSystemCondition);
+		}
+
+		if (conditions.size() > 0) {
+			criteriaQuery.where(criteriaBuilder.and(conditions.toArray(new Predicate[0])));
+		}
+
+		criteriaQuery.orderBy(criteriaBuilder.asc(root.get("timestamp")));
+
+		TypedQuery<Response> query = em.createQuery(criteriaQuery); 
+		List<Response> result = query.getResultList();
+
+		return result;
+	}
+
 }