diff --git a/oskar-analysis/pom.xml b/oskar-analysis/pom.xml index 4277cb0..2633d2a 100644 --- a/oskar-analysis/pom.xml +++ b/oskar-analysis/pom.xml @@ -26,10 +26,18 @@ commons-math3 ${commons-math.version} + + com.google.guava + guava + + + + 19.0 + org.reflections reflections - 0.9.11 + 0.9.10 junit diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AbstractAnalysis.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AbstractAnalysis.java deleted file mode 100644 index e771fd5..0000000 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AbstractAnalysis.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.opencb.oskar.analysis; - -import org.apache.commons.lang3.StringUtils; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.exceptions.AnalysisException; -import org.opencb.oskar.core.annotations.AnalysisExecutor; -import org.reflections.Reflections; -import org.reflections.scanners.SubTypesScanner; -import org.reflections.scanners.TypeAnnotationsScanner; - -import java.nio.file.Path; -import java.util.Set; - -public abstract class AbstractAnalysis { - - protected ObjectMap executorParams; - protected Path outDir; - - protected AbstractAnalysis() { - } - - protected AbstractAnalysis(ObjectMap executorParams, Path outDir) { - this.executorParams = executorParams; - this.outDir = outDir; - } - - protected Class getAnalysisExecutorId(String analysisExecutorId) { - return getAnalysisExecutorId(analysisExecutorId, ""); - } - - protected Class getAnalysisExecutorId(String analysisExecutorId, String analysisId) { - Reflections reflections = new Reflections("", new SubTypesScanner(), new TypeAnnotationsScanner()); - Set> typesAnnotatedWith = reflections.getTypesAnnotatedWith(AnalysisExecutor.class); - for (Class aClass : typesAnnotatedWith) { - AnalysisExecutor annotation = aClass.getAnnotation(AnalysisExecutor.class); - if (StringUtils.isNotEmpty(analysisId)) { - if (annotation.id().equals(analysisExecutorId) && annotation.analysis().equals(analysisId)) { - return aClass; - } - } else { - if (annotation.id().equals(analysisExecutorId)) { - return aClass; - } - } - } - return null; - } - - public abstract AnalysisResult execute() throws AnalysisException; -} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AbstractAnalysisExecutor.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AbstractAnalysisExecutor.java deleted file mode 100644 index 4dac5e2..0000000 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AbstractAnalysisExecutor.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.opencb.oskar.analysis; - -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.exceptions.AnalysisException; - -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -public abstract class AbstractAnalysisExecutor { - - protected ObjectMap executorParams; - protected Path outDir; - - protected AbstractAnalysisExecutor() { - } - - protected AbstractAnalysisExecutor(ObjectMap executorParams, Path outDir) { - setup(executorParams, outDir); - } - - protected void setup(ObjectMap executorParams, Path outDir) { - this.executorParams = executorParams; - this.outDir = outDir; - } - - public abstract AnalysisResult exec() throws AnalysisException; - - public String getDateTime() { - DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"); - LocalDateTime now = LocalDateTime.now(); - return dtf.format(now); - } - - public ObjectMap getExecutorParams() { - return executorParams; - } - - public AbstractAnalysisExecutor setExecutorParams(ObjectMap executorParams) { - this.executorParams = executorParams; - return this; - } - - public Path getOutDir() { - return outDir; - } - - public AbstractAnalysisExecutor setOutDir(Path outDir) { - this.outDir = outDir; - return this; - } -} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AnalysisResult.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AnalysisResult.java deleted file mode 100644 index b5f79d4..0000000 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/AnalysisResult.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.opencb.oskar.analysis; - -import org.opencb.commons.datastore.core.ObjectMap; - -import java.nio.file.Path; -import java.util.List; - -public class AnalysisResult { - public enum FileType { - IMAGE, - JSON, - TAB_SEPARATED // First line starts with # and contains the header - } - - String dateTime; - String analysisId; - ObjectMap executorParams; - List outputFiles; - long executionTime; - - public static class File { - Path path; - FileType type; - - public File(Path path, FileType type) { - this.path = path; - this.type = type; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("File{"); - sb.append("path=").append(path); - sb.append(", type=").append(type); - sb.append('}'); - return sb.toString(); - } - - public Path getPath() { - return path; - } - - public File setPath(Path path) { - this.path = path; - return this; - } - - public FileType getType() { - return type; - } - - public File setType(FileType type) { - this.type = type; - return this; - } - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("AnalysisResult{"); - sb.append("dateTime='").append(dateTime).append('\''); - sb.append(", analysisId='").append(analysisId).append('\''); - sb.append(", executorParams=").append(executorParams); - sb.append(", outputFiles=").append(outputFiles); - sb.append(", executionTime=").append(executionTime); - sb.append('}'); - return sb.toString(); - } - - public String getDateTime() { - return dateTime; - } - - public AnalysisResult setDateTime(String dateTime) { - this.dateTime = dateTime; - return this; - } - - public String getAnalysisId() { - return analysisId; - } - - public AnalysisResult setAnalysisId(String analysisId) { - this.analysisId = analysisId; - return this; - } - - public ObjectMap getExecutorParams() { - return executorParams; - } - - public AnalysisResult setExecutorParams(ObjectMap executorParams) { - this.executorParams = executorParams; - return this; - } - - public List getOutputFiles() { - return outputFiles; - } - - public AnalysisResult setOutputFiles(List outputFiles) { - this.outputFiles = outputFiles; - return this; - } - - public long getExecutionTime() { - return executionTime; - } - - public AnalysisResult setExecutionTime(long executionTime) { - this.executionTime = executionTime; - return this; - } -} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/OskarAnalysis.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/OskarAnalysis.java index a5d10a0..e73274d 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/OskarAnalysis.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/OskarAnalysis.java @@ -1,41 +1,169 @@ package org.opencb.oskar.analysis; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.oskar.analysis.exceptions.AnalysisException; import org.opencb.oskar.analysis.exceptions.AnalysisExecutorException; -import org.opencb.oskar.core.config.OskarConfiguration; +import org.opencb.oskar.analysis.result.AnalysisResult; +import org.opencb.oskar.analysis.result.AnalysisResultManager; +import org.opencb.oskar.core.annotations.Analysis; +import org.opencb.oskar.core.annotations.AnalysisExecutor; +import org.reflections.Reflections; +import org.reflections.scanners.SubTypesScanner; +import org.reflections.scanners.TypeAnnotationsScanner; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; +import java.nio.file.Path; +import java.util.*; public abstract class OskarAnalysis { - protected String studyId; - protected OskarConfiguration configuration; + protected ObjectMap executorParams; + protected Path outDir; + protected AnalysisExecutor.Source source; + protected Set availableFrameworks; - protected OskarAnalysis(String studyId, OskarConfiguration configuration) { - this.studyId = studyId; - this.configuration = configuration; + protected AnalysisResultManager arm; + + public OskarAnalysis() { + } + + public OskarAnalysis(ObjectMap executorParams, Path outDir) { + setUp(executorParams, outDir, null, null); + } + + /** + * Setup the analysis providing the parameters required for the execution. + * @param executorParams Params to be provided to the Executor + * @param outDir Output directory + * @param inputDataSource Input data source type + * @param availableFrameworks Available frameworks in this environment + */ + public final void setUp(ObjectMap executorParams, Path outDir, + AnalysisExecutor.Source inputDataSource, + Collection availableFrameworks) { + this.executorParams = executorParams; + this.outDir = outDir; + this.source = inputDataSource; + this.availableFrameworks = availableFrameworks == null ? null : new HashSet<>(availableFrameworks); + } + + /** + * Execute the analysis. The analysis should have been properly setup before being executed. + * + * @return AnalysisResult + * @throws AnalysisException on error + */ + public final AnalysisResult execute() throws AnalysisException { + arm = new AnalysisResultManager(outDir); + arm.init(getId(), executorParams); + try { + check(); + exec(); + return arm.close(); + } catch (Exception e) { + arm.close(e); + throw e; + } + } + + /** + * Check that the given parameters are correct. + * This method will be called before the {@link #exec()}. + * + * @throws AnalysisException if the parameters are not correct + */ + protected void check() throws AnalysisException { } - protected abstract void execute() throws AnalysisExecutorException; + /** + * Method to be implemented by subclasses with the actual execution of the analysis. + * @throws AnalysisException on error + */ + protected abstract void exec() throws AnalysisException; - @Override - public String toString() { - return "AnalysisExecutor{" + "studyId='" + studyId + '\'' + ", configuration=" + configuration + '}'; + /** + * @return the analysis id + */ + public final String getId() { + return this.getClass().getAnnotation(Analysis.class).id(); } - public String getStudyId() { - return studyId; + protected final Class getAnalysisExecutorClass(String analysisExecutorId) { + return getAnalysisExecutorClass(OskarAnalysisExecutor.class, analysisExecutorId, null, null); } - public OskarAnalysis setStudyId(String studyId) { - this.studyId = studyId; - return this; + protected final Class getAnalysisExecutorClass( + Class clazz, String analysisExecutorId, AnalysisExecutor.Source source, + Set availableFrameworks) { + Objects.requireNonNull(clazz); + String analysisId = getId(); + + if (source == null) { + source = this.source; + } + if (CollectionUtils.isEmpty(availableFrameworks)) { + availableFrameworks = this.availableFrameworks; + } + + TypeAnnotationsScanner annotationsScanner = new TypeAnnotationsScanner(); + annotationsScanner.setResultFilter(s -> StringUtils.equals(s, AnalysisExecutor.class.getName())); + + Reflections reflections = new Reflections(new ConfigurationBuilder() + .setScanners(new SubTypesScanner(), annotationsScanner) + .addUrls(ClasspathHelper.forJavaClassPath()) + .filterInputsBy(input -> input.endsWith(".class"))); + + Set> typesAnnotatedWith = reflections.getSubTypesOf(clazz); + for (Class aClass : typesAnnotatedWith) { + AnalysisExecutor annotation = aClass.getAnnotation(AnalysisExecutor.class); + if (annotation != null) { + if (annotation.analysis().equals(analysisId)) { + if (StringUtils.isEmpty(analysisExecutorId) || analysisExecutorId.equals(annotation.id())) { + if (source == null || annotation.source() == source) { + if (CollectionUtils.isEmpty(availableFrameworks) || availableFrameworks.contains(annotation.framework())) { + return aClass; + } + } + } + } + } + } + return null; } - public OskarConfiguration getConfiguration() { - return configuration; + protected final OskarAnalysisExecutor getAnalysisExecutor() + throws AnalysisExecutorException { + return getAnalysisExecutor(OskarAnalysisExecutor.class, null, null, null); } - public OskarAnalysis setConfiguration(OskarConfiguration configuration) { - this.configuration = configuration; - return this; + protected final T getAnalysisExecutor(Class clazz) + throws AnalysisExecutorException { + return getAnalysisExecutor(clazz, null, null, null); } + + protected final T getAnalysisExecutor(Class clazz, String analysisExecutorId) + throws AnalysisExecutorException { + return getAnalysisExecutor(clazz, analysisExecutorId, null, null); + } + + protected final T getAnalysisExecutor( + Class clazz, String analysisExecutorId, AnalysisExecutor.Source source, + Set availableFrameworks) + throws AnalysisExecutorException { + Class executorClass = getAnalysisExecutorClass(clazz, analysisExecutorId, source, availableFrameworks); + if (executorClass == null) { + throw AnalysisExecutorException.executorNotFound(clazz, getId(), analysisExecutorId, source, availableFrameworks); + } + try { + T t = executorClass.newInstance(); + t.init(arm); + return t; + } catch (InstantiationException | IllegalAccessException e) { + throw AnalysisExecutorException.cantInstantiate(executorClass, e); + } + } + } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/OskarAnalysisExecutor.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/OskarAnalysisExecutor.java new file mode 100644 index 0000000..d3887f3 --- /dev/null +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/OskarAnalysisExecutor.java @@ -0,0 +1,49 @@ +package org.opencb.oskar.analysis; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.result.AnalysisResultManager; +import org.opencb.oskar.core.annotations.AnalysisExecutor; + +import java.nio.file.Path; + +public abstract class OskarAnalysisExecutor { + + protected ObjectMap executorParams; + protected Path outDir; + protected AnalysisResultManager arm; + + protected OskarAnalysisExecutor() { + } + + protected OskarAnalysisExecutor(ObjectMap executorParams, Path outDir) { + setup(executorParams, outDir); + } + + public final void init(AnalysisResultManager arm) { + this.arm = arm; + } + + public final String getAnalysisId() { + return this.getClass().getAnnotation(AnalysisExecutor.class).analysis(); + } + + public final String getId() { + return this.getClass().getAnnotation(AnalysisExecutor.class).id(); + } + + protected final void setup(ObjectMap executorParams, Path outDir) { + this.executorParams = executorParams; + this.outDir = outDir; + } + + public abstract void exec() throws AnalysisException; + + public final ObjectMap getExecutorParams() { + return executorParams; + } + + public final Path getOutDir() { + return outDir; + } +} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/exceptions/AnalysisExecutorException.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/exceptions/AnalysisExecutorException.java index dc42fb6..edd4e5d 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/exceptions/AnalysisExecutorException.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/exceptions/AnalysisExecutorException.java @@ -1,12 +1,16 @@ package org.opencb.oskar.analysis.exceptions; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.oskar.analysis.OskarAnalysisExecutor; +import org.opencb.oskar.core.annotations.AnalysisExecutor; + +import java.util.Set; + /** * Created by jtarraga on 30/01/17. */ -public class AnalysisExecutorException extends Exception { - - public AnalysisExecutorException() { - } +public class AnalysisExecutorException extends AnalysisException { public AnalysisExecutorException(String msg) { super(msg); @@ -28,7 +32,28 @@ public AnalysisExecutorException(Throwable cause) { super(cause); } - public AnalysisExecutorException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); + public static AnalysisExecutorException executorNotFound(Class clazz, String analysis, String executorId, + AnalysisExecutor.Source source, Set frameworks) { + + String requirements = ""; + if (clazz != OskarAnalysisExecutor.class) { + requirements = " extending class " + clazz; + } + if (StringUtils.isNotEmpty(executorId)) { + requirements = " with executorId='" + executorId + "'"; + } + if (source != null) { + requirements = " for source ='" + source + "'"; + } + if (CollectionUtils.isNotEmpty(frameworks)) { + requirements = " for frameworks=" + frameworks; + } + requirements += "."; + return new AnalysisExecutorException("Could not find a valid Analysis executor for the analysis '" + analysis + "'" + requirements); } + + public static AnalysisExecutorException cantInstantiate(Class clazz, Exception cause) { + return new AnalysisExecutorException("Could not create class an instance of class " + clazz, cause); + } + } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisResult.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisResult.java new file mode 100644 index 0000000..a6a032c --- /dev/null +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisResult.java @@ -0,0 +1,121 @@ +package org.opencb.oskar.analysis.result; + +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.Date; +import java.util.LinkedList; +import java.util.List; + +public class AnalysisResult { + + private String id; + private String executorId; + private ObjectMap executorParams; + private Date start; + private Date end; + private Status status; + private List warnings; + private List outputFiles; + private List steps; + + private ObjectMap attributes; + + public AnalysisResult() { + executorParams = new ObjectMap(); + status = new Status(); + warnings = new LinkedList<>(); + outputFiles = new LinkedList<>(); + steps = new LinkedList<>(); + attributes = new ObjectMap(); + } + + public String getId() { + return id; + } + + public AnalysisResult setId(String id) { + this.id = id; + return this; + } + + public String getExecutorId() { + return executorId; + } + + public AnalysisResult setExecutorId(String executorId) { + this.executorId = executorId; + return this; + } + + public ObjectMap getExecutorParams() { + return executorParams; + } + + public AnalysisResult setExecutorParams(ObjectMap executorParams) { + this.executorParams = executorParams; + return this; + } + + public Date getStart() { + return start; + } + + public AnalysisResult setStart(Date start) { + this.start = start; + return this; + } + + public Date getEnd() { + return end; + } + + public AnalysisResult setEnd(Date end) { + this.end = end; + return this; + } + + public Status getStatus() { + return status; + } + + public AnalysisResult setStatus(Status status) { + this.status = status; + return this; + } + + public List getWarnings() { + return warnings; + } + + public AnalysisResult setWarnings(List warnings) { + this.warnings = warnings; + return this; + } + + public List getOutputFiles() { + return outputFiles; + } + + public AnalysisResult setOutputFiles(List outputFiles) { + this.outputFiles = outputFiles; + return this; + } + + public List getSteps() { + return steps; + } + + public AnalysisResult setSteps(List steps) { + this.steps = steps; + return this; + } + + public ObjectMap getAttributes() { + return attributes; + } + + public AnalysisResult setAttributes(ObjectMap attributes) { + this.attributes = attributes; + return this; + } +} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisResultManager.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisResultManager.java new file mode 100644 index 0000000..1da3792 --- /dev/null +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisResultManager.java @@ -0,0 +1,179 @@ +package org.opencb.oskar.analysis.result; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.oskar.analysis.exceptions.AnalysisException; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.function.Consumer; + +public class AnalysisResultManager { + + public static final String FILE_NAME = "status.json"; + public static final String RUNNING = "RUNNING"; + public static final String DONE = "DONE"; + public static final String ERROR = "ERROR"; + private final Path outDir; + private final ObjectWriter objectWriter; + private final ObjectReader objectReader; + + private File file; + private boolean initialized; + private boolean closed; + + public AnalysisResultManager(Path outDir) { + this.outDir = outDir.toAbsolutePath(); + ObjectMapper objectMapper = new ObjectMapper(); + objectWriter = objectMapper.writerFor(AnalysisResult.class).withDefaultPrettyPrinter(); + objectReader = objectMapper.readerFor(AnalysisResult.class); + initialized = false; + closed = false; + } + + public synchronized AnalysisResultManager init(String analysisId, ObjectMap executorParams) throws AnalysisException { + if (initialized) { + throw new AnalysisException("AnalysisResultManager already initialized!"); + } + initialized = true; + file = outDir.toFile(); + + if (!file.exists()) { + throw new AnalysisException("Output directory '" + outDir + "' does not exist"); + } + if (!file.isDirectory()) { + throw new AnalysisException("Output directory '" + outDir + "' does not a directory"); + } + if (!file.canWrite()) { + throw new AnalysisException("Write permission denied for output directory '" + outDir + "'"); + } + + file = outDir.resolve(FILE_NAME).toFile(); + Date now = now(); + AnalysisResult analysisResult = new AnalysisResult() + .setId(analysisId) + .setStart(now) + .setExecutorParams(executorParams); + analysisResult.getStatus() + .setDate(now) + .setId(RUNNING); + + write(analysisResult); + return this; + } + + public synchronized AnalysisResult close() throws AnalysisException { + return close(null); + } + + public AnalysisResult close(Exception exception) throws AnalysisException { + if (closed) { + throw new AnalysisException("AnalysisResultManager already closed!"); + } + closed = true; + + AnalysisResult analysisResult = read(); + + Date now = now(); + analysisResult.setEnd(now); + analysisResult.getStatus() + .setDate(now); + if (exception == null) { + analysisResult.getStatus() + .setCompletedPercentage(100) + .setStep("") + .setId(DONE); + } else { + analysisResult.getStatus() + .setId(ERROR); + } + + write(analysisResult); + return analysisResult; + } + + public void addWarning(String warningMessage) throws AnalysisException { + updateResult(analysisResult -> analysisResult.getWarnings().add(warningMessage)); + } + + public void addFile(Path file, FileResult.FileType fileType) throws AnalysisException { + String fileStr = file.toAbsolutePath().toString(); + String outDirStr = outDir.toString(); + String finalFileStr; + if (fileStr.startsWith(outDirStr)) { + finalFileStr = fileStr.substring(outDirStr.length()); + } else { + finalFileStr = fileStr; + } + updateResult(analysisResult -> analysisResult.getOutputFiles().add(new FileResult(finalFileStr, fileType))); + } + + public void startStep(String stepId) throws AnalysisException { + updateResult(analysisResult -> { + analysisResult.getStatus().setStep(stepId); + analysisResult + .getSteps() + .add(new AnalysisStep() + .setId(stepId) + .setStart(now()) + .setStatus(new Status() + .setId(RUNNING) + .setDate(now()) + ) + ); + }); + } + + public void endStep(float newTotalPercentage) throws AnalysisException { + updateResult(analysisResult -> { + AnalysisStep step = analysisResult.getSteps().get(analysisResult.getSteps().size() - 1); + step.setEnd(now()); + step.getStatus().setId(DONE).setCompletedPercentage(100); + analysisResult.getStatus() + .setStep("") + .setCompletedPercentage(newTotalPercentage); + }); + } + + public void updateResult(Consumer update) throws AnalysisException { + AnalysisResult analysisResult = read(); + update.accept(analysisResult); + write(analysisResult); + } + + private AnalysisResult read() throws AnalysisException { + try { + return objectReader.readValue(file); + } catch (IOException e) { + throw new AnalysisException("Error reading AnalysisResult", e); + } + + } + + private void write(AnalysisResult analysisResult) throws AnalysisException { + try { + objectWriter.writeValue(file, analysisResult); + } catch (IOException e) { + throw new AnalysisException("Error writing AnalysisResult", e); + } + } + + private Date now() { + return Date.from(Instant.now()); + } + + public String getDateTime() { + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"); + LocalDateTime now = LocalDateTime.now(); + return dtf.format(now); + } + +} + diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisStep.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisStep.java new file mode 100644 index 0000000..54fc844 --- /dev/null +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/AnalysisStep.java @@ -0,0 +1,72 @@ +package org.opencb.oskar.analysis.result; + +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.Date; + +public class AnalysisStep { + + private String id; + private Date start; + private Date end; + private Status status; + private ObjectMap attributes; + + public AnalysisStep() { + attributes = new ObjectMap(); + status = new Status(); + } + + public AnalysisStep(String id, Date start, Date end, Status status, ObjectMap attributes) { + this.id = id; + this.start = start; + this.end = end; + this.status = status; + this.attributes = attributes; + } + + public String getId() { + return id; + } + + public AnalysisStep setId(String id) { + this.id = id; + return this; + } + + public Date getStart() { + return start; + } + + public AnalysisStep setStart(Date start) { + this.start = start; + return this; + } + + public Date getEnd() { + return end; + } + + public AnalysisStep setEnd(Date end) { + this.end = end; + return this; + } + + public Status getStatus() { + return status; + } + + public AnalysisStep setStatus(Status status) { + this.status = status; + return this; + } + + public ObjectMap getAttributes() { + return attributes; + } + + public AnalysisStep setAttributes(ObjectMap attributes) { + this.attributes = attributes; + return this; + } +} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/FileResult.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/FileResult.java new file mode 100644 index 0000000..836cb8c --- /dev/null +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/FileResult.java @@ -0,0 +1,47 @@ +package org.opencb.oskar.analysis.result; + +public class FileResult { + public enum FileType { + IMAGE, + JSON, + TAB_SEPARATED // First line starts with # and contains the header + } + + private String path; + private FileType type; + + public FileResult() { + } + + public FileResult(String path, FileType type) { + this.path = path; + this.type = type; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("File{"); + sb.append("path=").append(path); + sb.append(", type=").append(type); + sb.append('}'); + return sb.toString(); + } + + public String getPath() { + return path; + } + + public FileResult setPath(String path) { + this.path = path; + return this; + } + + public FileType getType() { + return type; + } + + public FileResult setType(FileType type) { + this.type = type; + return this; + } +} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/Status.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/Status.java new file mode 100644 index 0000000..2301dbe --- /dev/null +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/result/Status.java @@ -0,0 +1,56 @@ +package org.opencb.oskar.analysis.result; + +import java.util.Date; + +public class Status { + private String id; + private String step; + private float completedPercentage; + private Date date; + + public Status() { + } + + public Status(String id, String step, float completedPercentage, Date date) { + this.id = id; + this.step = step; + this.completedPercentage = completedPercentage; + this.date = date; + } + + public String getId() { + return id; + } + + public Status setId(String id) { + this.id = id; + return this; + } + + public String getStep() { + return step; + } + + public Status setStep(String step) { + this.step = step; + return this; + } + + public float getCompletedPercentage() { + return completedPercentage; + } + + public Status setCompletedPercentage(float completedPercentage) { + this.completedPercentage = completedPercentage; + return this; + } + + public Date getDate() { + return date; + } + + public Status setDate(Date date) { + this.date = date; + return this; + } +} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/VariantOskarAnalysis.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/VariantOskarAnalysis.java index 10e5ae3..366edf8 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/VariantOskarAnalysis.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/VariantOskarAnalysis.java @@ -1,14 +1,10 @@ package org.opencb.oskar.analysis.variant; import org.opencb.oskar.analysis.OskarAnalysis; -import org.opencb.oskar.core.config.OskarConfiguration; /** * Created by jtarraga on 30/05/17. */ public abstract class VariantOskarAnalysis extends OskarAnalysis { - protected VariantOskarAnalysis(String studyId, OskarConfiguration configuration) { - super(studyId, configuration); - } } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/Gwas.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/Gwas.java index 74b2966..b22c87b 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/Gwas.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/Gwas.java @@ -3,8 +3,7 @@ import org.apache.commons.lang.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.CollectionUtils; -import org.opencb.oskar.analysis.AbstractAnalysis; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.OskarAnalysis; import org.opencb.oskar.analysis.exceptions.AnalysisException; import org.opencb.oskar.core.annotations.Analysis; @@ -12,7 +11,7 @@ import java.util.List; @Analysis(id = Gwas.ID, data = Analysis.AnalysisData.VARIANT) -public class Gwas extends AbstractAnalysis { +public class Gwas extends OskarAnalysis { public static final String ID = "GWAS"; @@ -37,32 +36,32 @@ public Gwas(String phenotype, ObjectMap executorParams, Path outDir, GwasConfigu /** * Checks if list and list2 are not empty and no common samples exist. */ - protected void checkSamples() { + @Override + protected void check() { + // checks } protected void createManhattanPlot() { } @Override - public AnalysisResult execute() throws AnalysisException { - // checks + protected void exec() throws AnalysisException { + GwasExecutor gwasExecutor = getAnalysisExecutor(GwasExecutor.class, executorParams.getString("ID")); - AnalysisResult analysisResult; - Class executor = getAnalysisExecutorId(executorParams.getString("ID"), Gwas.ID); - try { - AbstractGwasExecutor gwasExecutor = (AbstractGwasExecutor) executor.newInstance(); - if (CollectionUtils.isNotEmpty(list1) && CollectionUtils.isNotEmpty(list2)) { - gwasExecutor.setup(list1, list2, executorParams, outDir, configuration); - } else if (StringUtils.isNotEmpty(phenotype)) { - gwasExecutor.setup(phenotype, executorParams, outDir, configuration); - } else { - throw new AnalysisException("Invalid input parameters for GWAS analysis"); - } - analysisResult = gwasExecutor.exec(); - } catch (InstantiationException | IllegalAccessException e) { - throw new AnalysisException("Error when executing GWAS analysis", e); + if (CollectionUtils.isNotEmpty(list1) && CollectionUtils.isNotEmpty(list2)) { + gwasExecutor.setup(list1, list2, executorParams, outDir, configuration); + } else if (StringUtils.isNotEmpty(phenotype)) { + gwasExecutor.setup(phenotype, executorParams, outDir, configuration); + } else { + throw new AnalysisException("Invalid input parameters for GWAS analysis"); } - return analysisResult; + arm.startStep("gwas"); + gwasExecutor.exec(); + arm.endStep(70); + + arm.startStep("manhattan-plot"); + createManhattanPlot(); + arm.endStep(100); } } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/AbstractGwasExecutor.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/GwasExecutor.java similarity index 71% rename from oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/AbstractGwasExecutor.java rename to oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/GwasExecutor.java index b6ad236..d75031e 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/AbstractGwasExecutor.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/gwas/GwasExecutor.java @@ -1,33 +1,32 @@ package org.opencb.oskar.analysis.variant.gwas; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysisExecutor; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.OskarAnalysisExecutor; import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.result.FileResult; import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; -public abstract class AbstractGwasExecutor extends AbstractAnalysisExecutor { +public abstract class GwasExecutor extends OskarAnalysisExecutor { private List list1; private List list2; private String phenotype; private GwasConfiguration configuration; - public AbstractGwasExecutor() { + public GwasExecutor() { } - public AbstractGwasExecutor(List list1, List list2, ObjectMap executorParams, Path outDir, - GwasConfiguration configuration) { + public GwasExecutor(List list1, List list2, ObjectMap executorParams, Path outDir, + GwasConfiguration configuration) { this.setup(list1, list2, executorParams, outDir, configuration); } - public AbstractGwasExecutor(String phenotype, ObjectMap executorParams, Path outDir, - GwasConfiguration configuration) { + public GwasExecutor(String phenotype, ObjectMap executorParams, Path outDir, + GwasConfiguration configuration) { this.setup(phenotype, executorParams, outDir, configuration); } @@ -70,20 +69,16 @@ protected String getOutputFilename() throws AnalysisException { } } - protected AnalysisResult createAnalysisResult() throws AnalysisException { - List resultFiles = new ArrayList<>(); + protected void registerFiles() throws AnalysisException { String outFilename = getOutDir() + "/" + getOutputFilename(); if (new File(outFilename).exists()) { - resultFiles.add(new AnalysisResult.File(Paths.get(outFilename), AnalysisResult.FileType.TAB_SEPARATED)); + arm.addFile(Paths.get(outFilename), FileResult.FileType.TAB_SEPARATED); } - - return new AnalysisResult().setAnalysisId(Gwas.ID).setDateTime(getDateTime()).setExecutorParams(executorParams) - .setOutputFiles(resultFiles); } @Override public String toString() { - final StringBuilder sb = new StringBuilder("AbstractGwasExecutor{"); + final StringBuilder sb = new StringBuilder("GwasExecutor{"); sb.append("list1=").append(list1); sb.append(", list2=").append(list2); sb.append(", phenotype='").append(phenotype).append('\''); @@ -98,7 +93,7 @@ public List getList1() { return list1; } - public AbstractGwasExecutor setList1(List list1) { + public GwasExecutor setList1(List list1) { this.list1 = list1; return this; } @@ -107,7 +102,7 @@ public List getList2() { return list2; } - public AbstractGwasExecutor setList2(List list2) { + public GwasExecutor setList2(List list2) { this.list2 = list2; return this; } @@ -116,7 +111,7 @@ public String getPhenotype() { return phenotype; } - public AbstractGwasExecutor setPhenotype(String phenotype) { + public GwasExecutor setPhenotype(String phenotype) { this.phenotype = phenotype; return this; } @@ -125,7 +120,7 @@ public GwasConfiguration getConfiguration() { return configuration; } - public AbstractGwasExecutor setConfiguration(GwasConfiguration configuration) { + public GwasExecutor setConfiguration(GwasConfiguration configuration) { this.configuration = configuration; return this; } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractCohortStatsExecutor.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractCohortStatsExecutor.java deleted file mode 100644 index b37cf52..0000000 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractCohortStatsExecutor.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.opencb.oskar.analysis.variant.stats; - -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysisExecutor; -import org.opencb.oskar.analysis.AnalysisResult; -import org.opencb.oskar.analysis.exceptions.AnalysisException; - -import java.nio.file.Path; - -public class AbstractCohortStatsExecutor extends AbstractAnalysisExecutor { - - public AbstractCohortStatsExecutor() { - } - - public AbstractCohortStatsExecutor(ObjectMap executorParams, Path outDir) { - this.setup(executorParams, outDir); - } - - protected void setup(ObjectMap executorParams, Path outDir) { - super.setup(executorParams, outDir); - } - - @Override - public AnalysisResult exec() throws AnalysisException { - return null; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("AbstractCohortStatsExecutor{"); - sb.append(", executorParams=").append(executorParams); - sb.append(", outDir=").append(outDir); - sb.append('}'); - return sb.toString(); - } -} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStats.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStats.java deleted file mode 100644 index dbced72..0000000 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStats.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.opencb.oskar.analysis.variant.stats; - -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysis; -import org.opencb.oskar.analysis.AnalysisResult; -import org.opencb.oskar.analysis.exceptions.AnalysisException; -import org.opencb.oskar.core.annotations.Analysis; - -import java.nio.file.Path; - -@Analysis(id = CohortStats.ID, data = Analysis.AnalysisData.VARIANT) -public class CohortStats extends AbstractAnalysis { - - public static final String ID = "COHORT_STATS"; - - - public CohortStats(ObjectMap executorParams, Path outDir) { - super(executorParams, outDir); - } - - @Override - public AnalysisResult execute() throws AnalysisException { - return null; - } -} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStatsAnalysis.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStatsAnalysis.java new file mode 100644 index 0000000..9c091e3 --- /dev/null +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStatsAnalysis.java @@ -0,0 +1,21 @@ +package org.opencb.oskar.analysis.variant.stats; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.oskar.analysis.OskarAnalysis; +import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.core.annotations.Analysis; + +import java.nio.file.Path; + +@Analysis(id = CohortStatsAnalysis.ID, data = Analysis.AnalysisData.VARIANT) +public class CohortStatsAnalysis extends OskarAnalysis { + public static final String ID = "COHORT_STATS"; + + public CohortStatsAnalysis(ObjectMap executorParams, Path outDir) { + super(executorParams, outDir); + } + + @Override + public void exec() throws AnalysisException { + } +} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStatsExecutor.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStatsExecutor.java new file mode 100644 index 0000000..5ceb0f7 --- /dev/null +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/CohortStatsExecutor.java @@ -0,0 +1,25 @@ +package org.opencb.oskar.analysis.variant.stats; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.oskar.analysis.OskarAnalysisExecutor; + +import java.nio.file.Path; + +public abstract class CohortStatsExecutor extends OskarAnalysisExecutor { + + public CohortStatsExecutor() { + } + + public CohortStatsExecutor(ObjectMap executorParams, Path outDir) { + this.setup(executorParams, outDir); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CohortStatsExecutor{"); + sb.append(", executorParams=").append(executorParams); + sb.append(", outDir=").append(outDir); + sb.append('}'); + return sb.toString(); + } +} diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStats.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStatsAnalysis.java similarity index 54% rename from oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStats.java rename to oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStatsAnalysis.java index e5324bc..c93af06 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStats.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStatsAnalysis.java @@ -1,28 +1,26 @@ package org.opencb.oskar.analysis.variant.stats; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysis; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.OskarAnalysis; import org.opencb.oskar.analysis.exceptions.AnalysisException; import org.opencb.oskar.core.annotations.Analysis; import java.nio.file.Path; import java.util.List; -@Analysis(id = SampleStats.ID, data = Analysis.AnalysisData.VARIANT) -public class SampleStats extends AbstractAnalysis { +@Analysis(id = SampleStatsAnalysis.ID, data = Analysis.AnalysisData.VARIANT) +public class SampleStatsAnalysis extends OskarAnalysis { public static final String ID = "SAMPLE_STATS"; private List sampleNames; - public SampleStats(List sampleNames, ObjectMap executorParams, Path outDir) { + public SampleStatsAnalysis(List sampleNames, ObjectMap executorParams, Path outDir) { super(executorParams, outDir); this.sampleNames = sampleNames; } @Override - public AnalysisResult execute() throws AnalysisException { - return null; + public void exec() throws AnalysisException { } } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractSampleStatsExecutor.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStatsExecutor.java similarity index 60% rename from oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractSampleStatsExecutor.java rename to oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStatsExecutor.java index 8b7b5d6..3b76624 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractSampleStatsExecutor.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/SampleStatsExecutor.java @@ -1,26 +1,24 @@ package org.opencb.oskar.analysis.variant.stats; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysisExecutor; -import org.opencb.oskar.analysis.AnalysisResult; -import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.OskarAnalysisExecutor; import java.nio.file.Path; import java.util.List; -public class AbstractSampleStatsExecutor extends AbstractAnalysisExecutor { +public abstract class SampleStatsExecutor extends OskarAnalysisExecutor { protected List sampleNames; protected String cohortName; - public AbstractSampleStatsExecutor() { + public SampleStatsExecutor() { } - public AbstractSampleStatsExecutor(List sampleNames, ObjectMap executorParams, Path outDir) { + public SampleStatsExecutor(List sampleNames, ObjectMap executorParams, Path outDir) { this.setup(sampleNames, executorParams, outDir); } - public AbstractSampleStatsExecutor(String cohortName, ObjectMap executorParams, Path outDir) { + public SampleStatsExecutor(String cohortName, ObjectMap executorParams, Path outDir) { this.setup(sampleNames, executorParams, outDir); } @@ -34,14 +32,9 @@ protected void setup(String cohortName, ObjectMap executorParams, Path outDir) { this.cohortName = cohortName; } - @Override - public AnalysisResult exec() throws AnalysisException { - return null; - } - @Override public String toString() { - final StringBuilder sb = new StringBuilder("AbstractSampleStatsExecutor{"); + final StringBuilder sb = new StringBuilder("SampleStatsExecutor{"); sb.append("sampleNames=").append(sampleNames); sb.append(", cohortName='").append(cohortName).append('\''); sb.append(", executorParams=").append(executorParams); diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStats.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStatsAnalysis.java similarity index 51% rename from oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStats.java rename to oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStatsAnalysis.java index ab86878..e33cb8f 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStats.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStatsAnalysis.java @@ -1,31 +1,29 @@ package org.opencb.oskar.analysis.variant.stats; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysis; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.OskarAnalysis; import org.opencb.oskar.analysis.exceptions.AnalysisException; import org.opencb.oskar.core.annotations.Analysis; import java.nio.file.Path; -@Analysis(id = VariantStats.ID, data = Analysis.AnalysisData.VARIANT) -public class VariantStats extends AbstractAnalysis { +@Analysis(id = VariantStatsAnalysis.ID, data = Analysis.AnalysisData.VARIANT) +public class VariantStatsAnalysis extends OskarAnalysis { public static final String ID = "VARIANT_STATS"; private String cohort; - public VariantStats(ObjectMap executorParams, Path outDir) { + public VariantStatsAnalysis(ObjectMap executorParams, Path outDir) { this(null, executorParams, outDir); } - public VariantStats(String cohort, ObjectMap executorParams, Path outDir) { + public VariantStatsAnalysis(String cohort, ObjectMap executorParams, Path outDir) { super(executorParams, outDir); this.cohort = cohort; } @Override - public AnalysisResult execute() throws AnalysisException { - return null; + public void exec() throws AnalysisException { } } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractVariantStatsExecutor.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStatsExecutor.java similarity index 55% rename from oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractVariantStatsExecutor.java rename to oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStatsExecutor.java index d282ef6..b6b5c68 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/AbstractVariantStatsExecutor.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/stats/VariantStatsExecutor.java @@ -2,24 +2,22 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysisExecutor; -import org.opencb.oskar.analysis.AnalysisResult; -import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.OskarAnalysisExecutor; import java.nio.file.Path; -public class AbstractVariantStatsExecutor extends AbstractAnalysisExecutor { +public abstract class VariantStatsExecutor extends OskarAnalysisExecutor { private String cohort; - public AbstractVariantStatsExecutor() { + public VariantStatsExecutor() { } - public AbstractVariantStatsExecutor(ObjectMap executorParams, Path outDir) { + public VariantStatsExecutor(ObjectMap executorParams, Path outDir) { this(null, executorParams, outDir); } - public AbstractVariantStatsExecutor(String cohort, ObjectMap executorParams, Path outDir) { + public VariantStatsExecutor(String cohort, ObjectMap executorParams, Path outDir) { setup(cohort, executorParams, outDir); } @@ -28,14 +26,9 @@ protected void setup(String cohort, ObjectMap executorParams, Path outDir) { this.cohort = StringUtils.isEmpty(cohort) ? "ALL" : cohort; } - @Override - public AnalysisResult exec() throws AnalysisException { - return null; - } - @Override public String toString() { - final StringBuilder sb = new StringBuilder("AbstractVariantStatsExecutor{"); + final StringBuilder sb = new StringBuilder("VariantStatsExecutor{"); sb.append("cohort='").append(cohort).append('\''); sb.append(", executorParams=").append(executorParams); sb.append(", outDir=").append(outDir); @@ -47,7 +40,7 @@ public String getCohort() { return cohort; } - public AbstractVariantStatsExecutor setCohort(String cohort) { + public VariantStatsExecutor setCohort(String cohort) { this.cohort = cohort; return this; } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/Tdt.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/Tdt.java index 88778bf..820f6b0 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/Tdt.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/Tdt.java @@ -1,15 +1,14 @@ package org.opencb.oskar.analysis.variant.tdt; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysis; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.OskarAnalysis; import org.opencb.oskar.analysis.exceptions.AnalysisException; import org.opencb.oskar.core.annotations.Analysis; import java.nio.file.Path; @Analysis(id = Tdt.ID, data = Analysis.AnalysisData.VARIANT) -public class Tdt extends AbstractAnalysis { +public class Tdt extends OskarAnalysis { public static final String ID = "TDT"; @@ -20,29 +19,24 @@ public Tdt(String phenotype, ObjectMap executorParams, Path outDir) { this.phenotype = phenotype; } + protected void createManhattanPlot() { + } + /** * Checks if list and list2 are not empty and no common samples exist. */ - protected void checkSamples() { - } - - protected void createManhattanPlot() { + @Override + protected void check() throws AnalysisException { + // checks } @Override - public AnalysisResult execute() throws AnalysisException { - // checks + public void exec() throws AnalysisException { + TdtExecutor tdtExecutor = getAnalysisExecutor(TdtExecutor.class, executorParams.getString("ID")); + tdtExecutor.setup(phenotype, executorParams, outDir); - AnalysisResult analysisResult; - Class executor = getAnalysisExecutorId(executorParams.getString("ID"), Tdt.ID); - try { - AbstractTdtExecutor tdtExecutor = (AbstractTdtExecutor) executor.newInstance(); - tdtExecutor.setup(phenotype, executorParams, outDir); - analysisResult = tdtExecutor.exec(); - } catch (InstantiationException | IllegalAccessException e) { - throw new AnalysisException("Error when executing TDT analysis", e); - } - - return analysisResult; + arm.startStep("tdt"); + tdtExecutor.exec(); + arm.endStep(100); } } diff --git a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/AbstractTdtExecutor.java b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/TdtExecutor.java similarity index 63% rename from oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/AbstractTdtExecutor.java rename to oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/TdtExecutor.java index a33a4f8..389b456 100644 --- a/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/AbstractTdtExecutor.java +++ b/oskar-analysis/src/main/java/org/opencb/oskar/analysis/variant/tdt/TdtExecutor.java @@ -1,24 +1,22 @@ package org.opencb.oskar.analysis.variant.tdt; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AbstractAnalysisExecutor; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.OskarAnalysisExecutor; import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.result.FileResult; import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -public abstract class AbstractTdtExecutor extends AbstractAnalysisExecutor { +public abstract class TdtExecutor extends OskarAnalysisExecutor { private String phenotype; - public AbstractTdtExecutor() { + public TdtExecutor() { } - public AbstractTdtExecutor(String phenotype, ObjectMap executorParams, Path outDir) { + public TdtExecutor(String phenotype, ObjectMap executorParams, Path outDir) { this.setup(phenotype, executorParams, outDir); } @@ -37,22 +35,18 @@ protected String getHeaderLine() { return sb.toString(); } - protected AnalysisResult createAnalysisResult() throws AnalysisException { - List resultFiles = new ArrayList<>(); + protected void registerFiles() throws AnalysisException { String outFilename = getOutDir() + "/tdt.txt"; if (new File(outFilename).exists()) { - resultFiles.add(new AnalysisResult.File(Paths.get(outFilename), AnalysisResult.FileType.TAB_SEPARATED)); + arm.addFile(Paths.get(outFilename), FileResult.FileType.TAB_SEPARATED); } - - return new AnalysisResult().setAnalysisId(Tdt.ID).setDateTime(getDateTime()).setExecutorParams(executorParams) - .setOutputFiles(resultFiles); } public String getPhenotype() { return phenotype; } - public AbstractTdtExecutor setPhenotype(String phenotype) { + public TdtExecutor setPhenotype(String phenotype) { this.phenotype = phenotype; return this; } diff --git a/oskar-analysis/src/test/java/org/opencb/oskar/analysis/OskarAnalysisTest.java b/oskar-analysis/src/test/java/org/opencb/oskar/analysis/OskarAnalysisTest.java new file mode 100644 index 0000000..a0769a4 --- /dev/null +++ b/oskar-analysis/src/test/java/org/opencb/oskar/analysis/OskarAnalysisTest.java @@ -0,0 +1,45 @@ +package org.opencb.oskar.analysis; + +import org.junit.Test; +import org.opencb.oskar.analysis.exceptions.AnalysisExecutorException; +import org.opencb.oskar.core.annotations.Analysis; +import org.opencb.oskar.core.annotations.AnalysisExecutor; + +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.opencb.oskar.core.annotations.AnalysisExecutor.Framework; +import static org.opencb.oskar.core.annotations.AnalysisExecutor.Source; + +public class OskarAnalysisTest { + + @AnalysisExecutor(id="test-executor", analysis = "test-analysis", framework = Framework.ITERATOR, source= Source.VCF_FILE) + public static class MyExecutor1 extends OskarAnalysisExecutor { @Override public void exec() { } } + + @AnalysisExecutor(id="test-executor-mr", analysis = "test-analysis", framework = Framework.MAP_REDUCE, source= Source.HBASE) + public static class MyExecutor2 extends OskarAnalysisExecutor { @Override public void exec() { } } + + @Analysis(id = "test-analysis", data = Analysis.AnalysisData.VARIANT) + public static class MyAnalysis extends OskarAnalysis { @Override public void exec() { } } + + + @Test + public void testGetExecutorClass() throws AnalysisExecutorException { + OskarAnalysis analysis = new MyAnalysis(); + + assertEquals("test-analysis", analysis.getId()); + assertEquals(MyExecutor1.class, analysis.getAnalysisExecutorClass("test-executor")); + assertEquals(MyExecutor2.class, analysis.getAnalysisExecutorClass("test-executor-mr")); + + analysis.setUp(null, Paths.get(""), Source.HBASE, Arrays.asList(Framework.MAP_REDUCE, Framework.ITERATOR)); + assertEquals("test-executor-mr", analysis.getAnalysisExecutor().getId()); + + analysis.setUp(null, Paths.get(""), Source.VCF_FILE, Collections.singleton(Framework.ITERATOR)); + assertEquals("test-executor", analysis.getAnalysisExecutor().getId()); + + analysis.setUp(null, Paths.get(""), Source.VCF_FILE, Arrays.asList(Framework.MAP_REDUCE, Framework.ITERATOR)); + assertEquals("test-executor", analysis.getAnalysisExecutor().getId()); + } +} \ No newline at end of file diff --git a/oskar-analysis/src/test/java/org/opencb/oskar/analysis/result/AnalysisResultManagerTest.java b/oskar-analysis/src/test/java/org/opencb/oskar/analysis/result/AnalysisResultManagerTest.java new file mode 100644 index 0000000..e69b967 --- /dev/null +++ b/oskar-analysis/src/test/java/org/opencb/oskar/analysis/result/AnalysisResultManagerTest.java @@ -0,0 +1,50 @@ +package org.opencb.oskar.analysis.result; + +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.oskar.analysis.exceptions.AnalysisException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Date; + +import static org.junit.Assert.*; + +public class AnalysisResultManagerTest { + + private AnalysisResultManager arm; + private static Path rootDir; + + @BeforeClass + public static void beforeClass() throws IOException { + rootDir = Paths.get("target/test-data", "junit-opencga-storage-" + + new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss.SSS").format(new Date())); + Files.createDirectories(rootDir); + } + + @Before + public void setUp() throws Exception { + arm = new AnalysisResultManager(rootDir); + arm.init("myTest", new ObjectMap()); + } + + @After + public void tearDown() throws Exception { + arm.close(); + } + + @Test + public void testReadWrite() throws AnalysisException { + + arm.startStep("step1"); + arm.endStep(100); + arm.addFile(rootDir.resolve("file1.txt").toAbsolutePath(), FileResult.FileType.TAB_SEPARATED); + + } +} \ No newline at end of file diff --git a/oskar-core/src/main/java/org/opencb/oskar/core/annotations/AnalysisExecutor.java b/oskar-core/src/main/java/org/opencb/oskar/core/annotations/AnalysisExecutor.java index 757b2b2..22d482a 100644 --- a/oskar-core/src/main/java/org/opencb/oskar/core/annotations/AnalysisExecutor.java +++ b/oskar-core/src/main/java/org/opencb/oskar/core/annotations/AnalysisExecutor.java @@ -27,6 +27,20 @@ String id(); String analysis(); + Source source(); + Framework framework(); String description() default ""; + enum Source { + VCF_FILE, + PARQUET_FILE, + MONGODB, + HBASE + } + + enum Framework { + ITERATOR, + MAP_REDUCE, + SPARK + } } diff --git a/oskar-core/src/main/java/org/opencb/oskar/core/annotations/AnalysisExecutorRequirement.java b/oskar-core/src/main/java/org/opencb/oskar/core/annotations/AnalysisExecutorRequirement.java deleted file mode 100644 index 7588574..0000000 --- a/oskar-core/src/main/java/org/opencb/oskar/core/annotations/AnalysisExecutorRequirement.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2015-2017 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.oskar.core.annotations; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) -public @interface AnalysisExecutorRequirement { - - AnalysisRequirementSource source(); - AnalysisRequirementFramework framework(); - - enum AnalysisRequirementSource { - VCF_FILE, - PARQUET_FILE, - MONGODB, - HBASE - } - - enum AnalysisRequirementFramework { - ITERATOR, - MAP_REDUCE, - SPARK - } - -} diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/CohortStatsSparkParquetAnalysisExecutor.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/CohortStatsSparkParquetAnalysisExecutor.java index 05d399d..a5beb19 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/CohortStatsSparkParquetAnalysisExecutor.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/CohortStatsSparkParquetAnalysisExecutor.java @@ -3,17 +3,17 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.lang3.time.StopWatch; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; import org.opencb.biodata.models.variant.metadata.VariantSetStats; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; -import org.opencb.oskar.analysis.variant.stats.AbstractCohortStatsExecutor; -import org.opencb.oskar.analysis.variant.stats.VariantStats; +import org.opencb.oskar.analysis.result.FileResult; +import org.opencb.oskar.analysis.variant.stats.CohortStatsAnalysis; +import org.opencb.oskar.analysis.variant.stats.CohortStatsExecutor; +import org.opencb.oskar.core.annotations.AnalysisExecutor; import org.opencb.oskar.spark.commons.OskarException; import org.opencb.oskar.spark.commons.converters.RowToAvroConverter; import org.opencb.oskar.spark.variant.Oskar; @@ -23,10 +23,13 @@ import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -public class CohortStatsSparkParquetAnalysisExecutor extends AbstractCohortStatsExecutor { +@AnalysisExecutor( + id = "spark-parquet", + analysis = CohortStatsAnalysis.ID, + source= AnalysisExecutor.Source.PARQUET_FILE, + framework = AnalysisExecutor.Framework.SPARK) +public class CohortStatsSparkParquetAnalysisExecutor extends CohortStatsExecutor { public CohortStatsSparkParquetAnalysisExecutor() { } @@ -36,9 +39,7 @@ public CohortStatsSparkParquetAnalysisExecutor(ObjectMap executorParams, Path ou } @Override - public AnalysisResult exec() throws AnalysisException { - StopWatch watch = StopWatch.createStarted(); - + public void exec() throws AnalysisException { String parquetFilename = getExecutorParams().getString("FILE"); String studyId = getExecutorParams().getString("STUDY_ID"); String master = getExecutorParams().getString("MASTER"); @@ -80,16 +81,8 @@ public AnalysisResult exec() throws AnalysisException { throw new AnalysisException("Error writing output file: " + outFilename, e); } - List resultFiles = new ArrayList<>(); if (new File(outFilename).exists()) { - resultFiles.add(new AnalysisResult.File(Paths.get(outFilename), AnalysisResult.FileType.JSON)); + arm.addFile(Paths.get(outFilename), FileResult.FileType.JSON); } - - return new AnalysisResult() - .setAnalysisId(VariantStats.ID) - .setDateTime(getDateTime()) - .setExecutorParams(executorParams) - .setOutputFiles(resultFiles) - .setExecutionTime(watch.getTime()); } } diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/GwasSparkParquetAnalysisExecutor.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/GwasSparkParquetAnalysisExecutor.java index e6d5582..db709a1 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/GwasSparkParquetAnalysisExecutor.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/GwasSparkParquetAnalysisExecutor.java @@ -1,17 +1,17 @@ package org.opencb.oskar.spark.variant.analysis.executors; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.time.StopWatch; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.CollectionUtils; -import org.opencb.oskar.analysis.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; import org.opencb.oskar.analysis.stats.FisherExactTest; -import org.opencb.oskar.analysis.variant.gwas.AbstractGwasExecutor; +import org.opencb.oskar.analysis.variant.gwas.Gwas; +import org.opencb.oskar.analysis.variant.gwas.GwasExecutor; import org.opencb.oskar.analysis.variant.gwas.GwasConfiguration; +import org.opencb.oskar.core.annotations.AnalysisExecutor; import org.opencb.oskar.spark.commons.OskarException; import org.opencb.oskar.spark.variant.Oskar; import org.opencb.oskar.spark.variant.analysis.transformers.ChiSquareTransformer; @@ -28,7 +28,12 @@ import static org.opencb.oskar.analysis.variant.gwas.GwasConfiguration.Method.CHI_SQUARE_TEST; import static org.opencb.oskar.analysis.variant.gwas.GwasConfiguration.Method.FISHER_TEST; -public class GwasSparkParquetAnalysisExecutor extends AbstractGwasExecutor { +@AnalysisExecutor( + id = "spark-parquet", + analysis = Gwas.ID, + source= AnalysisExecutor.Source.PARQUET_FILE, + framework = AnalysisExecutor.Framework.SPARK) +public class GwasSparkParquetAnalysisExecutor extends GwasExecutor { public GwasSparkParquetAnalysisExecutor() { } @@ -43,8 +48,7 @@ public GwasSparkParquetAnalysisExecutor(String phenotype, ObjectMap executorPara } @Override - public AnalysisResult exec() throws AnalysisException { - StopWatch watch = StopWatch.createStarted(); + public void exec() throws AnalysisException { String parquetFilename = getExecutorParams().getString("FILE"); String studyId = getExecutorParams().getString("STUDY_ID"); @@ -80,7 +84,7 @@ public AnalysisResult exec() throws AnalysisException { + CHI_SQUARE_TEST + " and " + FISHER_TEST); } - return createAnalysisResult().setExecutionTime(watch.getTime()); + registerFiles(); } private void fisher(Dataset inputDastaset, String studyId, int mode) throws AnalysisException { diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/SampleStatsSparkParquetAnalysisExecutor.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/SampleStatsSparkParquetAnalysisExecutor.java index fc47875..564427b 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/SampleStatsSparkParquetAnalysisExecutor.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/SampleStatsSparkParquetAnalysisExecutor.java @@ -3,16 +3,16 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.lang3.time.StopWatch; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.opencb.biodata.models.variant.stats.VariantSampleStats; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; -import org.opencb.oskar.analysis.variant.stats.AbstractSampleStatsExecutor; -import org.opencb.oskar.analysis.variant.stats.VariantStats; +import org.opencb.oskar.analysis.result.FileResult; +import org.opencb.oskar.analysis.variant.stats.SampleStatsAnalysis; +import org.opencb.oskar.analysis.variant.stats.SampleStatsExecutor; +import org.opencb.oskar.core.annotations.AnalysisExecutor; import org.opencb.oskar.spark.commons.OskarException; import org.opencb.oskar.spark.variant.Oskar; import org.opencb.oskar.spark.variant.analysis.transformers.VariantSampleStatsTransformer; @@ -21,11 +21,15 @@ import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; import java.util.Map; -public class SampleStatsSparkParquetAnalysisExecutor extends AbstractSampleStatsExecutor { +@AnalysisExecutor( + id = "spark-parquet", + analysis = SampleStatsAnalysis.ID, + source= AnalysisExecutor.Source.PARQUET_FILE, + framework = AnalysisExecutor.Framework.SPARK) +public class SampleStatsSparkParquetAnalysisExecutor extends SampleStatsExecutor { public SampleStatsSparkParquetAnalysisExecutor() { } @@ -35,9 +39,7 @@ public SampleStatsSparkParquetAnalysisExecutor(List sampleNames, ObjectM } @Override - public AnalysisResult exec() throws AnalysisException { - StopWatch watch = StopWatch.createStarted(); - + public void exec() throws AnalysisException { String parquetFilename = getExecutorParams().getString("FILE"); String studyId = getExecutorParams().getString("STUDY_ID"); String master = getExecutorParams().getString("MASTER"); @@ -80,16 +82,8 @@ public AnalysisResult exec() throws AnalysisException { throw new AnalysisException("Error writing output file: " + outFilename, e); } - List resultFiles = new ArrayList<>(); if (new File(outFilename).exists()) { - resultFiles.add(new AnalysisResult.File(Paths.get(outFilename), AnalysisResult.FileType.JSON)); + arm.addFile(Paths.get(outFilename), FileResult.FileType.JSON); } - - return new AnalysisResult() - .setAnalysisId(VariantStats.ID) - .setDateTime(getDateTime()) - .setExecutorParams(executorParams) - .setOutputFiles(resultFiles) - .setExecutionTime(watch.getTime()); } } diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/TdtSparkParquetAnalysisExecutor.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/TdtSparkParquetAnalysisExecutor.java index 4852364..fc30dfe 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/TdtSparkParquetAnalysisExecutor.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/TdtSparkParquetAnalysisExecutor.java @@ -1,13 +1,13 @@ package org.opencb.oskar.spark.variant.analysis.executors; -import org.apache.commons.lang3.time.StopWatch; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; -import org.opencb.oskar.analysis.variant.tdt.AbstractTdtExecutor; +import org.opencb.oskar.analysis.variant.tdt.Tdt; +import org.opencb.oskar.analysis.variant.tdt.TdtExecutor; +import org.opencb.oskar.core.annotations.AnalysisExecutor; import org.opencb.oskar.spark.commons.OskarException; import org.opencb.oskar.spark.variant.Oskar; import org.opencb.oskar.spark.variant.analysis.transformers.TdtTransformer; @@ -20,7 +20,12 @@ import static org.apache.spark.sql.functions.col; import static org.apache.spark.sql.functions.explode; -public class TdtSparkParquetAnalysisExecutor extends AbstractTdtExecutor { +@AnalysisExecutor( + id="spark-parquet", + analysis = Tdt.ID, + source = AnalysisExecutor.Source.PARQUET_FILE, + framework = AnalysisExecutor.Framework.SPARK) +public class TdtSparkParquetAnalysisExecutor extends TdtExecutor { public TdtSparkParquetAnalysisExecutor() { } @@ -30,9 +35,7 @@ public TdtSparkParquetAnalysisExecutor(String phenotype, ObjectMap executorParam } @Override - public AnalysisResult exec() throws AnalysisException { - StopWatch watch = StopWatch.createStarted(); - + public void exec() throws AnalysisException { String parquetFilename = getExecutorParams().getString("FILE"); String studyId = getExecutorParams().getString("STUDY_ID"); String master = getExecutorParams().getString("MASTER"); @@ -83,6 +86,6 @@ public AnalysisResult exec() throws AnalysisException { throw new AnalysisException("Error saving TDT results", e); } - return createAnalysisResult().setExecutionTime(watch.getTime()); + registerFiles(); } } diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/VariantStatsSparkParquetAnalysisExecutor.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/VariantStatsSparkParquetAnalysisExecutor.java index 82622f9..73a9d3c 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/VariantStatsSparkParquetAnalysisExecutor.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/executors/VariantStatsSparkParquetAnalysisExecutor.java @@ -1,15 +1,15 @@ package org.opencb.oskar.spark.variant.analysis.executors; -import org.apache.commons.lang3.time.StopWatch; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.StructField; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; -import org.opencb.oskar.analysis.variant.stats.AbstractVariantStatsExecutor; -import org.opencb.oskar.analysis.variant.stats.SampleStats; +import org.opencb.oskar.analysis.result.FileResult; +import org.opencb.oskar.analysis.variant.stats.VariantStatsAnalysis; +import org.opencb.oskar.analysis.variant.stats.VariantStatsExecutor; +import org.opencb.oskar.core.annotations.AnalysisExecutor; import org.opencb.oskar.spark.commons.OskarException; import org.opencb.oskar.spark.variant.Oskar; import org.opencb.oskar.spark.variant.analysis.transformers.VariantStatsTransformer; @@ -19,13 +19,16 @@ import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; import static org.apache.spark.sql.functions.col; import static org.apache.spark.sql.functions.explode; -public class VariantStatsSparkParquetAnalysisExecutor extends AbstractVariantStatsExecutor { +@AnalysisExecutor( + id = "spark-parquet", + analysis = VariantStatsAnalysis.ID, + source= AnalysisExecutor.Source.PARQUET_FILE, + framework = AnalysisExecutor.Framework.SPARK) +public class VariantStatsSparkParquetAnalysisExecutor extends VariantStatsExecutor { public VariantStatsSparkParquetAnalysisExecutor() { } @@ -35,9 +38,7 @@ public VariantStatsSparkParquetAnalysisExecutor(String cohort, ObjectMap executo } @Override - public AnalysisResult exec() throws AnalysisException { - StopWatch watch = StopWatch.createStarted(); - + public void exec() throws AnalysisException { String parquetFilename = getExecutorParams().getString("FILE"); String studyId = getExecutorParams().getString("STUDY_ID"); String master = getExecutorParams().getString("MASTER"); @@ -89,16 +90,8 @@ public AnalysisResult exec() throws AnalysisException { pw.close(); - List resultFiles = new ArrayList<>(); if (new File(outFilename).exists()) { - resultFiles.add(new AnalysisResult.File(Paths.get(outFilename), AnalysisResult.FileType.TAB_SEPARATED)); + arm.addFile(Paths.get(outFilename), FileResult.FileType.TAB_SEPARATED); } - - return new AnalysisResult() - .setAnalysisId(SampleStats.ID) - .setDateTime(getDateTime()) - .setExecutorParams(executorParams) - .setOutputFiles(resultFiles) - .setExecutionTime(watch.getTime()); } } diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/LinearRegressionOskarAnalysis.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/LinearRegressionOskarAnalysis.java index cf21aa3..f86d90a 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/LinearRegressionOskarAnalysis.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/LinearRegressionOskarAnalysis.java @@ -1,7 +1,6 @@ package org.opencb.oskar.spark.variant.analysis.transformers; import org.opencb.oskar.analysis.variant.VariantOskarAnalysis; -import org.opencb.oskar.core.config.OskarConfiguration; /** * Created by jtarraga on 30/05/17. @@ -16,7 +15,7 @@ public class LinearRegressionOskarAnalysis extends VariantOskarAnalysis { private double elasticNet = 0.8; // elastic net mixing parameter @Override - public void execute() { + public void exec() { // LinearRegression lr = new LinearRegression() // .setMaxIter(numIterations) // .setRegParam(regularization) @@ -52,14 +51,12 @@ public void execute() { // System.out.println("r2: " + trainingSummary.r2()); } - public LinearRegressionOskarAnalysis(String studyId, String depVarName, String indepVarName, OskarConfiguration configuration) { - this(studyId, depVarName, indepVarName, 10, 0.3, 0.8, configuration); + public LinearRegressionOskarAnalysis(String depVarName, String indepVarName) { + this(depVarName, indepVarName, 10, 0.3, 0.8); } - public LinearRegressionOskarAnalysis(String studyId, String depVarName, String indepVarName, - int numIterations, double regularization, double elasticNet, - OskarConfiguration configuration) { - super(studyId, configuration); + public LinearRegressionOskarAnalysis(String depVarName, String indepVarName, + int numIterations, double regularization, double elasticNet) { this.depVarName = depVarName; this.indepVarName = indepVarName; this.numIterations = numIterations; diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/LogisticRegressionOskarAnalysis.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/LogisticRegressionOskarAnalysis.java index 4bdfff7..5e51259 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/LogisticRegressionOskarAnalysis.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/LogisticRegressionOskarAnalysis.java @@ -1,7 +1,6 @@ package org.opencb.oskar.spark.variant.analysis.transformers; import org.opencb.oskar.analysis.variant.VariantOskarAnalysis; -import org.opencb.oskar.core.config.OskarConfiguration; /** * Created by jtarraga on 30/05/17. @@ -16,7 +15,7 @@ public class LogisticRegressionOskarAnalysis extends VariantOskarAnalysis { private double elasticNet = 0.8; // elastic net mixing parameter @Override - public void execute() { + public void exec() { // LogisticRegression lr = new LogisticRegression() // .setMaxIter(numIterations) // .setRegParam(regularization) @@ -64,13 +63,12 @@ public void execute() { // lrModel.setThreshold(bestThreshold); } - public LogisticRegressionOskarAnalysis(String studyId, String depVarName, String indepVarName, OskarConfiguration configuration) { - this(studyId, depVarName, indepVarName, 10, 0.3, 0.8, configuration); + public LogisticRegressionOskarAnalysis(String depVarName, String indepVarName) { + this(depVarName, indepVarName, 10, 0.3, 0.8); } - public LogisticRegressionOskarAnalysis(String studyId, String depVarName, String indepVarName, - int numIterations, double regularization, double elasticNet, OskarConfiguration configuration) { - super(studyId, configuration); + public LogisticRegressionOskarAnalysis(String depVarName, String indepVarName, + int numIterations, double regularization, double elasticNet) { this.depVarName = depVarName; this.indepVarName = indepVarName; this.numIterations = numIterations; diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/VariantAnalysisUtils.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/VariantAnalysisUtils.java index 77f796f..985b6c1 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/VariantAnalysisUtils.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/transformers/VariantAnalysisUtils.java @@ -7,14 +7,13 @@ import org.opencb.biodata.tools.variant.converters.VCFExporter; import org.opencb.biodata.tools.variant.metadata.VariantMetadataManager; import org.opencb.commons.datastore.core.Query; -import org.opencb.oskar.analysis.AbstractAnalysis; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.OskarAnalysis; import java.io.IOException; import java.nio.file.Paths; import java.util.List; -public class VariantAnalysisUtils extends AbstractAnalysis { +public class VariantAnalysisUtils extends OskarAnalysis { /** * Export Avro variant into a VCF file. * @@ -68,7 +67,6 @@ public static void exportPedigree(String metaFilename, String studyId, String pe } @Override - public AnalysisResult execute() { - return null; + public void exec() { } } diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/PlinkWrapper.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/PlinkWrapper.java index ab2a9a8..42d253f 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/PlinkWrapper.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/PlinkWrapper.java @@ -25,22 +25,23 @@ public class PlinkWrapper extends VariantOskarAnalysisWrapper { private String metaFilename; private Query query; private Map plinkParams; + private final OskarConfiguration configuration; private Logger logger; - public PlinkWrapper(String studyId, String inFilename, String metaFilename, + public PlinkWrapper(String inFilename, String metaFilename, Query query, Map plinkParams, OskarConfiguration configuration) { - super(studyId, configuration); this.inFilename = inFilename; this.metaFilename = metaFilename; this.query = query; this.plinkParams = plinkParams; + this.configuration = configuration; this.logger = LoggerFactory.getLogger(PlinkWrapper.class); } - public void execute() throws AnalysisExecutorException { + public void exec() throws AnalysisExecutorException { // Sanity check Path binPath; try { diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/RvTestsWrapper.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/RvTestsWrapper.java index a2eb936..b3989a5 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/RvTestsWrapper.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/RvTestsWrapper.java @@ -21,26 +21,29 @@ public class RvTestsWrapper extends VariantOskarAnalysisWrapper { public static final String ANALYSIS_NAME = "rvtests"; + private final String studyId; private String inFilename; private String metaFilename; private Query query; private Map rvtestsParams; + private final OskarConfiguration configuration; private Logger logger; public RvTestsWrapper(String studyId, String inFilename, String metaFilename, Query query, Map rvTestsParams, OskarConfiguration configuration) { - super(studyId, configuration); + this.studyId = studyId; this.inFilename = inFilename; this.metaFilename = metaFilename; this.query = query; this.rvtestsParams = rvTestsParams; + this.configuration = configuration; this.logger = LoggerFactory.getLogger(RvTestsWrapper.class); } @Override - public void execute() throws AnalysisExecutorException { + public void exec() throws AnalysisExecutorException { // Sanity check Path binPath; try { @@ -77,12 +80,12 @@ public void execute() throws AnalysisExecutorException { // Compress vcf to bgz sb.setLength(0); sb.append(BGZIP_BIN).append(" ").append(vcfFilename); - Executor.execute(sb.toString(), outDir, true); + Executor.exec(sb.toString(), outDir, true); // ...create tabix index sb.setLength(0); sb.append(TABIX_BIN).append(" -p vcf ").append(vcfFilename).append(".gz"); - Executor.execute(sb.toString(), outDir, true); + Executor.exec(sb.toString(), outDir, true); */ // ...and finally, run rvtests sb.setLength(0); diff --git a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/VariantOskarAnalysisWrapper.java b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/VariantOskarAnalysisWrapper.java index 73c58ee..6b5ca48 100644 --- a/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/VariantOskarAnalysisWrapper.java +++ b/oskar-spark/src/main/java/org/opencb/oskar/spark/variant/analysis/wrappers/VariantOskarAnalysisWrapper.java @@ -1,11 +1,7 @@ package org.opencb.oskar.spark.variant.analysis.wrappers; import org.opencb.oskar.analysis.variant.VariantOskarAnalysis; -import org.opencb.oskar.core.config.OskarConfiguration; public abstract class VariantOskarAnalysisWrapper extends VariantOskarAnalysis { - protected VariantOskarAnalysisWrapper(String studyId, OskarConfiguration configuration) { - super(studyId, configuration); - } } diff --git a/oskar-spark/src/main/python/notebooks/analysis.ipynb b/oskar-spark/src/main/python/notebooks/analysis.ipynb index f985c9b..59f2983 100644 --- a/oskar-spark/src/main/python/notebooks/analysis.ipynb +++ b/oskar-spark/src/main/python/notebooks/analysis.ipynb @@ -699,7 +699,7 @@ "metadata": {}, "source": [ "## Identity by state\n", - "An interesting test we may want to execute is IBS, wich could give us important information about genotype similarity.\n", + "An interesting test we may want to exec is IBS, wich could give us important information about genotype similarity.\n", "
\n", "Usage:\n", "```python\n", @@ -788,7 +788,7 @@ "metadata": {}, "source": [ "## Identity by descent\n", - "Some other interesting test we may want to execute is IBD, wich could give us important information about common ancestors.\n", + "Some other interesting test we may want to exec is IBD, wich could give us important information about common ancestors.\n", "
\n", "Usage:\n", "```python\n", diff --git a/oskar-spark/src/main/python/notebooks/stats.ipynb b/oskar-spark/src/main/python/notebooks/stats.ipynb index c0a71aa..c93bc4b 100644 --- a/oskar-spark/src/main/python/notebooks/stats.ipynb +++ b/oskar-spark/src/main/python/notebooks/stats.ipynb @@ -332,7 +332,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "There are other options we could execute:\n", + "There are other options we could exec:\n", " - Getting stats from specific individuals. In that case we need to provide a list of samples.\n", " - Specify the name we want to give to the stats cohort, which by default is \"ALL\"." ] diff --git a/oskar-spark/src/main/python/notebooks/variant_filtering.ipynb b/oskar-spark/src/main/python/notebooks/variant_filtering.ipynb index d2b0aa5..45bf87f 100644 --- a/oskar-spark/src/main/python/notebooks/variant_filtering.ipynb +++ b/oskar-spark/src/main/python/notebooks/variant_filtering.ipynb @@ -128,7 +128,7 @@ "metadata": {}, "source": [ "## Region\n", - "In the first place of all we will execute a simple filtering based on a restricted zone for those people who are not familiarizes with spark syntax. In this example we chose the 22th chromosome and the nucleotides between 17.000.000 and 17.500.000 position. There is two main ways of querying our filter:" + "In the first place of all we will exec a simple filtering based on a restricted zone for those people who are not familiarizes with spark syntax. In this example we chose the 22th chromosome and the nucleotides between 17.000.000 and 17.500.000 position. There is two main ways of querying our filter:" ] }, { @@ -216,7 +216,7 @@ "metadata": {}, "source": [ "## Gene\n", - "We may want to execute a filtering which ties up the variants attached to a particular gene. NBEAP3 was chosen as the target. Here we start preciating the functionality of PyOskar API: [ **genes** ] will automatically locate and acces the field with that same name. We just need to specify the DataFrame column where we store the gene info: _annotation_. Now it's only left to define the filter.\n", + "We may want to exec a filtering which ties up the variants attached to a particular gene. NBEAP3 was chosen as the target. Here we start preciating the functionality of PyOskar API: [ **genes** ] will automatically locate and acces the field with that same name. We just need to specify the DataFrame column where we store the gene info: _annotation_. Now it's only left to define the filter.\n", "
\n", "Usage:\n", "```python\n", diff --git a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/CohortStatsSparkParquetAnalysisExecutorTest.java b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/CohortStatsSparkParquetAnalysisExecutorTest.java index 0c5d7ba..00c9356 100644 --- a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/CohortStatsSparkParquetAnalysisExecutorTest.java +++ b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/CohortStatsSparkParquetAnalysisExecutorTest.java @@ -3,14 +3,17 @@ import org.junit.Before; import org.junit.Test; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.result.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.result.AnalysisResultManager; import org.opencb.oskar.spark.OskarSparkTestUtils; import org.opencb.oskar.spark.variant.analysis.executors.CohortStatsSparkParquetAnalysisExecutor; import java.io.File; import java.io.IOException; +import static org.opencb.oskar.spark.OskarSparkTestUtils.getRootDir; + public class CohortStatsSparkParquetAnalysisExecutorTest { private String cohort; private ObjectMap executorParams; @@ -34,7 +37,10 @@ public void init() throws IOException { public void cohortStats() throws IOException, AnalysisException { CohortStatsSparkParquetAnalysisExecutor executor = new CohortStatsSparkParquetAnalysisExecutor(executorParams, oskarSparkTestUtils.getRootDir().toAbsolutePath()); - AnalysisResult analysisResult = executor.exec(); + AnalysisResultManager amr = new AnalysisResultManager(getRootDir()).init("", new ObjectMap()); + executor.init(amr); + executor.exec(); + AnalysisResult analysisResult = amr.close(); System.out.println("Cohort stats done! Results at " + oskarSparkTestUtils.getRootDir().toAbsolutePath()); System.out.println(analysisResult.toString()); diff --git a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/GwasSparkParquetAnalysisExecutorTest.java b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/GwasSparkParquetAnalysisExecutorTest.java index 7dd7da4..72d1b9a 100644 --- a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/GwasSparkParquetAnalysisExecutorTest.java +++ b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/GwasSparkParquetAnalysisExecutorTest.java @@ -3,8 +3,9 @@ import org.junit.Before; import org.junit.Test; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.result.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.result.AnalysisResultManager; import org.opencb.oskar.analysis.variant.gwas.GwasConfiguration; import org.opencb.oskar.spark.OskarSparkTestUtils; import org.opencb.oskar.spark.variant.analysis.executors.GwasSparkParquetAnalysisExecutor; @@ -14,6 +15,8 @@ import java.util.Arrays; import java.util.List; +import static org.opencb.oskar.spark.OskarSparkTestUtils.getRootDir; + public class GwasSparkParquetAnalysisExecutorTest { private List sampleList1; @@ -49,13 +52,21 @@ public void init() throws IOException { private AnalysisResult executeGwasByLists() throws IOException, AnalysisException { GwasSparkParquetAnalysisExecutor executor = new GwasSparkParquetAnalysisExecutor(sampleList1, sampleList2, executorParams, oskarSparkTestUtils.getRootDir().toAbsolutePath(), configuration); - return executor.exec(); + AnalysisResultManager amr = new AnalysisResultManager(getRootDir()).init("", new ObjectMap()); + executor.init(amr); + executor.exec(); + + return amr.close(); } private AnalysisResult executeGwasByPhenotype() throws IOException, AnalysisException { GwasSparkParquetAnalysisExecutor executor = new GwasSparkParquetAnalysisExecutor(phenotype, executorParams, oskarSparkTestUtils.getRootDir().toAbsolutePath(), configuration); - return executor.exec(); + AnalysisResultManager amr = new AnalysisResultManager(getRootDir()).init("", new ObjectMap()); + executor.init(amr); + executor.exec(); + + return amr.close(); } @Test diff --git a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/SampleStatsSparkParquetAnalysisExecutorTest.java b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/SampleStatsSparkParquetAnalysisExecutorTest.java index 6e80d9a..ff1c030 100644 --- a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/SampleStatsSparkParquetAnalysisExecutorTest.java +++ b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/SampleStatsSparkParquetAnalysisExecutorTest.java @@ -3,8 +3,9 @@ import org.junit.Before; import org.junit.Test; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.result.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.result.AnalysisResultManager; import org.opencb.oskar.spark.OskarSparkTestUtils; import org.opencb.oskar.spark.variant.analysis.executors.SampleStatsSparkParquetAnalysisExecutor; @@ -45,7 +46,10 @@ public void init() throws IOException { public void sampleStats() throws IOException, AnalysisException { SampleStatsSparkParquetAnalysisExecutor executor = new SampleStatsSparkParquetAnalysisExecutor(sampleNames, executorParams, oskarSparkTestUtils.getRootDir().toAbsolutePath()); - AnalysisResult analysisResult = executor.exec(); + AnalysisResultManager amr = new AnalysisResultManager(getRootDir()).init("", new ObjectMap()); + executor.init(amr); + executor.exec(); + AnalysisResult analysisResult = amr.close(); System.out.println("Sample stats done! Results at " + oskarSparkTestUtils.getRootDir().toAbsolutePath()); System.out.println(analysisResult.toString()); diff --git a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/TdtSparkParquetAnalysisExecutorTest.java b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/TdtSparkParquetAnalysisExecutorTest.java index 2e64b65..d0880d9 100644 --- a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/TdtSparkParquetAnalysisExecutorTest.java +++ b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/TdtSparkParquetAnalysisExecutorTest.java @@ -3,14 +3,17 @@ import org.junit.Before; import org.junit.Test; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.result.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.result.AnalysisResultManager; import org.opencb.oskar.spark.OskarSparkTestUtils; import org.opencb.oskar.spark.variant.analysis.executors.TdtSparkParquetAnalysisExecutor; import java.io.File; import java.io.IOException; +import static org.opencb.oskar.spark.OskarSparkTestUtils.getRootDir; + public class TdtSparkParquetAnalysisExecutorTest { private String phenotype; @@ -38,7 +41,10 @@ public void init() throws IOException { public void tdt() throws IOException, AnalysisException { TdtSparkParquetAnalysisExecutor executor = new TdtSparkParquetAnalysisExecutor(phenotype, executorParams, oskarSparkTestUtils.getRootDir().toAbsolutePath()); - AnalysisResult analysisResult = executor.exec(); + AnalysisResultManager amr = new AnalysisResultManager(getRootDir()).init("", new ObjectMap()); + executor.init(amr); + executor.exec(); + AnalysisResult analysisResult = amr.close(); System.out.println("TDT done! Results at " + oskarSparkTestUtils.getRootDir().toAbsolutePath()); System.out.println(analysisResult.toString()); diff --git a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/VariantStatsSparkParquetAnalysisExecutorTest.java b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/VariantStatsSparkParquetAnalysisExecutorTest.java index 8d2794e..52b5f7e 100644 --- a/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/VariantStatsSparkParquetAnalysisExecutorTest.java +++ b/oskar-spark/src/test/java/org/opencb/oskar/spark/variant/analysis/VariantStatsSparkParquetAnalysisExecutorTest.java @@ -3,14 +3,17 @@ import org.junit.Before; import org.junit.Test; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.oskar.analysis.AnalysisResult; +import org.opencb.oskar.analysis.result.AnalysisResult; import org.opencb.oskar.analysis.exceptions.AnalysisException; +import org.opencb.oskar.analysis.result.AnalysisResultManager; import org.opencb.oskar.spark.OskarSparkTestUtils; import org.opencb.oskar.spark.variant.analysis.executors.VariantStatsSparkParquetAnalysisExecutor; import java.io.File; import java.io.IOException; +import static org.opencb.oskar.spark.OskarSparkTestUtils.getRootDir; + public class VariantStatsSparkParquetAnalysisExecutorTest { private String cohort; private ObjectMap executorParams; @@ -34,7 +37,10 @@ public void init() throws IOException { public void variantStats() throws IOException, AnalysisException { VariantStatsSparkParquetAnalysisExecutor executor = new VariantStatsSparkParquetAnalysisExecutor(null, executorParams, oskarSparkTestUtils.getRootDir().toAbsolutePath()); - AnalysisResult analysisResult = executor.exec(); + AnalysisResultManager amr = new AnalysisResultManager(getRootDir()).init("", new ObjectMap()); + executor.init(amr); + executor.exec(); + AnalysisResult analysisResult = amr.close(); System.out.println("Variant stats done! Results at " + oskarSparkTestUtils.getRootDir().toAbsolutePath()); System.out.println(analysisResult.toString());