From c9eb8893b9b62c1de652c3a5529bdc4f115d7503 Mon Sep 17 00:00:00 2001 From: jose miguel mut Date: Tue, 9 May 2017 11:27:15 +0100 Subject: [PATCH 1/5] JobParametersIncrementer: add test to show bug (reuse params) Strictly, this is likely not to be a bug, but an unexpected feature. When a job has completed successfully, its parameters are copied into a new execution of the same job instance. The intuitive behaviour for us, is that the copying only happens from failed jobs --- ...elineJobLauncherCommandLineRunnerTest.java | 67 +++++++++++++++++-- .../ebi/eva/utils/EvaCommandLineBuilder.java | 4 ++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java index 04c76d748..a387e9a86 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java @@ -31,6 +31,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.eva.pipeline.parameters.JobParametersNames; import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; import uk.ac.ebi.eva.test.utils.GenotypedVcfJobTestUtils; @@ -43,8 +44,11 @@ import static org.hamcrest.core.StringContains.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.GENOTYPED_VCF_JOB; +import static uk.ac.ebi.eva.pipeline.runner.EvaPipelineJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS; import static uk.ac.ebi.eva.pipeline.runner.EvaPipelineJobLauncherCommandLineRunner.SPRING_BATCH_JOB_NAME_PROPERTY; import static uk.ac.ebi.eva.utils.FileUtils.getResource; @@ -139,8 +143,7 @@ public void genotypedVcfJobTest() throws JobExecutionException, IOException, URI .build() ); - assertEquals(EvaPipelineJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, - evaPipelineJobLauncherCommandLineRunner.getExitCode()); + assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); assertFalse(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1).isEmpty()); JobExecution jobExecution = jobExplorer.getJobExecution(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1) @@ -167,6 +170,63 @@ public void genotypedVcfJobTest() throws JobExecutionException, IOException, URI GenotypedVcfJobTestUtils.checkSkippedOneMalformedLine(jobExecution); } + @Test + public void doNotReuseParametersFromPreviousCompletedJobs() throws JobExecutionException, IOException, URISyntaxException, + ClassNotFoundException, StorageManagerException, InstantiationException, IllegalAccessException { + String databaseName = mongoRule.getRandomTemporaryDatabaseName(); + File inputFile = GenotypedVcfJobTestUtils.getInputFile(); + + evaPipelineJobLauncherCommandLineRunner.setJobNames(GENOTYPED_VCF_JOB); + EvaCommandLineBuilder evaCommandLineBuilderWithoutChunksize = new EvaCommandLineBuilder() + .inputVcf(inputFile.getAbsolutePath()) + .inputVcfAggregation("NONE") + .inputStudyName("small vcf") + .inputStudyId(GenotypedVcfJobTestUtils.INPUT_STUDY_ID) + .inputStudyType("COLLECTION") + .databaseName(databaseName) + .configDbReadPreference("secondary") + .dbCollectionsVariantsName("variants") + .dbCollectionsFilesName("files") + .dbCollectionsFeaturesName("features") + .annotationSkip(true) + .statisticsSkip(true); + + String[] commandLineWithoutChunksize = evaCommandLineBuilderWithoutChunksize + .inputVcfId("second_" + GenotypedVcfJobTestUtils.INPUT_VCF_ID) + .build(); + + String[] completeCommandLine = evaCommandLineBuilderWithoutChunksize + .inputVcfId("first_" + GenotypedVcfJobTestUtils.INPUT_VCF_ID) + .chunksize("100") + .build(); + + evaPipelineJobLauncherCommandLineRunner.run(completeCommandLine); + + assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); + + assertFalse(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1).isEmpty()); + JobExecution jobExecution = jobExplorer.getJobExecution(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1) + .get(0).getInstanceId()); + + assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + + assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + + + // check second run doesn't include the chunksize parameter + evaPipelineJobLauncherCommandLineRunner.run(commandLineWithoutChunksize); + assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); + + assertFalse(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1).isEmpty()); + JobExecution secondJobExecution = jobExplorer.getJobExecution(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1) + .get(0).getInstanceId()); + + assertNotNull(secondJobExecution.getJobParameters().getString(JobParametersNames.DB_NAME)); + assertNull(secondJobExecution.getJobParameters().getString(JobParametersNames.CONFIG_CHUNK_SIZE)); + } + @Test public void genotypedVcfJobTestWithParametersFileAndCommandLineParameters() throws JobExecutionException, IOException, URISyntaxException, ClassNotFoundException, StorageManagerException, InstantiationException, @@ -201,8 +261,7 @@ public void genotypedVcfJobTestWithParametersFileAndCommandLineParameters() thro .build() ); - assertEquals(EvaPipelineJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, - evaPipelineJobLauncherCommandLineRunner.getExitCode()); + assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); assertFalse(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1).isEmpty()); JobExecution jobExecution = jobExplorer.getJobExecution(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1) diff --git a/src/test/java/uk/ac/ebi/eva/utils/EvaCommandLineBuilder.java b/src/test/java/uk/ac/ebi/eva/utils/EvaCommandLineBuilder.java index 5ff39e4b4..b3f97b6de 100644 --- a/src/test/java/uk/ac/ebi/eva/utils/EvaCommandLineBuilder.java +++ b/src/test/java/uk/ac/ebi/eva/utils/EvaCommandLineBuilder.java @@ -171,4 +171,8 @@ public EvaCommandLineBuilder dbCollectionsStatisticsName(String name) { public EvaCommandLineBuilder dbCollectionsAnnotationMetadataName(String name) { return addString(JobParametersNames.DB_COLLECTIONS_ANNOTATION_METADATA_NAME, name); } + + public EvaCommandLineBuilder chunksize(String chunksize) { + return addString(JobParametersNames.CONFIG_CHUNK_SIZE, chunksize); + } } From b63bb85201b2877d67c09f50a440f95f851b259a Mon Sep 17 00:00:00 2001 From: jose miguel mut Date: Tue, 9 May 2017 11:04:54 +0100 Subject: [PATCH 2/5] add new JobParametersIncrementer to not reused params from completed jobs --- .../eva/pipeline/jobs/AggregatedVcfJob.java | 3 +- .../ebi/eva/pipeline/jobs/AnnotationJob.java | 4 +- .../jobs/DatabaseInitializationJob.java | 3 +- .../ebi/eva/pipeline/jobs/DropStudyJob.java | 3 +- .../eva/pipeline/jobs/GenotypedVcfJob.java | 3 +- .../jobs/PopulationStatisticsJob.java | 3 +- .../parameters/NewJobIncrementer.java | 50 +++++++++++++++++++ 7 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 src/main/java/uk/ac/ebi/eva/pipeline/parameters/NewJobIncrementer.java diff --git a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/AggregatedVcfJob.java b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/AggregatedVcfJob.java index 96447f6b9..bdf0a0f1f 100644 --- a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/AggregatedVcfJob.java +++ b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/AggregatedVcfJob.java @@ -35,6 +35,7 @@ import uk.ac.ebi.eva.pipeline.jobs.flows.AnnotationFlowOptional; import uk.ac.ebi.eva.pipeline.jobs.steps.LoadFileStep; import uk.ac.ebi.eva.pipeline.jobs.steps.VariantLoaderStep; +import uk.ac.ebi.eva.pipeline.parameters.NewJobIncrementer; import uk.ac.ebi.eva.pipeline.parameters.validation.job.AggregatedVcfJobParametersValidator; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.AGGREGATED_VCF_JOB; @@ -76,7 +77,7 @@ public Job aggregatedVcfJob(JobBuilderFactory jobBuilderFactory) { JobBuilder jobBuilder = jobBuilderFactory .get(AGGREGATED_VCF_JOB) - .incrementer(new RunIdIncrementer()) + .incrementer(new NewJobIncrementer()) .validator(new AggregatedVcfJobParametersValidator()); FlowJobBuilder builder = jobBuilder .flow(variantLoaderStep) diff --git a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJob.java b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJob.java index e13c110e0..0feafdcdf 100644 --- a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJob.java +++ b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJob.java @@ -23,7 +23,6 @@ import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.flow.Flow; -import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -32,6 +31,7 @@ import org.springframework.context.annotation.Scope; import uk.ac.ebi.eva.pipeline.jobs.flows.AnnotationFlow; +import uk.ac.ebi.eva.pipeline.parameters.NewJobIncrementer; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.ANNOTATE_VARIANTS_JOB; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.VEP_ANNOTATION_FLOW; @@ -67,7 +67,7 @@ public Job annotateVariantsJob(JobBuilderFactory jobBuilderFactory) { JobBuilder jobBuilder = jobBuilderFactory .get(ANNOTATE_VARIANTS_JOB) - .incrementer(new RunIdIncrementer()); + .incrementer(new NewJobIncrementer()); return jobBuilder.start(annotation).build().build(); } diff --git a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/DatabaseInitializationJob.java b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/DatabaseInitializationJob.java index db3ede086..7c23d22c5 100644 --- a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/DatabaseInitializationJob.java +++ b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/DatabaseInitializationJob.java @@ -32,6 +32,7 @@ import uk.ac.ebi.eva.pipeline.jobs.steps.CreateDatabaseIndexesStep; import uk.ac.ebi.eva.pipeline.jobs.steps.GeneLoaderStep; +import uk.ac.ebi.eva.pipeline.parameters.NewJobIncrementer; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.CREATE_DATABASE_INDEXES_STEP; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.GENES_LOAD_STEP; @@ -67,7 +68,7 @@ public Job initDatabaseJob(JobBuilderFactory jobBuilderFactory) { JobBuilder jobBuilder = jobBuilderFactory .get(INIT_DATABASE_JOB) - .incrementer(new RunIdIncrementer()); + .incrementer(new NewJobIncrementer()); return jobBuilder .start(createDatabaseIndexesStep) diff --git a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJob.java b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJob.java index 7e0b8b39d..c7a6b3ca9 100644 --- a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJob.java +++ b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJob.java @@ -35,6 +35,7 @@ import uk.ac.ebi.eva.pipeline.jobs.steps.DropFilesByStudyStep; import uk.ac.ebi.eva.pipeline.jobs.steps.DropSingleStudyVariantsStep; import uk.ac.ebi.eva.pipeline.jobs.steps.PullFilesAndStatisticsByStudyStep; +import uk.ac.ebi.eva.pipeline.parameters.NewJobIncrementer; import uk.ac.ebi.eva.pipeline.parameters.validation.job.DropStudyJobParametersValidator; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.DROP_FILES_BY_STUDY_STEP; @@ -73,7 +74,7 @@ public Job dropStudyJob(JobBuilderFactory jobBuilderFactory) { JobBuilder jobBuilder = jobBuilderFactory .get(DROP_STUDY_JOB) - .incrementer(new RunIdIncrementer()) + .incrementer(new NewJobIncrementer()) .validator(new DropStudyJobParametersValidator()); SimpleJobBuilder builder = jobBuilder diff --git a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJob.java b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJob.java index 88323f983..f0fac98dc 100644 --- a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJob.java +++ b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJob.java @@ -35,6 +35,7 @@ import uk.ac.ebi.eva.pipeline.jobs.flows.ParallelStatisticsAndAnnotationFlow; import uk.ac.ebi.eva.pipeline.jobs.steps.LoadFileStep; import uk.ac.ebi.eva.pipeline.jobs.steps.VariantLoaderStep; +import uk.ac.ebi.eva.pipeline.parameters.NewJobIncrementer; import uk.ac.ebi.eva.pipeline.parameters.validation.job.GenotypedVcfJobParametersValidator; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.GENOTYPED_VCF_JOB; @@ -77,7 +78,7 @@ public Job genotypedVcfJob(JobBuilderFactory jobBuilderFactory) { JobBuilder jobBuilder = jobBuilderFactory .get(GENOTYPED_VCF_JOB) - .incrementer(new RunIdIncrementer()) + .incrementer(new NewJobIncrementer()) .validator(new GenotypedVcfJobParametersValidator()); FlowJobBuilder builder = jobBuilder .flow(variantLoaderStep) diff --git a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJob.java b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJob.java index 392a7ba34..e30f9cbf5 100644 --- a/src/main/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJob.java +++ b/src/main/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJob.java @@ -31,6 +31,7 @@ import org.springframework.context.annotation.Scope; import uk.ac.ebi.eva.pipeline.jobs.flows.PopulationStatisticsFlow; +import uk.ac.ebi.eva.pipeline.parameters.NewJobIncrementer; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.CALCULATE_STATISTICS_FLOW; import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.CALCULATE_STATISTICS_JOB; @@ -58,7 +59,7 @@ public Job calculateStatisticsJob(JobBuilderFactory jobBuilderFactory) { JobBuilder jobBuilder = jobBuilderFactory .get(CALCULATE_STATISTICS_JOB) - .incrementer(new RunIdIncrementer()); + .incrementer(new NewJobIncrementer()); return jobBuilder .start(optionalStatisticsFlow) diff --git a/src/main/java/uk/ac/ebi/eva/pipeline/parameters/NewJobIncrementer.java b/src/main/java/uk/ac/ebi/eva/pipeline/parameters/NewJobIncrementer.java new file mode 100644 index 000000000..25c04d937 --- /dev/null +++ b/src/main/java/uk/ac/ebi/eva/pipeline/parameters/NewJobIncrementer.java @@ -0,0 +1,50 @@ +/* + * Copyright 2015-2017 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.pipeline.parameters; + +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.JobParametersIncrementer; + +/** + * Incrementer that does not reuse parameters from previous completed executions. + */ +public class NewJobIncrementer implements JobParametersIncrementer { + + private static String RUN_ID_KEY = "run.id"; + + private String key = RUN_ID_KEY; + + /** + * The name of the run id in the job parameters. Defaults to "run.id". + * + * @param key the key to set + */ + public void setKey(String key) { + this.key = key; + } + + /** + * Increment the run.id parameter (starting with 1), and ignore the other parameters. + */ + @Override + public JobParameters getNext(JobParameters parameters) { + JobParameters params = (parameters == null) ? new JobParameters() : parameters; + + long id = params.getLong(key, 0L) + 1; + return new JobParametersBuilder().addLong(key, id).toJobParameters(); + } +} From 604eca155c2fd50969a3742bcd6a31fbe9a6aeca Mon Sep 17 00:00:00 2001 From: jose miguel mut Date: Wed, 10 May 2017 09:45:39 +0100 Subject: [PATCH 3/5] small refactor EvaPipelineJobLauncherCommandLineRunnerTest --- ...elineJobLauncherCommandLineRunnerTest.java | 70 +++++++------------ 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java index a387e9a86..e1bcd90ab 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java @@ -23,6 +23,7 @@ import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; +import org.springframework.batch.core.JobInstance; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -40,6 +41,7 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.util.List; import static org.hamcrest.core.StringContains.containsString; import static org.junit.Assert.assertEquals; @@ -145,31 +147,31 @@ public void genotypedVcfJobTest() throws JobExecutionException, IOException, URI assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); - assertFalse(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1).isEmpty()); - JobExecution jobExecution = jobExplorer.getJobExecution(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1) - .get(0).getInstanceId()); - - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobExecution jobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); + assertCompleted(jobExecution); GenotypedVcfJobTestUtils.checkLoadStep(databaseName); GenotypedVcfJobTestUtils.checkCreateStatsStep(variantsStatsFile, sourceStatsFile); - GenotypedVcfJobTestUtils.checkLoadStatsStep(databaseName); GenotypedVcfJobTestUtils.checkAnnotationCreateStep(vepOutputFile); - GenotypedVcfJobTestUtils.checkOutputFileLength(vepOutputFile); - GenotypedVcfJobTestUtils.checkLoadedAnnotation(databaseName); - GenotypedVcfJobTestUtils.checkSkippedOneMalformedLine(jobExecution); } + private JobExecution getLastJobExecution(String jobName) { + List jobInstances = jobExplorer.getJobInstances(jobName, 0, 1); + assertFalse(jobInstances.isEmpty()); + return jobExplorer.getJobExecution(jobInstances.get(0).getInstanceId()); + } + + private void assertCompleted(JobExecution jobExecution) { + assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + } + @Test public void doNotReuseParametersFromPreviousCompletedJobs() throws JobExecutionException, IOException, URISyntaxException, ClassNotFoundException, StorageManagerException, InstantiationException, IllegalAccessException { @@ -192,36 +194,29 @@ public void doNotReuseParametersFromPreviousCompletedJobs() throws JobExecutionE .statisticsSkip(true); String[] commandLineWithoutChunksize = evaCommandLineBuilderWithoutChunksize - .inputVcfId("second_" + GenotypedVcfJobTestUtils.INPUT_VCF_ID) + .inputVcfId("without_chunksize_" + GenotypedVcfJobTestUtils.INPUT_VCF_ID) .build(); String[] completeCommandLine = evaCommandLineBuilderWithoutChunksize - .inputVcfId("first_" + GenotypedVcfJobTestUtils.INPUT_VCF_ID) + .inputVcfId(GenotypedVcfJobTestUtils.INPUT_VCF_ID) .chunksize("100") .build(); evaPipelineJobLauncherCommandLineRunner.run(completeCommandLine); - assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); - assertFalse(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1).isEmpty()); - JobExecution jobExecution = jobExplorer.getJobExecution(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1) - .get(0).getInstanceId()); - - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobExecution firstJobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); + assertCompleted(firstJobExecution); + assertNotNull(firstJobExecution.getJobParameters().getString(JobParametersNames.DB_NAME)); + assertNotNull(firstJobExecution.getJobParameters().getString(JobParametersNames.CONFIG_CHUNK_SIZE)); // check second run doesn't include the chunksize parameter evaPipelineJobLauncherCommandLineRunner.run(commandLineWithoutChunksize); assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); - assertFalse(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1).isEmpty()); - JobExecution secondJobExecution = jobExplorer.getJobExecution(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1) - .get(0).getInstanceId()); + JobExecution secondJobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); + assertCompleted(secondJobExecution); assertNotNull(secondJobExecution.getJobParameters().getString(JobParametersNames.DB_NAME)); assertNull(secondJobExecution.getJobParameters().getString(JobParametersNames.CONFIG_CHUNK_SIZE)); @@ -263,28 +258,17 @@ public void genotypedVcfJobTestWithParametersFileAndCommandLineParameters() thro assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); - assertFalse(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1).isEmpty()); - JobExecution jobExecution = jobExplorer.getJobExecution(jobExplorer.getJobInstances(GENOTYPED_VCF_JOB, 0, 1) - .get(0).getInstanceId()); - - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobExecution jobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); + assertCompleted(jobExecution); GenotypedVcfJobTestUtils.checkLoadStep(databaseName); GenotypedVcfJobTestUtils.checkCreateStatsStep(variantsStatsFile, sourceStatsFile); - GenotypedVcfJobTestUtils.checkLoadStatsStep(databaseName); GenotypedVcfJobTestUtils.checkAnnotationCreateStep(vepOutputFile); - GenotypedVcfJobTestUtils.checkOutputFileLength(vepOutputFile); - GenotypedVcfJobTestUtils.checkLoadedAnnotation(databaseName); - GenotypedVcfJobTestUtils.checkSkippedOneMalformedLine(jobExecution); } @@ -298,9 +282,7 @@ public void onlyFileWithoutParametersFailsValidation() throws JobExecutionExcept getResource(GENOTYPED_PROPERTIES_FILE).getAbsolutePath()); evaPipelineJobLauncherCommandLineRunner.setJobNames(GENOTYPED_VCF_JOB); - evaPipelineJobLauncherCommandLineRunner.run(new EvaCommandLineBuilder() - .build() - ); + evaPipelineJobLauncherCommandLineRunner.run(new EvaCommandLineBuilder().build()); assertEquals(EvaPipelineJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); From 2ab10beac2884c0594cf150812ef3155d766cd3a Mon Sep 17 00:00:00 2001 From: jose miguel mut Date: Thu, 11 May 2017 11:17:02 +0100 Subject: [PATCH 4/5] extracted and reused `assertCompleted` and `assertFailed` --- .../eva/pipeline/jobs/AggregatedVcfJobTest.java | 10 ++++------ .../ebi/eva/pipeline/jobs/AnnotationJobTest.java | 7 +++---- .../ebi/eva/pipeline/jobs/DropStudyJobTest.java | 3 +-- .../eva/pipeline/jobs/GenotypedVcfJobTest.java | 8 ++++---- .../jobs/GenotypedVcfJobWorkflowTest.java | 13 +++++-------- .../jobs/PopulationStatisticsJobTest.java | 4 ++-- .../jobs/steps/AnnotationLoaderStepTest.java | 4 ++-- .../jobs/steps/AnnotationMetadataStepTest.java | 4 ++-- .../jobs/steps/DropFilesByStudyStepTest.java | 3 +-- .../steps/DropSingleStudyVariantsStepTest.java | 4 ++-- .../steps/GenerateVepAnnotationStepTest.java | 11 +++-------- .../jobs/steps/IndexesGeneratorStepTest.java | 7 +++---- .../pipeline/jobs/steps/LoadFileStepTest.java | 4 ++-- .../PopulationStatisticsGeneratorStepTest.java | 7 ++++--- .../PopulationStatisticsLoaderStepTest.java | 7 ++++--- .../PullFilesAndStatisticsByStudyStepTest.java | 4 ++-- .../jobs/steps/VariantLoaderStepTest.java | 4 ++-- ...PipelineJobLauncherCommandLineRunnerTest.java | 16 +++++----------- .../uk/ac/ebi/eva/test/utils/JobTestUtils.java | 14 ++++++++++++++ 19 files changed, 65 insertions(+), 69 deletions(-) diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AggregatedVcfJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AggregatedVcfJobTest.java index c323c726d..2b3b217fe 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AggregatedVcfJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AggregatedVcfJobTest.java @@ -59,6 +59,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertFailed; import static uk.ac.ebi.eva.utils.FileUtils.getResource; /** @@ -111,11 +113,8 @@ public void aggregatedTransformAndLoadShouldBeExecuted() throws Exception { .toJobParameters(); JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - // check execution flow - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); + assertCompleted(jobExecution); Collection stepExecutions = jobExecution.getStepExecutions(); Set names = stepExecutions.stream().map(StepExecution::getStepName) @@ -161,7 +160,6 @@ public void aggregationNoneIsNotAllowed() throws Exception { .toJobParameters(); JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - assertEquals(ExitStatus.FAILED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.FAILED, jobExecution.getStatus()); + assertFailed(jobExecution); } } diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java index 750f21668..f6d0cd6a3 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java @@ -41,6 +41,7 @@ import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import uk.ac.ebi.eva.utils.URLHelper; @@ -114,8 +115,7 @@ public void allAnnotationStepsShouldBeExecuted() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); assertEquals(3, jobExecution.getStepExecutions().size()); List steps = new ArrayList<>(jobExecution.getStepExecutions()); @@ -176,8 +176,7 @@ public void noVariantsToAnnotateOnlyGenerateAnnotationStepShouldRun() throws Exc JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); assertEquals(1, jobExecution.getStepExecutions().size()); StepExecution findVariantsToAnnotateStep = new ArrayList<>(jobExecution.getStepExecutions()).get(0); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java index 6cdb91330..33e84f86f 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java @@ -99,8 +99,7 @@ public void fullDropStudyJob() throws Exception { .toJobParameters(); JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); assertDropSingleStudy(variantsCollection, STUDY_ID_TO_DROP, EXPECTED_VARIANTS_AFTER_DROP_STUDY); assertPullStudy(variantsCollection, STUDY_ID_TO_DROP, EXPECTED_FILE_COUNT, EXPECTED_STATS_COUNT); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java index 1f102e711..9e7cb056d 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java @@ -36,11 +36,13 @@ import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; import uk.ac.ebi.eva.test.utils.GenotypedVcfJobTestUtils; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import java.io.File; import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertFailed; /** * Test for {@link GenotypedVcfJob} @@ -108,8 +110,7 @@ public void fullGenotypedVcfJob() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); GenotypedVcfJobTestUtils.checkLoadStep(databaseName); @@ -163,7 +164,6 @@ public void aggregationIsNotAllowed() throws Exception { .toJobParameters(); JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - assertEquals(ExitStatus.FAILED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.FAILED, jobExecution.getStatus()); + assertFailed(jobExecution); } } diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobWorkflowTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobWorkflowTest.java index 3024dabf7..b39b4607c 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobWorkflowTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobWorkflowTest.java @@ -52,6 +52,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.utils.FileUtils.getResource; /** @@ -101,8 +102,7 @@ public void allStepsShouldBeExecuted() throws Exception { JobParameters jobParameters = builder.toJobParameters(); JobExecution execution = jobLauncherTestUtils.launchJob(jobParameters); - - assertEquals(ExitStatus.COMPLETED, execution.getExitStatus()); + assertCompleted(execution); Collection stepExecutions = execution.getStepExecutions(); Map nameToStepExecution = stepExecutions.stream().collect( @@ -138,8 +138,7 @@ public void optionalStepsShouldBeSkipped() throws Exception { JobParameters jobParameters = builder.annotationSkip(true).statisticsSkip(true).toJobParameters(); JobExecution execution = jobLauncherTestUtils.launchJob(jobParameters); - - assertEquals(ExitStatus.COMPLETED, execution.getExitStatus()); + assertCompleted(execution); Set names = execution.getStepExecutions().stream().map(StepExecution::getStepName) .collect(Collectors.toSet()); @@ -153,8 +152,7 @@ public void statsStepsShouldBeSkipped() throws Exception { JobParameters jobParameters = builder.statisticsSkip(true).toJobParameters(); JobExecution execution = jobLauncherTestUtils.launchJob(jobParameters); - - assertEquals(ExitStatus.COMPLETED, execution.getExitStatus()); + assertCompleted(execution); Collection stepExecutions = execution.getStepExecutions(); Map nameToStepExecution = stepExecutions.stream().collect( @@ -185,8 +183,7 @@ public void annotationStepsShouldBeSkipped() throws Exception { JobParameters jobParameters = builder.annotationSkip(true).toJobParameters(); JobExecution execution = jobLauncherTestUtils.launchJob(jobParameters); - - assertEquals(ExitStatus.COMPLETED, execution.getExitStatus()); + assertCompleted(execution); Collection stepExecutions = execution.getStepExecutions(); Map nameToStepExecution = stepExecutions.stream().collect( diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJobTest.java index 2fd0e07b2..ba2a10b47 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJobTest.java @@ -36,6 +36,7 @@ import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import uk.ac.ebi.eva.utils.URLHelper; @@ -88,8 +89,7 @@ public void fullPopulationStatisticsJob() throws Exception { .toJobParameters(); JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); //and the file containing statistics should exist File statsFile = new File(URLHelper.getVariantsStatsUri(statsDir, studyId, fileId)); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationLoaderStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationLoaderStepTest.java index 6bcffd05c..aa0569a2a 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationLoaderStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationLoaderStepTest.java @@ -42,6 +42,7 @@ import uk.ac.ebi.eva.test.data.VepOutputContent; import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import uk.ac.ebi.eva.utils.URLHelper; @@ -94,8 +95,7 @@ public void shouldLoadAllAnnotations() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_VEP_ANNOTATION_STEP, jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); //check that documents have the annotation DBCursor cursor = mongoRule.getCollection(dbName, collectionVariantsName).find(); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationMetadataStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationMetadataStepTest.java index 6ecdf83a4..205a07514 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationMetadataStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationMetadataStepTest.java @@ -40,6 +40,7 @@ import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import java.util.List; @@ -98,8 +99,7 @@ private void assertStepIsComplete(String databaseName, String vepCacheVersion, S JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_ANNOTATION_METADATA_STEP, jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); } @Test diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropFilesByStudyStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropFilesByStudyStepTest.java index bb62c0b73..291329829 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropFilesByStudyStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropFilesByStudyStepTest.java @@ -110,8 +110,7 @@ private void checkDrop(String databaseName, long expectedFilesAfterDropStudy) { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.DROP_FILES_BY_STUDY_STEP, jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); DBCollection filesCollection = mongoRule.getCollection(databaseName, COLLECTION_FILES_NAME); assertDropFiles(filesCollection, STUDY_ID_TO_DROP, expectedFilesAfterDropStudy); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropSingleStudyVariantsStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropSingleStudyVariantsStepTest.java index 90652515a..ffaf7fdd8 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropSingleStudyVariantsStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropSingleStudyVariantsStepTest.java @@ -37,6 +37,7 @@ import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.data.VariantData; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import java.util.Arrays; @@ -105,8 +106,7 @@ private void checkDrop(String databaseName, long expectedVariantsAfterDropStudy) JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.DROP_SINGLE_STUDY_VARIANTS_STEP, jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); DBCollection variantsCollection = mongoRule.getCollection(databaseName, COLLECTION_VARIANTS_NAME); assertDropSingleStudy(variantsCollection, STUDY_ID_TO_DROP, expectedVariantsAfterDropStudy); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/GenerateVepAnnotationStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/GenerateVepAnnotationStepTest.java index 8ab2c68ee..3482caaad 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/GenerateVepAnnotationStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/GenerateVepAnnotationStepTest.java @@ -20,8 +20,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.batch.core.BatchStatus; -import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameters; import org.springframework.batch.test.JobLauncherTestUtils; @@ -110,8 +108,7 @@ public void shouldGenerateVepAnnotations() throws Exception { .launchStep(BeanNames.GENERATE_VEP_ANNOTATION_STEP, jobParameters); //Then variantsAnnotCreate step should complete correctly - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); // And VEP output should exist and annotations should be in the file assertTrue(vepOutput.exists()); @@ -145,8 +142,7 @@ public void shouldResumeJob() throws Exception { JobExecution jobExecution = jobLauncherTestUtils .launchStep(BeanNames.GENERATE_VEP_ANNOTATION_STEP, jobParameters); - assertEquals(ExitStatus.FAILED.getExitCode(), jobExecution.getExitStatus().getExitCode()); - assertEquals(BatchStatus.FAILED, jobExecution.getStatus()); + JobTestUtils.assertFailed(jobExecution); assertTrue(!vepOutput.exists()); List files = Files.list(Paths.get(outputDirAnnot)) @@ -161,8 +157,7 @@ public void shouldResumeJob() throws Exception { JobExecution secondJobExecution = jobLauncherTestUtils .launchStep(BeanNames.GENERATE_VEP_ANNOTATION_STEP, jobParameters); - assertEquals(ExitStatus.COMPLETED, secondJobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, secondJobExecution.getStatus()); + JobTestUtils.assertCompleted(secondJobExecution); assertTrue(vepOutput.exists()); int chunks = 3; diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/IndexesGeneratorStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/IndexesGeneratorStepTest.java index 0efa023e9..e8d79193f 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/IndexesGeneratorStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/IndexesGeneratorStepTest.java @@ -36,6 +36,7 @@ import uk.ac.ebi.eva.pipeline.jobs.steps.tasklets.IndexesGeneratorStep; import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import static org.junit.Assert.assertEquals; @@ -68,8 +69,7 @@ public void testIndexesAreCreated() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.CREATE_DATABASE_INDEXES_STEP, jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); DBCollection genesCollection = mongoRule.getCollection(databaseName, COLLECTION_FEATURES_NAME); assertEquals("[{ \"v\" : 1 , \"key\" : { \"_id\" : 1} , \"name\" : \"_id_\" , \"ns\" : \"" + @@ -90,8 +90,7 @@ public void testNoDuplicatesCanBeInserted() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.CREATE_DATABASE_INDEXES_STEP, jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); DBCollection genesCollection = mongoRule.getCollection(databaseName, COLLECTION_FEATURES_NAME); genesCollection.insert(new BasicDBObject("_id", "example_id")); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java index 6650a08ef..4c6252f43 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java @@ -36,6 +36,7 @@ import uk.ac.ebi.eva.pipeline.jobs.GenotypedVcfJob; import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import static org.junit.Assert.assertEquals; @@ -82,8 +83,7 @@ public void loaderStepShouldLoadAllFiles() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_FILE_STEP, jobParameters); //Then variantsLoad step should complete correctly - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); // And the number of documents in the DB should be equals to the number of VCF files loaded DBCollection fileCollection = mongoRule.getCollection(databaseName, COLLECTION_FILES_NAME); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsGeneratorStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsGeneratorStepTest.java index db55ebc04..95e3f15ea 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsGeneratorStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsGeneratorStepTest.java @@ -34,6 +34,7 @@ import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import uk.ac.ebi.eva.utils.URLHelper; @@ -44,6 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertFailed; import static uk.ac.ebi.eva.test.utils.TestFileUtils.getResourceUrl; /** @@ -92,8 +94,7 @@ public void statisticsGeneratorStepShouldCalculateStats() throws IOException, In JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.CALCULATE_STATISTICS_STEP, jobParameters); //Then variantsStatsCreate step should complete correctly - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); //and the file containing statistics should exist assertTrue(statsFile.exists()); @@ -127,7 +128,7 @@ public void statisticsGeneratorStepShouldFailIfVariantLoadStepIsNotCompleted() t // When the execute method in variantsStatsCreate is executed JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.CALCULATE_STATISTICS_STEP, jobParameters); - assertEquals(ExitStatus.FAILED.getExitCode(), jobExecution.getExitStatus().getExitCode()); + assertFailed(jobExecution); } } diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsLoaderStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsLoaderStepTest.java index d2699e1a8..57aed886b 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsLoaderStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsLoaderStepTest.java @@ -29,6 +29,7 @@ import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import java.io.IOException; @@ -37,6 +38,7 @@ import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertFailed; import static uk.ac.ebi.eva.test.utils.TestFileUtils.copyResource; import static uk.ac.ebi.eva.test.utils.TestFileUtils.getResourceUrl; import static uk.ac.ebi.eva.utils.FileUtils.getResource; @@ -98,8 +100,7 @@ public void statisticsLoaderStepShouldLoadStatsIntoDb() throws StorageManagerExc JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_STATISTICS_STEP, jobParameters); // Then variantsStatsLoad step should complete correctly - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); // The DB docs should have the field "st" DBCursor cursor = mongoRule.getCollection(dbName, COLLECTION_VARIANTS_NAME).find(); @@ -147,7 +148,7 @@ public void statisticsLoaderStepShouldFaildBecauseVariantStatsFileIsMissing() JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_STATISTICS_STEP, jobParameters); assertThat(capture.toString(), containsString(FILE_NOT_FOUND_EXCEPTION)); - assertEquals(ExitStatus.FAILED.getExitCode(), jobExecution.getExitStatus().getExitCode()); + assertFailed(jobExecution); } } diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PullFilesAndStatisticsByStudyStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PullFilesAndStatisticsByStudyStepTest.java index 5c67b27ef..f62fa1ed3 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PullFilesAndStatisticsByStudyStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PullFilesAndStatisticsByStudyStepTest.java @@ -36,6 +36,7 @@ import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.data.VariantData; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import java.io.IOException; @@ -139,8 +140,7 @@ private void executeStep(String databaseName) { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.PULL_FILES_AND_STATISTICS_BY_STUDY_STEP, jobParameters); - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); } private void checkPull(String databaseName, int expectedFileCount, int expectedStatsCount) { diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/VariantLoaderStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/VariantLoaderStepTest.java index 3bceb2b7b..f7819f533 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/VariantLoaderStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/VariantLoaderStepTest.java @@ -41,6 +41,7 @@ import uk.ac.ebi.eva.pipeline.jobs.GenotypedVcfJob; import uk.ac.ebi.eva.test.configuration.BatchTestConfiguration; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import static org.junit.Assert.assertEquals; @@ -89,8 +90,7 @@ public void loaderStepShouldLoadAllVariants() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_VARIANTS_STEP, jobParameters); //Then variantsLoad step should complete correctly - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + JobTestUtils.assertCompleted(jobExecution); // And the number of documents in the DB should be equals to the number of lines in the VCF file VariantStorageManager variantStorageManager = StorageManagerFactory.getVariantStorageManager(); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java index e1bcd90ab..1ac5e9f83 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java @@ -19,8 +19,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.opencb.opencga.storage.core.StorageManagerException; -import org.springframework.batch.core.BatchStatus; -import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobInstance; @@ -36,6 +34,7 @@ import uk.ac.ebi.eva.test.rules.PipelineTemporaryFolderRule; import uk.ac.ebi.eva.test.rules.TemporaryMongoRule; import uk.ac.ebi.eva.test.utils.GenotypedVcfJobTestUtils; +import uk.ac.ebi.eva.test.utils.JobTestUtils; import uk.ac.ebi.eva.utils.EvaCommandLineBuilder; import java.io.File; @@ -148,7 +147,7 @@ public void genotypedVcfJobTest() throws JobExecutionException, IOException, URI assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); JobExecution jobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); - assertCompleted(jobExecution); + JobTestUtils.assertCompleted(jobExecution); GenotypedVcfJobTestUtils.checkLoadStep(databaseName); @@ -167,11 +166,6 @@ private JobExecution getLastJobExecution(String jobName) { return jobExplorer.getJobExecution(jobInstances.get(0).getInstanceId()); } - private void assertCompleted(JobExecution jobExecution) { - assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); - assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); - } - @Test public void doNotReuseParametersFromPreviousCompletedJobs() throws JobExecutionException, IOException, URISyntaxException, ClassNotFoundException, StorageManagerException, InstantiationException, IllegalAccessException { @@ -206,7 +200,7 @@ public void doNotReuseParametersFromPreviousCompletedJobs() throws JobExecutionE assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); JobExecution firstJobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); - assertCompleted(firstJobExecution); + JobTestUtils.assertCompleted(firstJobExecution); assertNotNull(firstJobExecution.getJobParameters().getString(JobParametersNames.DB_NAME)); assertNotNull(firstJobExecution.getJobParameters().getString(JobParametersNames.CONFIG_CHUNK_SIZE)); @@ -216,7 +210,7 @@ public void doNotReuseParametersFromPreviousCompletedJobs() throws JobExecutionE assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); JobExecution secondJobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); - assertCompleted(secondJobExecution); + JobTestUtils.assertCompleted(secondJobExecution); assertNotNull(secondJobExecution.getJobParameters().getString(JobParametersNames.DB_NAME)); assertNull(secondJobExecution.getJobParameters().getString(JobParametersNames.CONFIG_CHUNK_SIZE)); @@ -259,7 +253,7 @@ public void genotypedVcfJobTestWithParametersFileAndCommandLineParameters() thro assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); JobExecution jobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); - assertCompleted(jobExecution); + JobTestUtils.assertCompleted(jobExecution); GenotypedVcfJobTestUtils.checkLoadStep(databaseName); diff --git a/src/test/java/uk/ac/ebi/eva/test/utils/JobTestUtils.java b/src/test/java/uk/ac/ebi/eva/test/utils/JobTestUtils.java index 045382caa..bf2da4653 100644 --- a/src/test/java/uk/ac/ebi/eva/test/utils/JobTestUtils.java +++ b/src/test/java/uk/ac/ebi/eva/test/utils/JobTestUtils.java @@ -21,6 +21,9 @@ import com.mongodb.util.JSON; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; @@ -38,6 +41,7 @@ import java.util.TreeSet; import java.util.zip.GZIPInputStream; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -151,4 +155,14 @@ public static String buildFilesDocumentString(String studyId, String fileId) { return "{\"" + STUDYID_FIELD + "\":\"" + studyId + "\", \"" + FILEID_FIELD + "\":\"" + fileId + "\"}"; } + + public static void assertCompleted(JobExecution jobExecution) { + assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus()); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + } + + public static void assertFailed(JobExecution jobExecution) { + assertEquals(ExitStatus.FAILED.getExitCode(), jobExecution.getExitStatus().getExitCode()); + assertEquals(BatchStatus.FAILED, jobExecution.getStatus()); + } } From 22e2af31d7528a3bce40b1822025666e8ea289ac Mon Sep 17 00:00:00 2001 From: jose miguel mut Date: Fri, 12 May 2017 14:33:41 +0100 Subject: [PATCH 5/5] consistent use of assertCompleted/Failed --- .../uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java | 5 +++-- .../uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java | 3 ++- .../uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java | 3 ++- .../eva/pipeline/jobs/PopulationStatisticsJobTest.java | 3 ++- .../pipeline/jobs/steps/AnnotationLoaderStepTest.java | 3 ++- .../pipeline/jobs/steps/AnnotationMetadataStepTest.java | 3 ++- .../pipeline/jobs/steps/DropFilesByStudyStepTest.java | 3 ++- .../jobs/steps/DropSingleStudyVariantsStepTest.java | 3 ++- .../jobs/steps/GenerateVepAnnotationStepTest.java | 8 +++++--- .../pipeline/jobs/steps/IndexesGeneratorStepTest.java | 5 +++-- .../ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java | 3 ++- .../steps/PopulationStatisticsGeneratorStepTest.java | 3 ++- .../jobs/steps/PopulationStatisticsLoaderStepTest.java | 3 ++- .../steps/PullFilesAndStatisticsByStudyStepTest.java | 3 ++- .../eva/pipeline/jobs/steps/VariantLoaderStepTest.java | 3 ++- .../EvaPipelineJobLauncherCommandLineRunnerTest.java | 9 +++++---- 16 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java index f6d0cd6a3..d0f4c2e4f 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/AnnotationJobTest.java @@ -55,6 +55,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.test.utils.TestFileUtils.getResourceUrl; import static uk.ac.ebi.eva.utils.FileUtils.getResource; @@ -115,7 +116,7 @@ public void allAnnotationStepsShouldBeExecuted() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); assertEquals(3, jobExecution.getStepExecutions().size()); List steps = new ArrayList<>(jobExecution.getStepExecutions()); @@ -176,7 +177,7 @@ public void noVariantsToAnnotateOnlyGenerateAnnotationStepShouldRun() throws Exc JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); assertEquals(1, jobExecution.getStepExecutions().size()); StepExecution findVariantsToAnnotateStep = new ArrayList<>(jobExecution.getStepExecutions()).get(0); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java index 33e84f86f..1f9921463 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/DropStudyJobTest.java @@ -42,6 +42,7 @@ import static uk.ac.ebi.eva.test.utils.DropStudyJobTestUtils.assertDropFiles; import static uk.ac.ebi.eva.test.utils.DropStudyJobTestUtils.assertDropSingleStudy; import static uk.ac.ebi.eva.test.utils.DropStudyJobTestUtils.assertPullStudy; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; /** * Test for {@link PopulationStatisticsJob} @@ -99,7 +100,7 @@ public void fullDropStudyJob() throws Exception { .toJobParameters(); JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); assertDropSingleStudy(variantsCollection, STUDY_ID_TO_DROP, EXPECTED_VARIANTS_AFTER_DROP_STUDY); assertPullStudy(variantsCollection, STUDY_ID_TO_DROP, EXPECTED_FILE_COUNT, EXPECTED_STATS_COUNT); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java index 9e7cb056d..7143651b1 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/GenotypedVcfJobTest.java @@ -42,6 +42,7 @@ import java.io.File; import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertFailed; /** @@ -110,7 +111,7 @@ public void fullGenotypedVcfJob() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); GenotypedVcfJobTestUtils.checkLoadStep(databaseName); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJobTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJobTest.java index ba2a10b47..75432e7e5 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJobTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/PopulationStatisticsJobTest.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.test.utils.TestFileUtils.getResourceUrl; import static uk.ac.ebi.eva.utils.FileUtils.getResource; @@ -89,7 +90,7 @@ public void fullPopulationStatisticsJob() throws Exception { .toJobParameters(); JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); //and the file containing statistics should exist File statsFile = new File(URLHelper.getVariantsStatsUri(statsDir, studyId, fileId)); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationLoaderStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationLoaderStepTest.java index aa0569a2a..e037916a6 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationLoaderStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationLoaderStepTest.java @@ -50,6 +50,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.test.utils.TestFileUtils.getResourceUrl; @@ -95,7 +96,7 @@ public void shouldLoadAllAnnotations() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_VEP_ANNOTATION_STEP, jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); //check that documents have the annotation DBCursor cursor = mongoRule.getCollection(dbName, collectionVariantsName).find(); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationMetadataStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationMetadataStepTest.java index 205a07514..285154355 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationMetadataStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/AnnotationMetadataStepTest.java @@ -46,6 +46,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; /** * TODO jmmut remove import AnnotationJob when we add the stepLauncherTestUtils @@ -99,7 +100,7 @@ private void assertStepIsComplete(String databaseName, String vepCacheVersion, S JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_ANNOTATION_METADATA_STEP, jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); } @Test diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropFilesByStudyStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropFilesByStudyStepTest.java index 291329829..7e67c97c7 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropFilesByStudyStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropFilesByStudyStepTest.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertEquals; import static uk.ac.ebi.eva.test.utils.DropStudyJobTestUtils.assertDropFiles; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; /** * Test for {@link DropFilesByStudyStep} @@ -110,7 +111,7 @@ private void checkDrop(String databaseName, long expectedFilesAfterDropStudy) { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.DROP_FILES_BY_STUDY_STEP, jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); DBCollection filesCollection = mongoRule.getCollection(databaseName, COLLECTION_FILES_NAME); assertDropFiles(filesCollection, STUDY_ID_TO_DROP, expectedFilesAfterDropStudy); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropSingleStudyVariantsStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropSingleStudyVariantsStepTest.java index ffaf7fdd8..886ee017e 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropSingleStudyVariantsStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/DropSingleStudyVariantsStepTest.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertEquals; import static uk.ac.ebi.eva.test.utils.DropStudyJobTestUtils.assertDropSingleStudy; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; /** * Test for {@link DropSingleStudyVariantsStep} @@ -106,7 +107,7 @@ private void checkDrop(String databaseName, long expectedVariantsAfterDropStudy) JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.DROP_SINGLE_STUDY_VARIANTS_STEP, jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); DBCollection variantsCollection = mongoRule.getCollection(databaseName, COLLECTION_VARIANTS_NAME); assertDropSingleStudy(variantsCollection, STUDY_ID_TO_DROP, expectedVariantsAfterDropStudy); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/GenerateVepAnnotationStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/GenerateVepAnnotationStepTest.java index 3482caaad..68326f406 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/GenerateVepAnnotationStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/GenerateVepAnnotationStepTest.java @@ -50,6 +50,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertFailed; import static uk.ac.ebi.eva.test.utils.TestFileUtils.getResourceUrl; import static uk.ac.ebi.eva.utils.FileUtils.getResource; @@ -108,7 +110,7 @@ public void shouldGenerateVepAnnotations() throws Exception { .launchStep(BeanNames.GENERATE_VEP_ANNOTATION_STEP, jobParameters); //Then variantsAnnotCreate step should complete correctly - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); // And VEP output should exist and annotations should be in the file assertTrue(vepOutput.exists()); @@ -142,7 +144,7 @@ public void shouldResumeJob() throws Exception { JobExecution jobExecution = jobLauncherTestUtils .launchStep(BeanNames.GENERATE_VEP_ANNOTATION_STEP, jobParameters); - JobTestUtils.assertFailed(jobExecution); + assertFailed(jobExecution); assertTrue(!vepOutput.exists()); List files = Files.list(Paths.get(outputDirAnnot)) @@ -157,7 +159,7 @@ public void shouldResumeJob() throws Exception { JobExecution secondJobExecution = jobLauncherTestUtils .launchStep(BeanNames.GENERATE_VEP_ANNOTATION_STEP, jobParameters); - JobTestUtils.assertCompleted(secondJobExecution); + assertCompleted(secondJobExecution); assertTrue(vepOutput.exists()); int chunks = 3; diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/IndexesGeneratorStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/IndexesGeneratorStepTest.java index e8d79193f..ec02d68ef 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/IndexesGeneratorStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/IndexesGeneratorStepTest.java @@ -40,6 +40,7 @@ import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; /** @@ -69,7 +70,7 @@ public void testIndexesAreCreated() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.CREATE_DATABASE_INDEXES_STEP, jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); DBCollection genesCollection = mongoRule.getCollection(databaseName, COLLECTION_FEATURES_NAME); assertEquals("[{ \"v\" : 1 , \"key\" : { \"_id\" : 1} , \"name\" : \"_id_\" , \"ns\" : \"" + @@ -90,7 +91,7 @@ public void testNoDuplicatesCanBeInserted() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.CREATE_DATABASE_INDEXES_STEP, jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); DBCollection genesCollection = mongoRule.getCollection(databaseName, COLLECTION_FEATURES_NAME); genesCollection.insert(new BasicDBObject("_id", "example_id")); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java index 4c6252f43..21785dbe8 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/LoadFileStepTest.java @@ -40,6 +40,7 @@ import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.test.utils.JobTestUtils.count; import static uk.ac.ebi.eva.utils.FileUtils.getResource; @@ -83,7 +84,7 @@ public void loaderStepShouldLoadAllFiles() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_FILE_STEP, jobParameters); //Then variantsLoad step should complete correctly - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); // And the number of documents in the DB should be equals to the number of VCF files loaded DBCollection fileCollection = mongoRule.getCollection(databaseName, COLLECTION_FILES_NAME); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsGeneratorStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsGeneratorStepTest.java index 95e3f15ea..62030996b 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsGeneratorStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsGeneratorStepTest.java @@ -45,6 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertFailed; import static uk.ac.ebi.eva.test.utils.TestFileUtils.getResourceUrl; @@ -94,7 +95,7 @@ public void statisticsGeneratorStepShouldCalculateStats() throws IOException, In JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.CALCULATE_STATISTICS_STEP, jobParameters); //Then variantsStatsCreate step should complete correctly - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); //and the file containing statistics should exist assertTrue(statsFile.exists()); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsLoaderStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsLoaderStepTest.java index 57aed886b..6cfe074a0 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsLoaderStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PopulationStatisticsLoaderStepTest.java @@ -38,6 +38,7 @@ import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertFailed; import static uk.ac.ebi.eva.test.utils.TestFileUtils.copyResource; import static uk.ac.ebi.eva.test.utils.TestFileUtils.getResourceUrl; @@ -100,7 +101,7 @@ public void statisticsLoaderStepShouldLoadStatsIntoDb() throws StorageManagerExc JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_STATISTICS_STEP, jobParameters); // Then variantsStatsLoad step should complete correctly - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); // The DB docs should have the field "st" DBCursor cursor = mongoRule.getCollection(dbName, COLLECTION_VARIANTS_NAME).find(); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PullFilesAndStatisticsByStudyStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PullFilesAndStatisticsByStudyStepTest.java index f62fa1ed3..89547ffda 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PullFilesAndStatisticsByStudyStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/PullFilesAndStatisticsByStudyStepTest.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertEquals; import static uk.ac.ebi.eva.test.utils.DropStudyJobTestUtils.assertPullStudy; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; /** * Test for {@link PullFilesAndStatisticsByStudyStep} @@ -140,7 +141,7 @@ private void executeStep(String databaseName) { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.PULL_FILES_AND_STATISTICS_BY_STUDY_STEP, jobParameters); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); } private void checkPull(String databaseName, int expectedFileCount, int expectedStatsCount) { diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/VariantLoaderStepTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/VariantLoaderStepTest.java index f7819f533..073e1877f 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/VariantLoaderStepTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/jobs/steps/VariantLoaderStepTest.java @@ -45,6 +45,7 @@ import uk.ac.ebi.eva.utils.EvaJobParameterBuilder; import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.test.utils.JobTestUtils.count; import static uk.ac.ebi.eva.utils.FileUtils.getResource; @@ -90,7 +91,7 @@ public void loaderStepShouldLoadAllVariants() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.LOAD_VARIANTS_STEP, jobParameters); //Then variantsLoad step should complete correctly - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); // And the number of documents in the DB should be equals to the number of lines in the VCF file VariantStorageManager variantStorageManager = StorageManagerFactory.getVariantStorageManager(); diff --git a/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java b/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java index 1ac5e9f83..b91af5c97 100644 --- a/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java +++ b/src/test/java/uk/ac/ebi/eva/pipeline/runner/EvaPipelineJobLauncherCommandLineRunnerTest.java @@ -51,6 +51,7 @@ import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.GENOTYPED_VCF_JOB; import static uk.ac.ebi.eva.pipeline.runner.EvaPipelineJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS; import static uk.ac.ebi.eva.pipeline.runner.EvaPipelineJobLauncherCommandLineRunner.SPRING_BATCH_JOB_NAME_PROPERTY; +import static uk.ac.ebi.eva.test.utils.JobTestUtils.assertCompleted; import static uk.ac.ebi.eva.utils.FileUtils.getResource; /** @@ -147,7 +148,7 @@ public void genotypedVcfJobTest() throws JobExecutionException, IOException, URI assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); JobExecution jobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); GenotypedVcfJobTestUtils.checkLoadStep(databaseName); @@ -200,7 +201,7 @@ public void doNotReuseParametersFromPreviousCompletedJobs() throws JobExecutionE assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); JobExecution firstJobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); - JobTestUtils.assertCompleted(firstJobExecution); + assertCompleted(firstJobExecution); assertNotNull(firstJobExecution.getJobParameters().getString(JobParametersNames.DB_NAME)); assertNotNull(firstJobExecution.getJobParameters().getString(JobParametersNames.CONFIG_CHUNK_SIZE)); @@ -210,7 +211,7 @@ public void doNotReuseParametersFromPreviousCompletedJobs() throws JobExecutionE assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); JobExecution secondJobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); - JobTestUtils.assertCompleted(secondJobExecution); + assertCompleted(secondJobExecution); assertNotNull(secondJobExecution.getJobParameters().getString(JobParametersNames.DB_NAME)); assertNull(secondJobExecution.getJobParameters().getString(JobParametersNames.CONFIG_CHUNK_SIZE)); @@ -253,7 +254,7 @@ public void genotypedVcfJobTestWithParametersFileAndCommandLineParameters() thro assertEquals(EXIT_WITHOUT_ERRORS, evaPipelineJobLauncherCommandLineRunner.getExitCode()); JobExecution jobExecution = getLastJobExecution(GENOTYPED_VCF_JOB); - JobTestUtils.assertCompleted(jobExecution); + assertCompleted(jobExecution); GenotypedVcfJobTestUtils.checkLoadStep(databaseName);