diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java index a09f5f7f6c1..2293ee7c963 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java @@ -704,7 +704,7 @@ public OpenCGAResult geneQuery(String studyStr, Query query, .append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS) .append(SampleDBAdaptor.QueryParams.INTERNAL_RGA_STATUS.key(), RgaIndex.Status.INDEXED) .append(SampleDBAdaptor.QueryParams.ID.key(), includeIndividuals); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) authorisedSampleIdResult.getResults()); } else { @@ -720,7 +720,7 @@ public OpenCGAResult geneQuery(String studyStr, Query query, // 3. Get list of sample ids for which the user has permissions Query sampleQuery = new Query(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS) .append(SampleDBAdaptor.QueryParams.ID.key(), sampleIds); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); // TODO: The number of samples to include could be really high includeSampleIds = new HashSet<>((List) authorisedSampleIdResult.getResults()); @@ -737,7 +737,7 @@ public OpenCGAResult geneQuery(String studyStr, Query query, logger.warn("Include only the samples that are actually necessary"); } - OpenCGAResult sampleResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), + OpenCGAResult sampleResult = catalogManager.getSampleManager().distinct(study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) sampleResult.getResults()); } @@ -831,7 +831,7 @@ public OpenCGAResult variantQuery(String studyStr, Query quer .append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS) .append(SampleDBAdaptor.QueryParams.INTERNAL_RGA_STATUS.key(), RgaIndex.Status.INDEXED) .append(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), includeIndividuals); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) authorisedSampleIdResult.getResults()); } else { @@ -850,7 +850,7 @@ public OpenCGAResult variantQuery(String studyStr, Query quer Query sampleQuery = new Query() .append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS) .append(SampleDBAdaptor.QueryParams.ID.key(), sampleIds); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) authorisedSampleIdResult.getResults()); } @@ -865,7 +865,7 @@ public OpenCGAResult variantQuery(String studyStr, Query quer logger.warn("Include only the samples that are actually necessary"); } - OpenCGAResult sampleResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), + OpenCGAResult sampleResult = catalogManager.getSampleManager().distinct(study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) sampleResult.getResults()); } @@ -1773,7 +1773,7 @@ private Set getAuthorisedSamples(String organizationId, String study, Se query.put(ACL_PARAM, userId + ":" + StringUtils.join(otherPermissions, ",")); } - OpenCGAResult distinct = catalogManager.getSampleManager().distinct(organizationId, study, SampleDBAdaptor.QueryParams.ID.key(), + OpenCGAResult distinct = catalogManager.getSampleManager().distinct(study, SampleDBAdaptor.QueryParams.ID.key(), query, token); return distinct.getResults().stream().map(String::valueOf).collect(Collectors.toSet()); } @@ -1874,7 +1874,7 @@ private Preprocess individualQueryPreprocess(String organizationId, Study study, List tmpValues = values.subList(currentBatch, Math.min(values.size(), batchSize + currentBatch)); sampleQuery.put(sampleQueryField, tmpValues); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); authorisedSamples.addAll((Collection) authorisedSampleIdResult.getResults()); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java index 55e40d9dddb..ad22b2fb8dc 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java @@ -33,6 +33,7 @@ import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.FileUtils; import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.common.UriUtils; @@ -85,7 +86,7 @@ public class VariantFileIndexerOperationManager extends OperationManager { public static final String TRANSFORMED_FILES = "transformedFiles"; public static final String SKIP_INDEXED_FILES = "skipIndexedFiles"; - private final Logger logger; + private static final Logger LOGGER = LoggerFactory.getLogger(VariantFileIndexerOperationManager.class); private String studyFqn; private String organizationId; @@ -107,7 +108,6 @@ public class VariantFileIndexerOperationManager extends OperationManager { public VariantFileIndexerOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) { super(variantStorageManager, engine); - logger = LoggerFactory.getLogger(VariantFileIndexerOperationManager.class); } public List index(String study, List files, URI outDirUri, ObjectMap params, String token) @@ -120,7 +120,7 @@ public List index(String study, List files, URI o List fileUris = findFilesToIndex(params, token); if (fileUris.size() == 0) { - logger.warn("Nothing to do."); + LOGGER.warn("Nothing to do."); return Collections.emptyList(); } @@ -194,38 +194,12 @@ private void updateProject(String studyFqn, String token) throws CatalogExceptio private List findFilesToIndex(ObjectMap params, String token) throws CatalogException, URISyntaxException, StorageEngineException { synchronizer = new CatalogStorageMetadataSynchronizer(catalogManager, variantStorageEngine.getMetadataManager()); - List inputFiles = new ArrayList<>(); - for (String file : files) { - File inputFile = catalogManager.getFileManager().get(studyFqn, file, FILE_GET_QUERY_OPTIONS, token).first(); - - if (inputFile.getType() == File.Type.FILE) { - // If is a transformed file, get the related VCF file - if (VariantReaderUtils.isTransformedVariants(inputFile.getName())) { - inputFiles.add(getOriginalFromTransformed(studyFqn, inputFile, token)); - } else { - inputFiles.add(inputFile); - } - } else { - if (inputFile.getType() == File.Type.DIRECTORY) { - Query query = new Query(FileDBAdaptor.QueryParams.DIRECTORY.key(), inputFile.getPath()); - query.append(FileDBAdaptor.QueryParams.FORMAT.key(), -// Arrays.asList(File.Format.VCF, File.Format.GVCF, File.Format.AVRO)); - Arrays.asList(File.Format.VCF, File.Format.GVCF)); - DataResult fileDataResult = catalogManager.getFileManager().search(studyFqn, query, FILE_GET_QUERY_OPTIONS, - token); -// fileDataResult.getResults().sort(Comparator.comparing(File::getName)); - inputFiles.addAll(fileDataResult.getResults()); - } else { - throw new CatalogException(String.format("Expected file type %s or %s instead of %s", - File.Type.FILE, File.Type.DIRECTORY, inputFile.getType())); - } - } - } + List inputFiles = getInputFiles(catalogManager, studyFqn, files, token); // Update Catalog from the storage metadata. This may change the index status of the inputFiles . synchronizer.synchronizeCatalogFilesFromStorage(studyFqn, inputFiles, token, FILE_GET_QUERY_OPTIONS); - logger.debug("Index - Number of files to be indexed: {}, list of files: {}", inputFiles.size(), + LOGGER.debug("Index - Number of files to be indexed: {}, list of files: {}", inputFiles.size(), inputFiles.stream().map(File::getName).collect(Collectors.toList())); String fileStatus; @@ -291,6 +265,38 @@ private List findFilesToIndex(ObjectMap params, String token) throws Catalo return fileUris; } + public static List getInputFiles(CatalogManager catalogManager, String studyFqn, List files, String token) + throws CatalogException { + List inputFiles = new ArrayList<>(); + for (String file : files) { + File inputFile = catalogManager.getFileManager().get(studyFqn, file, FILE_GET_QUERY_OPTIONS, token).first(); + + if (inputFile.getType() == File.Type.FILE) { + // If is a transformed file, get the related VCF file + if (VariantReaderUtils.isTransformedVariants(inputFile.getName())) { + inputFiles.add(getOriginalFromTransformed(catalogManager, studyFqn, inputFile, token)); + } else { + inputFiles.add(inputFile); + } + } else { + if (inputFile.getType() == File.Type.DIRECTORY) { + Query query = new Query(FileDBAdaptor.QueryParams.DIRECTORY.key(), inputFile.getPath()); + query.append(FileDBAdaptor.QueryParams.FORMAT.key(), +// Arrays.asList(File.Format.VCF, File.Format.GVCF, File.Format.AVRO)); + Arrays.asList(File.Format.VCF, File.Format.GVCF)); + DataResult fileDataResult = catalogManager.getFileManager().search(studyFqn, query, FILE_GET_QUERY_OPTIONS, + token); +// fileDataResult.getResults().sort(Comparator.comparing(File::getName)); + inputFiles.addAll(fileDataResult.getResults()); + } else { + throw new CatalogException(String.format("Expected file type %s or %s instead of %s", + File.Type.FILE, File.Type.DIRECTORY, inputFile.getType())); + } + } + } + return inputFiles; + } + private List indexFiles(List fileUris, String token, ObjectMap params) throws Exception { String prevDefaultCohortStatus = CohortStatus.NONE; @@ -300,7 +306,7 @@ private List indexFiles(List fileUris, String token, } } - logger.info("Starting to {}", step); + LOGGER.info("Starting to {}", step); // Save exception to throw at the end StorageEngineException exception = null; @@ -308,17 +314,17 @@ private List indexFiles(List fileUris, String token, try { storagePipelineResults = variantStorageEngine.index(fileUris, outDirUri, false, transform, load); } catch (StoragePipelineException e) { - logger.error("Error executing " + step, e); + LOGGER.error("Error executing " + step, e); storagePipelineResults = e.getResults(); exception = e; throw e; } catch (StorageEngineException e) { - logger.error("Error executing " + step, e); + LOGGER.error("Error executing " + step, e); storagePipelineResults = Collections.emptyList(); exception = e; throw e; } catch (RuntimeException e) { - logger.error("Error executing " + step, e); + LOGGER.error("Error executing " + step, e); storagePipelineResults = Collections.emptyList(); exception = new StorageEngineException("Error executing " + step, e); throw e; @@ -396,7 +402,7 @@ private void updateFileInfo(String study, List filesToIndex, VariantReader FileInternalVariantIndex index = indexedFile.getInternal().getVariant().getIndex(); if (index == null) { - logger.error("The execution should never get into this condition. Critical error."); + LOGGER.error("The execution should never get into this condition. Critical error."); throw new CatalogException("Critical error. Empty index parameter in file " + indexedFile.getUid()); } else { switch (index.getStatus().getId()) { @@ -405,7 +411,7 @@ private void updateFileInfo(String study, List filesToIndex, VariantReader indexStatusMessage = "Unexpected index status. Expected " + VariantIndexStatus.TRANSFORMING + ", " + VariantIndexStatus.LOADING + " or " + VariantIndexStatus.INDEXING + " and got " + index.getStatus(); - logger.warn(indexStatusMessage); + LOGGER.warn(indexStatusMessage); case VariantIndexStatus.READY: //Do not show warn message when index status is READY. indexStatusId = index.getStatus().getId(); break; @@ -413,7 +419,7 @@ private void updateFileInfo(String study, List filesToIndex, VariantReader if (jobFailed) { indexStatusMessage = "Job failed. Restoring status from " + VariantIndexStatus.TRANSFORMING + " to " + VariantIndexStatus.NONE; - logger.warn(indexStatusMessage); + LOGGER.warn(indexStatusMessage); indexStatusId = VariantIndexStatus.NONE; } else { indexStatusMessage = "Job finished. File transformed"; @@ -429,7 +435,7 @@ private void updateFileInfo(String study, List filesToIndex, VariantReader } indexStatusMessage = "Job failed. Restoring status from " + VariantIndexStatus.LOADING + " to " + indexStatusId; - logger.warn(indexStatusMessage); + LOGGER.warn(indexStatusMessage); } else { indexStatusMessage = "Job finished. File index ready"; indexStatusId = VariantIndexStatus.READY; @@ -448,7 +454,7 @@ private void updateFileInfo(String study, List filesToIndex, VariantReader } indexStatusMessage = "Job failed. Restoring status from " + VariantIndexStatus.INDEXING + " to " + indexStatusId; - logger.warn(indexStatusMessage); + LOGGER.warn(indexStatusMessage); } else { indexStatusId = VariantIndexStatus.READY; indexStatusMessage = "Job finished. File index ready"; @@ -480,7 +486,7 @@ private void updateFileInfo(String study, List filesToIndex, VariantReader DataResult queryResult = catalogManager.getCohortManager() .search(study, query, new QueryOptions(), sessionId); if (queryResult.getNumResults() != 0) { - logger.debug("Default cohort status set to READY"); + LOGGER.debug("Default cohort status set to READY"); Cohort defaultCohort = queryResult.first(); catalogManager.getCohortManager().setStatus(study, defaultCohort.getId(), CohortStatus.READY, null, sessionId); @@ -605,7 +611,7 @@ private List filterTransformFiles(List fileList, boolean resume) thr + "We can only transform VCF files not transformed, the status is " + indexStatus + ". " + "Do '" + VariantStorageOptions.RESUME.key() + "' to continue."; if (skipIndexedFiles) { - logger.warn(message); + LOGGER.warn(message); } else { throw new StorageEngineException(message); } @@ -619,7 +625,7 @@ private List filterTransformFiles(List fileList, boolean resume) thr filteredFiles.add(file); } else { if (skipIndexedFiles) { - logger.warn(msg); + LOGGER.warn(msg); } else { throw new StorageEngineException(msg); } @@ -627,7 +633,7 @@ private List filterTransformFiles(List fileList, boolean resume) thr break; } } else { - logger.warn("Skip file " + file.getName() + " with format " + file.getFormat() + " and status " + LOGGER.warn("Skip file " + file.getName() + " with format " + file.getFormat() + " and status " + file.getInternal().getStatus().getId()); } } @@ -679,7 +685,7 @@ private List filterLoadFiles(String studyFQN, List fileList, ObjectM } } transformed = file; - file = getOriginalFromTransformed(studyFQN, file, sessionId); + file = getOriginalFromTransformed(catalogManager, studyFQN, file, sessionId); } if (OperationManager.isVcfFormat(file)) { @@ -693,13 +699,13 @@ private List filterLoadFiles(String studyFQN, List fileList, ObjectM filteredFiles.add(file); fileUris.add(UriUtils.createUri(transformedFiles.get(i))); } else { - logger.warn("Cannot load vcf file " + file.getName() + " if no avro file is provided."); + LOGGER.warn("Cannot load vcf file " + file.getName() + " if no avro file is provided."); } break; case VariantIndexStatus.INDEXING: case VariantIndexStatus.LOADING: if (!resume) { - logger.warn("Unable to load this file. Already being loaded. Skipping file {}", file.getName()); + LOGGER.warn("Unable to load this file. Already being loaded. Skipping file {}", file.getName()); break; } case VariantIndexStatus.TRANSFORMED: @@ -713,17 +719,17 @@ private List filterLoadFiles(String studyFQN, List fileList, ObjectM filteredFiles.add(file); break; case VariantIndexStatus.TRANSFORMING: - logger.warn("We can only load files previously transformed. Skipping file {}", file.getName()); + LOGGER.warn("We can only load files previously transformed. Skipping file {}", file.getName()); break; case VariantIndexStatus.READY: - logger.warn("Already loaded file. Skipping file {}", file.getName()); + LOGGER.warn("Already loaded file. Skipping file {}", file.getName()); break; default: - logger.warn("We can only load files previously transformed, File {} with status is {}", file.getName(), status); + LOGGER.warn("We can only load files previously transformed, File {} with status is {}", file.getName(), status); break; } } else { - logger.warn("The input file is not a variant file. Format {}", file.getFormat()); + LOGGER.warn("The input file is not a variant file. Format {}", file.getFormat()); } } if (!transformedToOrigFileIdsMap.isEmpty()) { @@ -755,30 +761,30 @@ private List filterLoadFiles(String studyFQN, List fileList, ObjectM return filteredFiles; } - private File getOriginalFromTransformed(String study, File file, String sessionId) + private static File getOriginalFromTransformed(CatalogManager catalogManager, String study, File file, String token) throws CatalogException { // Look for the vcf file String vcfId = null; // Matchup variant files, if missing if (file.getRelatedFiles() == null || file.getRelatedFiles().isEmpty()) { - catalogManager.getFileManager().matchUpVariantFiles(study, Collections.singletonList(file), sessionId); + catalogManager.getFileManager().matchUpVariantFiles(study, Collections.singletonList(file), token); } for (FileRelatedFile relatedFile : file.getRelatedFiles()) { if (FileRelatedFile.Relation.PRODUCED_FROM.equals(relatedFile.getRelation())) { long fileUid = relatedFile.getFile().getUid(); // FIXME!!! vcfId = catalogManager.getFileManager().search(study, new Query(UID.key(), fileUid), - new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.ID.key()), sessionId).first().getId(); + new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.ID.key()), token).first().getId(); break; } } if (vcfId == null) { - logger.error("This code should never be executed. Every transformed avro file should come from a registered vcf file"); + LOGGER.error("This code should never be executed. Every transformed avro file should come from a registered vcf file"); throw new CatalogException("Internal error. No vcf file could be found for file " + file.getPath()); } - DataResult vcfDataResult = catalogManager.getFileManager().get(study, vcfId, FILE_GET_QUERY_OPTIONS, sessionId); + DataResult vcfDataResult = catalogManager.getFileManager().get(study, vcfId, FILE_GET_QUERY_OPTIONS, token); if (vcfDataResult.getNumResults() != 1) { - logger.error("This code should never be executed. No vcf file could be found for vcf id " + vcfId); + LOGGER.error("This code should never be executed. No vcf file could be found for vcf id " + vcfId); throw new CatalogException("Internal error. No vcf file could be found under id " + vcfId); } file = vcfDataResult.first(); @@ -790,7 +796,7 @@ private File getTransformedFromOriginal(String sessionId, File file) String transformedFileId = getTransformedFileIdFromOriginal(file); DataResult queryResult = catalogManager.getFileManager().get(studyFqn, transformedFileId, FILE_GET_QUERY_OPTIONS, sessionId); if (queryResult.getNumResults() != 1) { - logger.error("This code should never be executed. No transformed file could be found under "); + LOGGER.error("This code should never be executed. No transformed file could be found under "); throw new CatalogException("Internal error. No transformed file could be found under id " + transformedFileId); } @@ -803,7 +809,7 @@ private String getTransformedFileIdFromOriginal(File file) throws CatalogExcepti ? index.getTransform().getFileId() : null; if (StringUtils.isEmpty(transformedFileId)) { - logger.error("This code should never be executed. Every vcf file containing the transformed status should have" + LOGGER.error("This code should never be executed. Every vcf file containing the transformed status should have" + " a registered transformed file"); throw new CatalogException("Internal error. No transformed file could be found for file " + file.getUid()); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java index ebbc7b62143..3681260e3f9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java @@ -25,6 +25,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.job.ExecutionTime; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.response.OpenCGAResult; @@ -99,6 +100,9 @@ OpenCGAResult getAllInStudy(long studyId, QueryOptions options) */ OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) throws CatalogException; + OpenCGAResult executionTimeByMonth(Query query) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + enum QueryParams implements QueryParam { ID("id", TEXT, ""), UID("uid", LONG, ""), @@ -132,6 +136,7 @@ enum QueryParams implements QueryParam { OUTPUT("output", OBJECT, ""), DEPENDS_ON("dependsOn", TEXT_ARRAY, ""), TAGS("tags", TEXT_ARRAY, ""), + SCHEDULED_START_TIME("scheduledStartTime", TEXT, ""), EXECUTION("execution", OBJECT, ""), EXECUTION_START("execution.start", DATE, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java index 485ab7b10dd..6dc973eed70 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java @@ -92,6 +92,9 @@ default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) th OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException; + OpenCGAResult distinct(List field, Query query) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + enum QueryParams implements QueryParam { ID("id", TEXT, ""), UID("uid", LONG, ""), @@ -125,6 +128,7 @@ enum QueryParams implements QueryParam { INTERNAL_RGA_STATUS("internal.rga.status", TEXT, ""), INTERNAL_VARIANT("internal.variant", TEXT_ARRAY, ""), INTERNAL_VARIANT_INDEX("internal.variant.index", TEXT_ARRAY, ""), + INTERNAL_VARIANT_INDEX_STATUS_ID("internal.variant.index.status.id", STRING, ""), @Deprecated INTERNAL_VARIANT_GENOTYPE_INDEX("internal.variant.sampleGenotypeIndex", TEXT_ARRAY, ""), INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX("internal.variant.secondarySampleIndex", TEXT_ARRAY, ""), INTERNAL_VARIANT_ANNOTATION_INDEX("internal.variant.annotationIndex", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index d89e0020f93..b0b9aea3e40 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -17,7 +17,9 @@ package org.opencb.opencga.catalog.db.mongodb; import com.mongodb.client.ClientSession; +import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; @@ -44,10 +46,7 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; -import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.job.JobInternalWebhook; -import org.opencb.opencga.core.models.job.JobPermissions; -import org.opencb.opencga.core.models.job.ToolInfo; +import org.opencb.opencga.core.models.job.*; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; @@ -294,6 +293,45 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Job job, Object return endWrite(tmpStartTime, 1, 1, events); } + @Override + public OpenCGAResult executionTimeByMonth(Query query) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + long startTime = startQuery(); + + Bson bsonQuery = parseQuery(query, QueryOptions.empty()); + List aggregation = new ArrayList<>(); + aggregation.add(Aggregates.match(bsonQuery)); + aggregation.add(Aggregates.project(Projections.fields( + Projections.computed("date", "$" + PRIVATE_MODIFICATION_DATE), + Projections.computed("difference", new Document("$toDouble", + new Document("$subtract", + Arrays.asList("$" + QueryParams.EXECUTION_END.key(), "$" + QueryParams.EXECUTION_START.key())))) + ))); + aggregation.add(new Document("$group", new Document() + .append("_id", new Document() + .append("month", new Document("$month", "$date")) + .append("year", new Document("$year", "$date"))) + .append("sum", new Document("$sum", "$difference")))); + + DataResult aggregate = jobCollection.aggregate(aggregation, QueryOptions.empty()); + // Result comes in this format: + // { "_id" : { "month" : 5, "year" : 2024 }, "sum" : 13196 } + + // Parse result + List executionTimeList = new ArrayList<>(aggregate.getNumResults()); + for (Document result : aggregate.getResults()) { + Document id = result.get("_id", Document.class); + String month = id.getInteger("month").toString(); + String year = id.getInteger("year").toString(); + double seconds = result.get("sum", Number.class).doubleValue() / 1000; // convert milliseconds to seconds + double minutes = seconds / 60.0; + double hours = minutes / 60.0; + ExecutionTime.Time time = new ExecutionTime.Time(hours, minutes, seconds); + executionTimeList.add(new ExecutionTime(month, year, time)); + } + return endQuery(startTime, executionTimeList); + } + @Override public OpenCGAResult delete(Job job) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { @@ -369,7 +407,8 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document jobDoc private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryOptions options) throws CatalogDBException { UpdateDocument document = new UpdateDocument(); - String[] acceptedParams = {QueryParams.USER_ID.key(), QueryParams.DESCRIPTION.key(), QueryParams.COMMAND_LINE.key()}; + String[] acceptedParams = {QueryParams.USER_ID.key(), QueryParams.DESCRIPTION.key(), QueryParams.COMMAND_LINE.key(), + QueryParams.SCHEDULED_START_TIME.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); String[] acceptedBooleanParams = {QueryParams.VISITED.key(), QueryParams.INTERNAL_KILL_JOB_REQUESTED.key()}; @@ -889,7 +928,7 @@ private Bson parseQuery(Query query, Document extraQuery, QueryOptions options, // case END_TIME: // case OUTPUT_ERROR: // case EXECUTION_START: -// case EXECUTION_END: + case EXECUTION_END: // case COMMAND_LINE: case VISITED: case RELEASE: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index d9058193e7d..ff0189056d7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -1163,7 +1163,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu } @Override - public OpenCGAResult distinct(long studyUid, String field, Query query, String userId) + public OpenCGAResult distinct(long studyUid, String field, Query query, String userId) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Query finalQuery = query != null ? new Query(query) : new Query(); finalQuery.put(QueryParams.STUDY_UID.key(), studyUid); @@ -1171,6 +1171,22 @@ public OpenCGAResult distinct(long studyUid, String field, Query query, String u return new OpenCGAResult<>(sampleCollection.distinct(field, bson)); } + @Override + public OpenCGAResult distinct(List fields, Query query) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + StopWatch stopWatch = StopWatch.createStarted(); + Query finalQuery = query != null ? new Query(query) : new Query(); + Bson bson = parseQuery(finalQuery); + + Set results = new LinkedHashSet<>(); + for (String field : fields) { + results.addAll(sampleCollection.distinct(field, bson, String.class).getResults()); + } + + return new OpenCGAResult<>((int) stopWatch.getTime(TimeUnit.MILLISECONDS), Collections.emptyList(), results.size(), + new ArrayList<>(results), -1); + } + @Override public OpenCGAResult distinct(long studyUid, List fields, Query query, String userId) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java index fe7ffc18e44..dad603843ec 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java @@ -30,13 +30,12 @@ import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogIOException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.io.IOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.models.InternalGetDataResult; import org.opencb.opencga.catalog.utils.CatalogFqn; +import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.FieldConstants; @@ -572,9 +571,11 @@ public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority job.setAttributes(attributes); try { autoCompleteNewJob(organizationId, study, job, tokenPayload); - authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.EXECUTE_JOBS); + // Check if we have already reached the limit of job hours in the Organisation + checkExecutionLimitQuota(organizationId); + // Check params ParamUtils.checkObj(params, "params"); for (Map.Entry entry : params.entrySet()) { @@ -613,6 +614,115 @@ public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority } } + /** + * Method to reschedule a list of jobs. Only intended for "Opencga" administrators. + * @param studyStr study where the list of jobs belong. + * @param jobUids List of job uids to be rescheduled. + * @param scheduledStartTime New scheduled start time. + * @param token OpenCGA token. + * @return OpenCGAResult with the list of jobs that have been rescheduled. + * @throws CatalogException if there is any error. + */ + public OpenCGAResult rescheduleJobs(String studyStr, List jobUids, String scheduledStartTime, String token) + throws CatalogException { + return rescheduleJobs(studyStr, jobUids, scheduledStartTime, null, token); + } + + /** + * Method to reschedule a list of jobs. Only intended for "Opencga" administrators. + * @param studyStr study where the list of jobs belong. + * @param jobUids List of job uids to be rescheduled. + * @param scheduledStartTime New scheduled start time. + * @param eventMessage Event message to be added to the rescheduled jobs. + * @param token OpenCGA token. + * @return OpenCGAResult with the list of jobs that have been rescheduled. + * @throws CatalogException if there is any error. + */ + public OpenCGAResult rescheduleJobs(String studyStr, List jobUids, String scheduledStartTime, String eventMessage, + String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + + ObjectMap auditParams = new ObjectMap() + .append("study", studyStr) + .append("jobUids", jobUids) + .append("scheduledStartTime", scheduledStartTime) + .append("eventMessage", eventMessage) + .append("token", token); + try { + Study study = catalogManager.getStudyManager().resolveId(studyFqn, tokenPayload); + authorizationManager.checkIsOpencgaAdministrator(tokenPayload); + + if (CollectionUtils.isEmpty(jobUids)) { + throw new CatalogException("Missing job uids"); + } + if (StringUtils.isEmpty(scheduledStartTime)) { + throw new CatalogException("Missing scheduled start time"); + } + ParamUtils.checkDateIsNotExpired(scheduledStartTime, JobDBAdaptor.QueryParams.SCHEDULED_START_TIME.key()); + + ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.SCHEDULED_START_TIME.key(), scheduledStartTime); + QueryOptions queryOptions = new QueryOptions(); + + if (StringUtils.isNotEmpty(eventMessage)) { + // Add event message + List eventList = Collections.singletonList(new Event(Event.Type.INFO, "reschedule", eventMessage)); + params.append(JobDBAdaptor.QueryParams.INTERNAL_EVENTS.key(), eventList); + + Map actionMap = new HashMap<>(); + actionMap.put(JobDBAdaptor.QueryParams.INTERNAL_EVENTS.key(), ParamUtils.BasicUpdateAction.ADD); + queryOptions.getMap(Constants.ACTIONS, actionMap); + } + + Query query = new Query() + .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) + .append(JobDBAdaptor.QueryParams.UID.key(), jobUids); + OpenCGAResult update = getJobDBAdaptor(organizationId).update(query, params, queryOptions); + + auditManager.audit(organizationId, userId, Enums.Action.RESCHEDULE_JOB, Enums.Resource.JOB, "", "", "", + "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + return update; + } catch (Exception e) { + auditManager.audit(organizationId, userId, Enums.Action.RESCHEDULE_JOB, Enums.Resource.JOB, "", "", "", + "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); + throw e; + } + } + + public OpenCGAResult getExecutionTimeByMonth(String organizationId, Query query, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + ParamUtils.checkParameter(organizationId, "organizationId"); + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, tokenPayload.getUserId(organizationId)); + return getJobDBAdaptor(organizationId).executionTimeByMonth(query); + } + + private void checkExecutionLimitQuota(String organizationId) throws CatalogException { + if (exceedsExecutionLimitQuota(organizationId)) { + throw new CatalogException("The organization '" + organizationId + "' has reached the maximum quota of execution hours (" + + configuration.getQuota().getMaxNumJobHours() + ") for the current month."); + } + } + + public boolean exceedsExecutionLimitQuota(String organizationId) throws CatalogException { + if (configuration.getQuota().getMaxNumJobHours() <= 0) { + return false; + } + // Get current year/month + String time = TimeUtils.getTime(TimeUtils.getDate(), TimeUtils.yyyyMM); + Query query = new Query(JobDBAdaptor.QueryParams.EXECUTION_END.key(), time); + OpenCGAResult result = getJobDBAdaptor(organizationId).executionTimeByMonth(query); + if (result.getNumResults() > 0) { + ExecutionTime executionTime = result.first(); + if (executionTime.getTime().getHours() >= configuration.getQuota().getMaxNumJobHours()) { + return true; + } + } + return false; + } + /** * Check if the job is eligible to be reused, and if so, try to find an equivalent job. * Eligible job: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index 92591852817..7cb92deec37 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -168,6 +168,9 @@ public OpenCGAResult create(ProjectCreateParams projectCreateParams, Qu authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); ParamUtils.checkObj(projectCreateParams, "ProjectCreateParams"); project = projectCreateParams.toProject(); + + // Check if we have already reached the limit of projects in the Organisation + checkProjectLimitQuota(organizationId); validateProjectForCreation(organizationId, project); queryResult = getProjectDBAdaptor(organizationId).insert(project, options); @@ -202,6 +205,18 @@ public OpenCGAResult create(ProjectCreateParams projectCreateParams, Qu return queryResult; } + private void checkProjectLimitQuota(String organizationId) throws CatalogException { + if (configuration.getQuota().getMaxNumProjects() <= 0) { + logger.debug("No project limit quota set. Skipping quota check."); + return; + } + long numProjects = getProjectDBAdaptor(organizationId).count(new Query()).getNumMatches(); + if (numProjects >= configuration.getQuota().getMaxNumProjects()) { + throw new CatalogException("The organization '" + organizationId + "' has reached the maximum quota of projects (" + + configuration.getQuota().getMaxNumProjects() + ")."); + } + } + private void validateProjectForCreation(String organizationId, Project project) throws CatalogParameterException { ParamUtils.checkParameter(project.getId(), ProjectDBAdaptor.QueryParams.ID.key()); project.setName(ParamUtils.defaultString(project.getName(), project.getId())); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ResourceManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ResourceManager.java index 9584a4a7301..8c271faaf76 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ResourceManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ResourceManager.java @@ -226,17 +226,15 @@ public abstract OpenCGAResult search(String studyId, Query query, QueryOption /** * Fetch a list containing all the distinct values of the key {@code field}. * - * @param organizationId Organization id. - * @param studyId study id in string format. Could be one of - * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] - * @param field The field for which to return distinct values. - * @param query Query object. - * @param token Token of the user logged in. + * @param studyId study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param field The field for which to return distinct values. + * @param query Query object. + * @param token Token of the user logged in. * @return The list of distinct values. * @throws CatalogException CatalogException. */ - public OpenCGAResult distinct(String organizationId, String studyId, String field, Query query, String token) - throws CatalogException { + public OpenCGAResult distinct(String studyId, String field, Query query, String token) throws CatalogException { return distinct(studyId, Collections.singletonList(field), query, token); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java index 48976806622..bc216fc7082 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java @@ -325,6 +325,48 @@ public OpenCGAResult search(String studyStr, Query query, QueryOptions o } } + public OpenCGAResult distinct(String field, Query query, String token) throws CatalogException { + return distinct(Collections.singletonList(field), query, token); + } + + /** + * Fetch a list containing all the distinct values of the key {@code field}. + * Query object may or may not contain a study parameter, so only organization administrators will be able to call to this method. + * + * @param fields Fields for which to return distinct values. + * @param query Query object. + * @param token Token of the user logged in. + * @return The list of distinct values. + * @throws CatalogException CatalogException. + */ + public OpenCGAResult distinct(List fields, Query query, String token) throws CatalogException { + query = ParamUtils.defaultObject(query, Query::new); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + String userId = tokenPayload.getUserId(organizationId); + + ObjectMap auditParams = new ObjectMap() + .append("fields", fields) + .append("query", new Query(query)) + .append("token", token); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); + fixQueryObject(organizationId, null, query, userId); + + OpenCGAResult result = getSampleDBAdaptor(organizationId).distinct(fields, query); + + auditManager.auditDistinct(organizationId, userId, Enums.Resource.SAMPLE, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + return result; + } catch (CatalogException e) { + auditManager.auditDistinct(organizationId, userId, Enums.Resource.SAMPLE, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + } + @Override public OpenCGAResult distinct(String studyStr, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); @@ -359,6 +401,8 @@ public OpenCGAResult distinct(String studyStr, List fields, Query que } } + + void fixQueryObject(String organizationId, Study study, Query query, String userId) throws CatalogException { changeQueryId(query, ParamConstants.SAMPLE_RGA_STATUS_PARAM, SampleDBAdaptor.QueryParams.INTERNAL_RGA_STATUS.key()); changeQueryId(query, ParamConstants.SAMPLE_PROCESSING_PREPARATION_METHOD_PARAM, diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index 70f933f6430..9dd1265c7f1 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -170,6 +170,10 @@ Study resolveId(String studyStr, QueryOptions options, String userId, String org return studyDataResult.first(); } + Study resolveId(CatalogFqn catalogFqn, JwtPayload payload) throws CatalogException { + return resolveId(catalogFqn, QueryOptions.empty(), payload); + } + Study resolveId(CatalogFqn catalogFqn, QueryOptions options, JwtPayload payload) throws CatalogException { OpenCGAResult studyDataResult = smartResolutor(catalogFqn, options, payload); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index 7e8e8cdffeb..4018de78ff0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -194,6 +194,10 @@ public OpenCGAResult create(User user, String password, String token) thro } } + // Check if we have already reached the limit of users in the Organisation + checkUserLimitQuota(organizationId); + + // Check if the user already exists checkUserExists(organizationId, user.getId()); try { @@ -218,6 +222,19 @@ public OpenCGAResult create(User user, String password, String token) thro } } + private void checkUserLimitQuota(String organizationId) throws CatalogException { + if (configuration.getQuota().getMaxNumUsers() <= 0) { + logger.debug("Skipping user quota check. No limit set."); + return; + } + // Check if we have already reached the limit of users in the Organisation + long numUsers = getUserDBAdaptor(organizationId).count(new Query()).getNumMatches(); + if (numUsers >= configuration.getQuota().getMaxNumUsers()) { + throw new CatalogException("The organization '" + organizationId + "' has reached the maximum quota of allowed users (" + + configuration.getQuota().getMaxNumUsers() + ")."); + } + } + /** * Create a new user. * diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 2beb12d1c2d..28b2fd5a7fb 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -108,6 +108,7 @@ {"collections": ["sample", "sample_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"internal.variant.index.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"internal.rga.status": 1, "studyUid": 1}, "options": {}} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java index dc857e15965..e4b7bca3aab 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java @@ -30,6 +30,8 @@ import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; +import org.opencb.opencga.catalog.db.api.JobDBAdaptor; +import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; @@ -481,8 +483,19 @@ protected CatalogManager mockCatalogManager() throws CatalogDBException { UserManager userManager = spy.getUserManager(); UserManager userManagerSpy = Mockito.spy(userManager); Mockito.doReturn(userManagerSpy).when(spy).getUserManager(); - MongoDBAdaptorFactory mongoDBAdaptorFactory = mockMongoDBAdaptorFactory(); - Mockito.doReturn(mongoDBAdaptorFactory).when(userManagerSpy).getCatalogDBAdaptorFactory(); + MongoDBAdaptorFactory mockMongoFactory = mockMongoDBAdaptorFactory(); + Mockito.doReturn(mockMongoFactory).when(userManagerSpy).getCatalogDBAdaptorFactory(); + + JobManager jobManager = spy.getJobManager(); + JobManager jobManagerSpy = Mockito.spy(jobManager); + Mockito.doReturn(jobManagerSpy).when(spy).getJobManager(); + Mockito.doReturn(mockMongoFactory).when(jobManagerSpy).getCatalogDBAdaptorFactory(); + + ProjectManager projectManager = spy.getProjectManager(); + ProjectManager projectManagerSpy = Mockito.spy(projectManager); + Mockito.doReturn(projectManagerSpy).when(spy).getProjectManager(); + Mockito.doReturn(mockMongoFactory).when(projectManagerSpy).getCatalogDBAdaptorFactory(); + return spy; } @@ -492,6 +505,15 @@ protected MongoDBAdaptorFactory mockMongoDBAdaptorFactory() throws CatalogDBExce UserDBAdaptor userDBAdaptor = dbAdaptorFactorySpy.getCatalogUserDBAdaptor(organizationId); UserDBAdaptor userDBAdaptorSpy = Mockito.spy(userDBAdaptor); Mockito.doReturn(userDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogUserDBAdaptor(organizationId); + + JobDBAdaptor jobDBAdaptor = dbAdaptorFactorySpy.getCatalogJobDBAdaptor(organizationId); + JobDBAdaptor jobDBAdaptorSpy = Mockito.spy(jobDBAdaptor); + Mockito.doReturn(jobDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogJobDBAdaptor(organizationId); + + ProjectDBAdaptor projectDBAdaptor = dbAdaptorFactorySpy.getCatalogProjectDbAdaptor(organizationId); + ProjectDBAdaptor projectDBAdaptorSpy = Mockito.spy(projectDBAdaptor); + Mockito.doReturn(projectDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogProjectDbAdaptor(organizationId); + return dbAdaptorFactorySpy; } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java index 8f0bce1401e..4145af76dda 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.time.StopWatch; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.mockito.Mockito; import org.opencb.biodata.models.common.Status; import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.DataResult; @@ -970,6 +971,57 @@ public void visitJob() throws CatalogException { assertEquals(1, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches()); } + @Test + public void testJobQuotaLimit() throws CatalogException { + // Submit a dummy job. This shouldn't raise any error + catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken); + + OpenCGAResult result = catalogManager.getJobManager().getExecutionTimeByMonth(organizationId, new Query(), ownerToken); + assertEquals(1, result.getNumResults()); + assertEquals(0, result.first().getTime().getHours(), 0.0); + assertEquals(0, result.first().getTime().getMinutes(), 0.0); + assertEquals(0, result.first().getTime().getSeconds(), 0.0); + + try (CatalogManager mockManager = mockCatalogManager()) { + // Mock check result + OpenCGAResult results = new OpenCGAResult<>(0, Collections.singletonList(new ExecutionTime("1", "2024", + new ExecutionTime.Time(1000.0, 1000 * 60.0, 1000.0 * 60 * 60)))); + JobDBAdaptor jobDBAdaptor = mockManager.getJobManager().getJobDBAdaptor(organizationId); + + Mockito.doReturn(results).when(jobDBAdaptor).executionTimeByMonth(Mockito.any(Query.class)); + + // Submit a job. This should raise an error + CatalogException exception = assertThrows(CatalogException.class, () -> mockManager.getJobManager() + .submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken)); + assertTrue(exception.getMessage().contains("quota")); + } + } + + @Test + public void rescheduleJobTest() throws CatalogException { + Job job = catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken).first(); + + Date firstDayOfNextMonth = TimeUtils.getFirstDayOfNextMonth(new Date()); + String scheduleStartTime = TimeUtils.getTime(firstDayOfNextMonth); + catalogManager.getJobManager().rescheduleJobs(studyFqn, Collections.singletonList(job.getUid()), scheduleStartTime, "My message", + opencgaToken); + + OpenCGAResult result = catalogManager.getJobManager().get(studyFqn, job.getId(), QueryOptions.empty(), ownerToken); + assertEquals(1, result.getNumResults()); + assertEquals(scheduleStartTime, result.first().getScheduledStartTime()); + assertEquals(1, result.first().getInternal().getEvents().size()); + assertEquals("My message", result.first().getInternal().getEvents().get(0).getMessage()); + + // Ensure only "opencga" are authorised + assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getJobManager().rescheduleJobs(studyFqn, Collections.singletonList(job.getUid()), scheduleStartTime, + "My message", ownerToken)); + CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getJobManager().rescheduleJobs(studyFqn, Collections.singletonList(job.getUid()), scheduleStartTime, + "My message", normalToken1)); + assertTrue(authException.getMessage().contains("OPENCGA ADMINISTRATOR")); + } + /** * VariableSet methods *************************** */ diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java index 670ead3493e..c7b57488fab 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java @@ -997,7 +997,7 @@ public void disordersDistinctTest() throws CatalogException { catalogManager.getIndividualManager().update(studyFqn, "child1", params1, new QueryOptions(), ownerToken); catalogManager.getIndividualManager().update(studyFqn, "child2", params2, new QueryOptions(), ownerToken); - OpenCGAResult distinct = catalogManager.getFamilyManager().distinct(organizationId, studyFqn, "disorders.name", new Query(), ownerToken); + OpenCGAResult distinct = catalogManager.getFamilyManager().distinct(studyFqn, "disorders.name", new Query(), ownerToken); System.out.println(distinct); assertEquals(2, distinct.getNumResults()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java index 8755cc50c4a..2b1de95a458 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java @@ -110,11 +110,11 @@ public void testDistinctDisorders() throws CatalogException { .setDisorders(Collections.singletonList(new Disorder().setId("adisorder2"))); catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); - OpenCGAResult result = catalogManager.getIndividualManager().distinct(organizationId, studyFqn, + OpenCGAResult result = catalogManager.getIndividualManager().distinct(studyFqn, IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), new Query(), ownerToken); assertEquals(3, result.getNumResults()); - result = catalogManager.getIndividualManager().distinct(organizationId, studyFqn, IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), + result = catalogManager.getIndividualManager().distinct(studyFqn, IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), new Query(IndividualDBAdaptor.QueryParams.DISORDERS.key(), "~^disor"), ownerToken); assertEquals(2, result.getNumResults()); @@ -142,7 +142,7 @@ public void testSearchDisordersWithCommas() throws CatalogException { .setDisorders(Collections.singletonList(new Disorder().setId("disorder2"))); catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); - OpenCGAResult result = catalogManager.getIndividualManager().distinct(organizationId, studyFqn, + OpenCGAResult result = catalogManager.getIndividualManager().distinct(studyFqn, IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), new Query(), ownerToken); assertEquals(3, result.getNumResults()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java index 85e13914da5..1862db0db7d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java @@ -20,6 +20,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.mockito.Mockito; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; @@ -52,6 +53,27 @@ @Category(MediumTests.class) public class ProjectManagerTest extends AbstractManagerTest { + @Test + public void createProjectQuotaTest() throws CatalogException { + catalogManager.getConfiguration().getQuota().setMaxNumProjects(5); + try (CatalogManager mockCatalogManager = mockCatalogManager()) { + ProjectDBAdaptor projectDBAdaptor = mockCatalogManager.getProjectManager().getProjectDBAdaptor(organizationId); + + // Mock there already exists 50 projects + OpenCGAResult result = new OpenCGAResult<>(0, Collections.emptyList()); + result.setNumMatches(50); + Mockito.doReturn(result).when(projectDBAdaptor).count(); + Mockito.doReturn(result).when(projectDBAdaptor).count(Mockito.any(Query.class)); + + ProjectCreateParams projectCreateParams = new ProjectCreateParams() + .setId("newProject") + .setName("Project about some genomes"); + CatalogException exception = assertThrows(CatalogException.class, + () -> mockCatalogManager.getProjectManager().create(projectCreateParams, QueryOptions.empty(), ownerToken)); + assertTrue(exception.getMessage().contains("quota")); + } + } + @Test public void searchProjectByStudy() throws CatalogException { OpenCGAResult result = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), "phase1"), null, ownerToken); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java index 69042379f29..af44b05cc1d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java @@ -681,7 +681,7 @@ public void updateQualityControlTest2() throws CatalogException { @Test public void distinctTest() throws CatalogException { - OpenCGAResult distinct = catalogManager.getSampleManager().distinct(organizationId, studyFqn, SampleDBAdaptor.QueryParams.ID.key(), null, ownerToken); + OpenCGAResult distinct = catalogManager.getSampleManager().distinct(studyFqn, SampleDBAdaptor.QueryParams.ID.key(), null, ownerToken); assertEquals(String.class.getName(), distinct.getResultType()); assertEquals(9, distinct.getNumResults()); assertEquals(9, distinct.getResults().size()); @@ -692,12 +692,12 @@ public void distinctTest() throws CatalogException { assertEquals(18, distinct.getNumResults()); assertEquals(18, distinct.getResults().size()); - distinct = catalogManager.getSampleManager().distinct(organizationId, studyFqn, SampleDBAdaptor.QueryParams.UID.key(), null, ownerToken); + distinct = catalogManager.getSampleManager().distinct(studyFqn, SampleDBAdaptor.QueryParams.UID.key(), null, ownerToken); assertEquals(Long.class.getName(), distinct.getResultType()); assertEquals(9, distinct.getNumResults()); assertEquals(9, distinct.getResults().size()); - distinct = catalogManager.getSampleManager().distinct(organizationId, studyFqn, SampleDBAdaptor.QueryParams.SOMATIC.key(), null, ownerToken); + distinct = catalogManager.getSampleManager().distinct(studyFqn, SampleDBAdaptor.QueryParams.SOMATIC.key(), null, ownerToken); assertEquals(Boolean.class.getName(), distinct.getResultType()); assertEquals(1, distinct.getNumResults()); assertEquals(1, distinct.getResults().size()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java index f23295fbd60..6f3e507b726 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java @@ -40,7 +40,6 @@ import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; @Category(MediumTests.class) @@ -507,6 +506,25 @@ public void loginUserPasswordExpiredTest() throws CatalogException { } } + @Test + public void createUserQuotaTest() throws CatalogException { + catalogManager.getConfiguration().getQuota().setMaxNumUsers(15); + try (CatalogManager mockCatalogManager = mockCatalogManager()) { + UserDBAdaptor userDBAdaptor = mockCatalogManager.getUserManager().getUserDBAdaptor(organizationId); + + // Mock there already exists 50 users + OpenCGAResult result = new OpenCGAResult<>(0, Collections.emptyList()); + result.setNumMatches(50); + Mockito.doReturn(result).when(userDBAdaptor).count(); + Mockito.doReturn(result).when(userDBAdaptor).count(Mockito.any(Query.class)); + + User user = new User("newUser"); + CatalogException exception = assertThrows(CatalogException.class, + () -> mockCatalogManager.getUserManager().create(user, TestParamConstants.PASSWORD, ownerToken)); + assertTrue(exception.getMessage().contains("quota")); + } + } + @Test public void changePasswordTest() throws CatalogException { String newPassword = PasswordUtils.getStrongRandomPassword(); diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index d0ebcf32ba6..4dd567c2911 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -60,6 +60,12 @@ monitor: fileDaemonInterval: 8000 # number of milliseconds between checks port: 9092 +quota: # Quotas per organisation + maxNumUsers: 0 # Maximum number of users that an organisation can have + maxNumProjects: 0 # Maximum number of projects that an organisation can have + maxNumVariantIndexSamples: 15000 # Maximum number of variant index samples that an organisation can have + maxNumJobHours: 100 # Maximum number of hours that the organisation can use in jobs on a monthly basis + email: host: "localhost" port: "" diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java index c0574f0afdf..41cdbffbffb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java @@ -30,9 +30,10 @@ public class TimeUtils { - private static final String yyyyMMdd = "yyyyMMdd"; - private static final String yyyyMMddHHmmss = "yyyyMMddHHmmss"; - private static final String yyyyMMddHHmmssSSS = "yyyyMMddHHmmssSSS"; + public static final String yyyyMM = "yyyyMM"; + public static final String yyyyMMdd = "yyyyMMdd"; + public static final String yyyyMMddHHmmss = "yyyyMMddHHmmss"; + public static final String yyyyMMddHHmmssSSS = "yyyyMMddHHmmssSSS"; private static final Logger logger = LoggerFactory.getLogger(TimeUtils.class); @@ -49,6 +50,11 @@ public static String getTime(Date date) { return sdf.format(date); } + public static String getTime(Date date, String format) { + SimpleDateFormat sdf = new SimpleDateFormat(format); + return sdf.format(date); + } + public static String getTimeMillis() { return getTimeMillis(new Date()); } @@ -119,6 +125,19 @@ public static Date add24HtoDate(Date date) { return new Date(cal.getTimeInMillis()); } + public static Date getFirstDayOfNextMonth(Date date) { + Calendar cal = new GregorianCalendar(); + cal.setTime(date); + cal.setTimeInMillis(date.getTime()); + cal.add(Calendar.MONTH, 1); + cal.set(Calendar.DAY_OF_MONTH, 1); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 1); + return new Date(cal.getTimeInMillis()); + } + public static Date add1MonthtoDate(Date date) { Calendar cal = new GregorianCalendar(); cal.setTime(date); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index b7b869c728c..5ae858227b6 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -45,6 +45,7 @@ public class Configuration { private String jobDir; private AccountConfiguration account; + private QuotaConfiguration quota; private Monitor monitor; private HealthCheck healthCheck; @@ -80,6 +81,7 @@ public Configuration() { panel = new Panel(); server = new ServerConfiguration(); account = new AccountConfiguration(); + quota = QuotaConfiguration.init(); } public void serialize(OutputStream configurationOututStream) throws IOException { @@ -132,6 +134,9 @@ private static void addDefaultValueIfMissing(Configuration configuration) { // Disable password expiration by default configuration.getAccount().setPasswordExpirationDays(0); } + if (configuration.getQuota() == null) { + configuration.setQuota(QuotaConfiguration.init()); + } } private static void overwriteWithEnvironmentVariables(Configuration configuration) { @@ -230,6 +235,7 @@ public String toString() { sb.append(", workspace='").append(workspace).append('\''); sb.append(", jobDir='").append(jobDir).append('\''); sb.append(", account=").append(account); + sb.append(", quota=").append(quota); sb.append(", monitor=").append(monitor); sb.append(", healthCheck=").append(healthCheck); sb.append(", audit=").append(audit); @@ -319,6 +325,15 @@ public Configuration setAccount(AccountConfiguration account) { return this; } + public QuotaConfiguration getQuota() { + return quota; + } + + public Configuration setQuota(QuotaConfiguration quota) { + this.quota = quota; + return this; + } + @Deprecated public int getMaxLoginAttempts() { return account.getMaxLoginAttempts(); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/QuotaConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/QuotaConfiguration.java new file mode 100644 index 00000000000..c3e3d30bf41 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/QuotaConfiguration.java @@ -0,0 +1,70 @@ +package org.opencb.opencga.core.config; + +public class QuotaConfiguration { + + private int maxNumUsers; + private int maxNumProjects; + private int maxNumVariantIndexSamples; + private int maxNumJobHours; + + public QuotaConfiguration() { + } + + public QuotaConfiguration(int maxNumUsers, int maxNumProjects, int maxNumVariantIndexSamples, int maxNumJobHours) { + this.maxNumUsers = maxNumUsers; + this.maxNumProjects = maxNumProjects; + this.maxNumVariantIndexSamples = maxNumVariantIndexSamples; + this.maxNumJobHours = maxNumJobHours; + } + + public static QuotaConfiguration init() { + return new QuotaConfiguration(0, 0, 0, 0); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("QuotaConfiguration{"); + sb.append("maxNumUsers=").append(maxNumUsers); + sb.append(", maxNumProjects=").append(maxNumProjects); + sb.append(", maxNumVariantIndexSamples=").append(maxNumVariantIndexSamples); + sb.append(", maxNumJobHours=").append(maxNumJobHours); + sb.append('}'); + return sb.toString(); + } + + public int getMaxNumUsers() { + return maxNumUsers; + } + + public QuotaConfiguration setMaxNumUsers(int maxNumUsers) { + this.maxNumUsers = maxNumUsers; + return this; + } + + public int getMaxNumProjects() { + return maxNumProjects; + } + + public QuotaConfiguration setMaxNumProjects(int maxNumProjects) { + this.maxNumProjects = maxNumProjects; + return this; + } + + public int getMaxNumVariantIndexSamples() { + return maxNumVariantIndexSamples; + } + + public QuotaConfiguration setMaxNumVariantIndexSamples(int maxNumVariantIndexSamples) { + this.maxNumVariantIndexSamples = maxNumVariantIndexSamples; + return this; + } + + public int getMaxNumJobHours() { + return maxNumJobHours; + } + + public QuotaConfiguration setMaxNumJobHours(int maxNumJobHours) { + this.maxNumJobHours = maxNumJobHours; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java index 2777222715b..e70421a0712 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java @@ -251,6 +251,7 @@ public enum Action { VISIT, KILL_JOB, + RESCHEDULE_JOB, IMPORT, diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ExecutionTime.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ExecutionTime.java new file mode 100644 index 00000000000..170ef4f72b3 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ExecutionTime.java @@ -0,0 +1,107 @@ +package org.opencb.opencga.core.models.job; + +public class ExecutionTime { + + private String month; + private String year; + private Time time; + + public ExecutionTime() { + } + + public ExecutionTime(String month, String year, Time time) { + this.month = month; + this.year = year; + this.time = time; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ExecutionTime{"); + sb.append("month='").append(month).append('\''); + sb.append(", year='").append(year).append('\''); + sb.append(", time=").append(time); + sb.append('}'); + return sb.toString(); + } + + public String getMonth() { + return month; + } + + public ExecutionTime setMonth(String month) { + this.month = month; + return this; + } + + public String getYear() { + return year; + } + + public ExecutionTime setYear(String year) { + this.year = year; + return this; + } + + public Time getTime() { + return time; + } + + public ExecutionTime setTime(Time time) { + this.time = time; + return this; + } + + public static class Time { + private double hours; + private double minutes; + private double seconds; + + public Time() { + } + + public Time(double hours, double minutes, double seconds) { + this.hours = hours; + this.minutes = minutes; + this.seconds = seconds; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Time{"); + sb.append("hours=").append(hours); + sb.append(", minutes=").append(minutes); + sb.append(", seconds=").append(seconds); + sb.append('}'); + return sb.toString(); + } + + public double getHours() { + return hours; + } + + public Time setHours(double hours) { + this.hours = hours; + return this; + } + + public double getMinutes() { + return minutes; + } + + public Time setMinutes(double minutes) { + this.minutes = minutes; + return this; + } + + public double getSeconds() { + return seconds; + } + + public Time setSeconds(double seconds) { + this.seconds = seconds; + return this; + } + } + +} diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 4eb3543f791..f18e25a580e 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -43,6 +43,12 @@ server: grpc: port: ${OPENCGA.SERVER.GRPC.PORT} +quota: # Quotas per organisation + maxNumUsers: 15 # Maximum number of users that an organisation can have + maxNumProjects: 5 # Maximum number of projects that an organisation can have + maxNumVariantIndexSamples: 15000 # Maximum number of variant index samples that an organisation can have + maxNumJobHours: 100 # Maximum number of hours that the organisation can use in jobs on a monthly basis + audit: manager: "" # Java manager of the audit implementation to be used to audit. If empty, catalog database will be used. maxDocuments: 20000000 # Maximum number of documents that will be created in the audit collection. diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index 01c02bde4ad..33e77c32a36 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -51,6 +51,7 @@ import org.opencb.opencga.analysis.variant.inferredSex.InferredSexAnalysis; import org.opencb.opencga.analysis.variant.julie.JulieTool; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; +import org.opencb.opencga.analysis.variant.manager.operations.VariantFileIndexerOperationManager; import org.opencb.opencga.analysis.variant.mendelianError.MendelianErrorAnalysis; import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis; import org.opencb.opencga.analysis.variant.operations.*; @@ -70,10 +71,7 @@ import org.opencb.opencga.analysis.wrappers.rvtests.RvtestsWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.samtools.SamtoolsWrapperAnalysis; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; -import org.opencb.opencga.catalog.db.api.DBIterator; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.db.api.JobDBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -85,6 +83,7 @@ import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.ExceptionUtils; @@ -94,6 +93,7 @@ import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileAclParams; import org.opencb.opencga.core.models.file.FilePermissions; @@ -557,6 +557,13 @@ protected int checkPendingJob(Job job) { return 0; } + try { + checkIndexedSamplesQuota(job); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return abortJob(job, e); + } + Tool tool; try { tool = new ToolFactory().getTool(job.getTool().getId(), packages); @@ -667,6 +674,54 @@ protected int checkPendingJob(Job job) { return 1; } + private void checkIndexedSamplesQuota(Job job) throws CatalogException { + if (!job.getTool().getId().equals(VariantIndexOperationTool.ID)) { + logger.debug("Check variant index samples quota. Skipping '{}' because it is not a variant index job.", job.getId()); + return; + } + if (catalogManager.getConfiguration().getQuota().getMaxNumVariantIndexSamples() <= 0) { + logger.debug("Check variant index samples quota. Skipping '{}' because the quota is set to 0.", job.getId()); + return; + } + + List fileUriList = new ArrayList<>(job.getInput().size()); + for (File file : job.getInput()) { + fileUriList.add(file.getUri().getPath()); + } + List inputFiles = VariantFileIndexerOperationManager.getInputFiles(catalogManager, job.getStudy().getId(), fileUriList, + token); + // Fetch the samples that are going to be indexed + Set sampleIds = new HashSet<>(); + inputFiles.forEach((f) -> sampleIds.addAll(f.getSampleIds())); + + // Obtain the project in order to get all the studies belonging to the project + FqnUtils.FQN fqn = FqnUtils.parse(job.getStudy().getId()); + String projectFqn = fqn.getProjectFqn(); + List studies = catalogManager.getStudyManager().search(projectFqn, new Query(), StudyManager.INCLUDE_STUDY_IDS, token) + .getResults(); + List studyList = studies.stream().map(Study::getUid).collect(Collectors.toList()); + + // Get all the sampleIds that are already indexed in the project + Query sampleQuery = new Query() + .append(SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_INDEX_STATUS_ID.key(), InternalStatus.READY) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyList); + OpenCGAResult distinct = catalogManager.getSampleManager().distinct(SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); + Set indexedSamples = new HashSet<>(); + indexedSamples.addAll((Collection) distinct.getResults()); + + if (indexedSamples.containsAll(sampleIds)) { + logger.info("All samples are already indexed. Skipping quota check."); + return; + } + + long nonIncludedSamples = sampleIds.stream().filter(sampleId -> !indexedSamples.contains(sampleId)).count(); + + if (catalogManager.getConfiguration().getQuota().getMaxNumVariantIndexSamples() < nonIncludedSamples + indexedSamples.size()) { + throw new CatalogException("Can't index more samples. The project '" + projectFqn + "' has reached the maximum quota of" + + " indexed samples (" + catalogManager.getConfiguration().getQuota().getMaxNumVariantIndexSamples() + ")."); + } + } + private boolean killSignalSent(Job job) { return job.getInternal().isKillJobRequested(); } @@ -934,14 +989,6 @@ private boolean canBeQueued(String organizationId, Job job) { return false; } - if (StringUtils.isNotEmpty(job.getScheduledStartTime())) { - Date date = TimeUtils.toDate(job.getScheduledStartTime()); - if (date.after(new Date())) { - logger.debug("Job '{}' can't be queued yet. It is scheduled to start at '{}'.", job.getId(), job.getScheduledStartTime()); - return false; - } - } - Integer maxJobs = catalogManager.getConfiguration().getAnalysis().getExecution().getMaxConcurrentJobs().get(job.getTool().getId()); if (maxJobs == null) { // No limit for this tool diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java index e17fc25e0ab..93e06642713 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java @@ -1,5 +1,6 @@ package org.opencb.opencga.master.monitor.schedulers; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -152,6 +153,7 @@ private float getPriorityWeight(Job job) { } public Iterator schedule(List pendingJobs, List queuedJobs, List runningJobs) { + Date currentDate = new Date(); TreeMap> jobTreeMap = new TreeMap<>(); try { @@ -160,8 +162,40 @@ public Iterator schedule(List pendingJobs, List queuedJobs, List< throw new RuntimeException("Scheduler exception: " + e.getMessage(), e); } + Map organizationJobStatus = new HashMap<>(); + try { + List organizationIds = catalogManager.getOrganizationManager().getOrganizationIds(token); + for (String organizationId : organizationIds) { + boolean exceeds = catalogManager.getJobManager().exceedsExecutionLimitQuota(organizationId); + organizationJobStatus.put(organizationId, exceeds); + } + } catch (CatalogException e) { + throw new RuntimeException("Couldn't get the execution quota status: " + e.getMessage(), e); + } + + Map> rescheduleJobs = new HashMap<>(); + StopWatch stopWatch = StopWatch.createStarted(); for (Job job : pendingJobs) { + if (StringUtils.isNotEmpty(job.getScheduledStartTime())) { + Date date = TimeUtils.toDate(job.getScheduledStartTime()); + if (date.after(currentDate)) { + logger.debug("Job '{}' can't be queued yet. It is scheduled to start at '{}'.", job.getId(), + job.getScheduledStartTime()); + continue; + } + } + + // Check if execution limit quota has been exceeded + String organizationId = CatalogFqn.extractFqnFromStudyFqn(job.getStudy().getId()).getOrganizationId(); + if (organizationJobStatus.get(organizationId)) { + logger.debug("Job '{}' can't be queued yet. The organization '{}' has exceeded the execution limit quota." + + " It will be scheduled for next month.", job.getId(), organizationId); + rescheduleJobs.putIfAbsent(job.getStudy().getId(), new ArrayList<>()); + rescheduleJobs.get(job.getStudy().getId()).add(job.getUid()); + continue; + } + float priority = getPriorityWeight(job); if (!jobTreeMap.containsKey(priority)) { jobTreeMap.put(priority, new ArrayList<>()); @@ -179,6 +213,29 @@ public Iterator schedule(List pendingJobs, List queuedJobs, List< } logger.debug("Time spent creating iterator: {}", TimeUtils.durationToString(stopWatch)); + // Reschedule jobs that have exceeded the execution limit quota for next month + if (!rescheduleJobs.isEmpty()) { + stopWatch.reset(); + stopWatch.start(); + + for (Map.Entry> entry : rescheduleJobs.entrySet()) { + String studyFqn = entry.getKey(); + List jobUids = entry.getValue(); + + Date firstDayOfNextMonth = TimeUtils.getFirstDayOfNextMonth(currentDate); + String scheduledStartTime = TimeUtils.getTime(firstDayOfNextMonth); + logger.debug("Rescheduling all pending jobs from study '{}' for next month ({}): {}", studyFqn, scheduledStartTime, + jobUids); + try { + catalogManager.getJobManager().rescheduleJobs(studyFqn, jobUids, scheduledStartTime, "Execution limit quota exceeded" + + " for this month. Automatically rescheduling job for date '" + scheduledStartTime + "'.", token); + } catch (CatalogException e) { + throw new RuntimeException("Couldn't reschedule jobs for next month: " + e.getMessage(), e); + } + } + logger.debug("Time spent rescheduling jobs for next month: {}", TimeUtils.durationToString(stopWatch)); + } + return allJobs.iterator(); }