Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TASK-6446 - Implement quotas management #2537

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
Open

TASK-6446 - Implement quotas management #2537

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
fd5bc7d
core: add quota configuration, #TASL-6446
pfurio Nov 4, 2024
fe6f97d
catalog: implement quota restrictions, #TASK-6446
pfurio Nov 4, 2024
484e763
master: abort jobs when the quota is exceeded, #TASK-6446
pfurio Nov 5, 2024
e141395
catalog: implement job rescheduler, #TASK-6446
pfurio Nov 6, 2024
2b880bb
analysis: implement sample index quota validation, #TASK-6446
pfurio Nov 19, 2024
ce73e56
catalog: change default quotas for testing, #TASK-6446
pfurio Dec 18, 2024
e5dee6f
catalog: fix revision comments, #TASK-6446
pfurio Dec 20, 2024
08800de
Merge branch 'develop' into TASK-6446
pfurio Dec 20, 2024
1786f56
Merge branch 'develop' into TASK-6446
pfurio Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -704,7 +704,7 @@ public OpenCGAResult<RgaKnockoutByGene> geneQuery(String studyStr, Query query,
.append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS)
.append(SampleDBAdaptor.QueryParams.INTERNAL_RGA_STATUS.key(), RgaIndex.Status.INDEXED)
.append(SampleDBAdaptor.QueryParams.ID.key(), includeIndividuals);
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(),
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(),
SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token);
includeSampleIds = new HashSet<>((List<String>) authorisedSampleIdResult.getResults());
} else {
Expand All @@ -720,7 +720,7 @@ public OpenCGAResult<RgaKnockoutByGene> geneQuery(String studyStr, Query query,
// 3. Get list of sample ids for which the user has permissions
Query sampleQuery = new Query(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS)
.append(SampleDBAdaptor.QueryParams.ID.key(), sampleIds);
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(),
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(),
SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token);
// TODO: The number of samples to include could be really high
includeSampleIds = new HashSet<>((List<String>) authorisedSampleIdResult.getResults());
Expand All @@ -737,7 +737,7 @@ public OpenCGAResult<RgaKnockoutByGene> geneQuery(String studyStr, Query query,
logger.warn("Include only the samples that are actually necessary");
}

OpenCGAResult<?> sampleResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(),
OpenCGAResult<?> sampleResult = catalogManager.getSampleManager().distinct(study.getFqn(),
SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token);
includeSampleIds = new HashSet<>((List<String>) sampleResult.getResults());
}
Expand Down Expand Up @@ -831,7 +831,7 @@ public OpenCGAResult<KnockoutByVariant> variantQuery(String studyStr, Query quer
.append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS)
.append(SampleDBAdaptor.QueryParams.INTERNAL_RGA_STATUS.key(), RgaIndex.Status.INDEXED)
.append(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), includeIndividuals);
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(),
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(),
SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token);
includeSampleIds = new HashSet<>((List<String>) authorisedSampleIdResult.getResults());
} else {
Expand All @@ -850,7 +850,7 @@ public OpenCGAResult<KnockoutByVariant> variantQuery(String studyStr, Query quer
Query sampleQuery = new Query()
.append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS)
.append(SampleDBAdaptor.QueryParams.ID.key(), sampleIds);
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(),
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(),
SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token);
includeSampleIds = new HashSet<>((List<String>) authorisedSampleIdResult.getResults());
}
Expand All @@ -865,7 +865,7 @@ public OpenCGAResult<KnockoutByVariant> variantQuery(String studyStr, Query quer
logger.warn("Include only the samples that are actually necessary");
}

OpenCGAResult<?> sampleResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(),
OpenCGAResult<?> sampleResult = catalogManager.getSampleManager().distinct(study.getFqn(),
SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token);
includeSampleIds = new HashSet<>((List<String>) sampleResult.getResults());
}
Expand Down Expand Up @@ -1773,7 +1773,7 @@ private Set<String> getAuthorisedSamples(String organizationId, String study, Se
query.put(ACL_PARAM, userId + ":" + StringUtils.join(otherPermissions, ","));
}

OpenCGAResult<?> distinct = catalogManager.getSampleManager().distinct(organizationId, study, SampleDBAdaptor.QueryParams.ID.key(),
OpenCGAResult<?> distinct = catalogManager.getSampleManager().distinct(study, SampleDBAdaptor.QueryParams.ID.key(),
query, token);
return distinct.getResults().stream().map(String::valueOf).collect(Collectors.toSet());
}
Expand Down Expand Up @@ -1874,7 +1874,7 @@ private Preprocess individualQueryPreprocess(String organizationId, Study study,
List<String> tmpValues = values.subList(currentBatch, Math.min(values.size(), batchSize + currentBatch));

sampleQuery.put(sampleQueryField, tmpValues);
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(),
OpenCGAResult<?> authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(),
SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token);
authorisedSamples.addAll((Collection<? extends String>) authorisedSampleIdResult.getResults());

Expand Down
Loading
Loading