Argo data submission [email protected] #93
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,7 +22,7 @@ | |
| */ | ||
|
|
||
| nextflow.enable.dsl = 2 | ||
| version = '1.0.4' | ||
|
|
||
| // universal params go here, change default value as needed | ||
| params.container = "" | ||
|
|
@@ -44,10 +44,11 @@ params.download_mode="local" | |
| params.song_container = "ghcr.io/overture-stack/song-client" | ||
| params.song_container_version = "5.0.2" | ||
| params.score_container = "ghcr.io/overture-stack/score" | ||
| params.score_container_version = "5.10.0" | ||
| params.score_mem = 20 | ||
| params.score_cpus = 8 | ||
| params.score_force = false | ||
| params.dry_run = false | ||
|
There was a problem hiding this comment. We mentioned using There was a problem hiding this comment. When we run the following command: |
||
|
|
||
| // sanityChecks | ||
| params.song_url="" | ||
|
|
@@ -58,7 +59,7 @@ params.api_token="" | |
| // payloadJsonToTsvs | ||
| params.data_directory="NO_FILE1" | ||
| params.skip_duplicate_check=false | ||
| params.skip_sanity_check=false | ||
| // payloadGenSeqExperiment | ||
| params.schema_url="" | ||
| params.experiment_info_tsv="NO_FILE2" | ||
|
|
@@ -237,6 +238,19 @@ workflow ArgoDataSubmissionWf { | |
| clinical_url | ||
| main: | ||
|
|
||
| if (!"${workflow.profile}".contains('docker') && !"${workflow.profile}".contains('singularity')){ | ||
| exit 1, "Error Missing profile. `-profile` must be specified with the engines :`docker` or `singularity`." | ||
| } | ||
| if (!"${workflow.profile}".contains('rdpc_qa') && !"${workflow.profile}".contains('rdpc_dev') && !"${workflow.profile}".contains('rdpc')){ | ||
| exit 1, "Error Missing profile. `-profile` must be specified with the rdpc environments : `rdpc_qa`,`rdpc_dev`, or `rdpc`." | ||
| } | ||
|
|
||
| if (!params.api_token){ | ||
| if (!params.api_download_token || !params.api_upload_token) { | ||
| exit 1, "Error SONG parameters detected but missing token params. `--api_token` or `api_upload_token` and `api_download_token` must be supplied when uploading." | ||
| } | ||
| } | ||
|
|
||
| if ( | ||
| og_experiment_info_tsv.startsWith("NO_FILE") && \ | ||
| og_read_group_info_tsv.startsWith("NO_FILE") && \ | ||
|
|
@@ -278,29 +292,41 @@ workflow ArgoDataSubmissionWf { | |
| file(data_directory) | ||
| ) | ||
|
|
||
| if (params.skip_sanity_check){ | ||
| experiment_info_tsv=file(payloadJsonToTsvs.out.experiment_tsv) | ||
| read_group_info_tsv=file(og_read_group_info_tsv) | ||
| file_info_tsv=file(og_file_info_tsv) | ||
| } else { | ||
| sanityCheck( | ||
| payloadJsonToTsvs.out.experiment_tsv, | ||
| api_token, | ||
| song_url, | ||
| clinical_url, | ||
| params.skip_duplicate_check | ||
| ) | ||
|
|
||
| experiment_info_tsv=sanityCheck.out.updated_experiment_info_tsv | ||
| read_group_info_tsv=file(og_read_group_info_tsv) | ||
| file_info_tsv=file(og_file_info_tsv) | ||
| } | ||
| } else { | ||
| if (params.skip_sanity_check){ | ||
| experiment_info_tsv=file(og_experiment_info_tsv) | ||
| read_group_info_tsv=file(og_read_group_info_tsv) | ||
| file_info_tsv=file(og_file_info_tsv) | ||
| } else { | ||
| sanityCheck( | ||
| file(og_experiment_info_tsv), | ||
| api_token, | ||
| song_url, | ||
| clinical_url, | ||
| params.skip_duplicate_check | ||
| ) | ||
|
|
||
| experiment_info_tsv=sanityCheck.out.updated_experiment_info_tsv | ||
| read_group_info_tsv=file(og_read_group_info_tsv) | ||
| file_info_tsv=file(og_file_info_tsv) | ||
| } | ||
| } | ||
|
|
||
| checkCramReference( | ||
|
|
@@ -362,20 +388,22 @@ workflow ArgoDataSubmissionWf { | |
| skipping_tests | ||
| ) | ||
|
|
||
| if (!params.dry_run){ | ||
| uploadWf( | ||
| study_id, | ||
| valSeq.out.validated_payload, | ||
| sequence_files.collect(), | ||
| '' | ||
| ) | ||
|
|
||
| submissionReceipt( | ||
| study_id, | ||
| uploadWf.out.analysis_id, | ||
| song_url, | ||
| params.skip_submission_check, | ||
| sequence_files.collect() | ||
| ) | ||
| } | ||
|
|
||
| } else if (checkCramReference.out.check_status && !ref_genome_fa.startsWith("NO_FILE")){ | ||
| // If reference genome is provided... | ||
|
|
@@ -403,52 +431,62 @@ workflow ArgoDataSubmissionWf { | |
| sequence_files.collect().concat(cram2bam.out.output_bam.collect()).collect(), | ||
| skipping_tests | ||
| ) | ||
| if (!params.dry_run){ | ||
| uploadWf( | ||
| study_id, | ||
| valSeq.out.validated_payload, | ||
| not_cram_sequence_files.concat(cram2bam.out.output_bam.collect()).collect(), | ||
| '' | ||
| ) | ||
|
|
||
| submissionReceipt( | ||
| study_id, | ||
| uploadWf.out.analysis_id, | ||
| song_url, | ||
| params.skip_submission_check, | ||
| not_cram_sequence_files.concat(cram2bam.out.output_bam.collect()).collect() | ||
| ) | ||
| } | ||
| } | ||
| if (params.cleanup && params.download_mode!='local' && ref_genome_fa.startsWith("NO_FILE") && !params.dry_run) { | ||
| // only cleanup the sequence files when they are not from local | ||
| cleanup( | ||
| sequence_files.collect(), | ||
| submissionReceipt.out.receipt // wait until upload is done | ||
| ) | ||
| } else if (params.cleanup && params.download_mode!='local' && !ref_genome_fa.startsWith("NO_FILE") && !params.dry_run){ | ||
| // only cleanup the sequence files and cram2bam output when they are not from local | ||
| cleanup( | ||
| sequence_files.collect().concat(cram2bam.out.output_bam.collect()).collect(), | ||
| submissionReceipt.out.receipt // wait until upload is done | ||
| ) | ||
| } else if (params.cleanup && params.download_mode=='local' && !ref_genome_fa.startsWith("NO_FILE") && !params.dry_run){ | ||
| // only cleanup output from cram2bam on local | ||
| cleanup( | ||
| cram2bam.out.output_bam.collect(), | ||
| submissionReceipt.out.receipt // wait until upload is done | ||
| ) | ||
| } | ||
|
|
||
| if (!params.dry_run){ | ||
| printOut( | ||
| pGenExp.out.payload, | ||
| uploadWf.out.analysis_id, | ||
| submissionReceipt.out.receipt | ||
| ) | ||
| out_payload = pGenExp.out.payload | ||
| out_analysis_id = uploadWf.out.analysis_id | ||
| out_receipt = submissionReceipt.out.receipt | ||
| } else { | ||
| out_payload = null | ||
| out_analysis_id = null | ||
| out_receipt = null | ||
| } | ||
|
|
||
| emit: | ||
| out_json_file=out_payload | ||
| out_output_analysis_id=out_analysis_id | ||
| out_receipt=out_receipt | ||
| } | ||
|
|
||
| // this provides an entry point for this main script, so it can be run directly without clone the repo | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,7 +26,7 @@ | |
| */ | ||
|
|
||
| nextflow.enable.dsl = 2 | ||
| version = '1.0.4' | ||
|
|
||
| // universal params | ||
| params.publish_dir = "" | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| type submitter_sequencing_experiment_id program_id submitter_donor_id submitter_specimen_id submitter_sample_id submitter_matched_normal_sample_id sequencing_center platform platform_model experimental_strategy sequencing_date read_group_count | ||
| sequencing_experiment SubWf_exp_02_FASTQ_input TEST-CA TEST_SUBMITTER_DONOR_ID_zzespofnsa TEST_SUBMITTER_SPECIMEN_ID_zzespofnsaT1 TEST_SUBMITTER_SAMPLE_ID_zzespofnsaT1 TEST_SUBMITTER_SAMPLE_ID_zzespofnsaN1 EXT ILLUMINA HiSeq 2000 WGS 2014-12-12 3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| type name format size md5sum path | ||
| file C0HVY.2_r1.fq.gz FASTQ 2040 c4971f805930e9e31e1c45314c652d3c input/C0HVY.2_r1.fq.gz | ||
| file C0HVY.2_r2.fq.gz FASTQ 2042 f60026e682bf55f014c84d494112aa13 input/C0HVY.2_r2.fq.gz | ||
| file D0RE2.1_r1.fq.gz FASTQ 2309 835b35f6e7f8263636fce4224b1aca69 input/D0RE2.1_r1.fq.gz | ||
| file D0RE2.1_r2.fq.gz FASTQ 2365 23d030ddba2916f8430b13234e3e9a0c input/D0RE2.1_r2.fq.gz | ||
| file D0RH0.2_r1.fq.gz FASTQ 1872 b3aa75d0585a989bb25fae252a10b532 input/D0RH0.2_r1.fq.gz | ||
| file D0RH0.2_r2.fq.gz FASTQ 1894 c59ebfa4199287bf3911a4c1d6eda71e input/D0RH0.2_r2.fq.gz |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| type submitter_read_group_id read_group_id_in_bam submitter_sequencing_experiment_id platform_unit is_paired_end file_r1 file_r2 read_length_r1 read_length_r2 insert_size sample_barcode library_name | ||
| read_group C0HVY.2 SubWf_exp_02_FASTQ_input 74_8a true C0HVY.2_r1.fq.gz C0HVY.2_r2.fq.gz 150 150 298 Pond-147580 | ||
| read_group D0RE2.1 SubWf_exp_02_FASTQ_input 74_8b true D0RE2.1_r1.fq.gz D0RE2.1_r2.fq.gz 150 150 298 Pond-147580 | ||
| read_group D0RH0.2 SubWf_exp_02_FASTQ_input 74_8c true D0RH0.2_r1.fq.gz D0RH0.2_r2.fq.gz 150 150 298 Pond-147580 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@edsu7 can we bump the version to
1.1.0as this version contains the url changes to backend infrastructure which are breaking changes.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For WFPM, would it be easier to start a new branch as
1.1.0?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
for sure, that's the only way I guess.