From c73d2d666436063cd1c8c33440143ee755edd5eb Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 17:41:21 -0400 Subject: [PATCH 1/9] feat: run spooker on workflow completion resolves #110 --- MANIFEST.in | 1 + lib/Utils.groovy | 23 +++++++++++++++++++++++ main.nf | 23 +++++++++++++++++++++++ pyproject.toml | 2 +- 4 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 lib/Utils.groovy diff --git a/MANIFEST.in b/MANIFEST.in index b4ce3a3e..2b423948 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ recursive-include assets recursive-include bin recursive-include conf +recursive-include lib recursive-include modules recursive-include src recursive-include submodules diff --git a/lib/Utils.groovy b/lib/Utils.groovy new file mode 100644 index 00000000..02a514b7 --- /dev/null +++ b/lib/Utils.groovy @@ -0,0 +1,23 @@ +class Utils { + // Run spooker and direct output to a log file + public static String run(command_string) { + def message = "" + def out_str = new StringBuilder() + def error_str = new StringBuilder() + try { + def command = command_string.execute() + command.consumeProcessOutput(out_str, error_str) + command.waitForOrKill(1000) + + if (!error_str.toString().equals("")){ + message = "Error executing `${command_string}`:\n${error_str}" + + } else { + message = "Executed `${command_string}`\n${out_str}" + } + } catch(IOException e) { + message = e + } + return message + } +} diff --git a/main.nf b/main.nf index 9ebead8c..9263f3b2 100644 --- a/main.nf +++ b/main.nf @@ -35,6 +35,29 @@ include { PPQT_PROCESS MULTIQC } from "./modules/local/qc.nf" include { NORMALIZE_INPUT } from "./modules/local/deeptools.nf" +workflow.onComplete { + message = Utils.run("spooker ${workflow.launchDir} | tee ${workflow.launchDir}/log/spooker.log") + println "${message}" +} + + +process YEET { + input: + val text + + output: + stdout + + script: + """ + echo $text + """ + +} +workflow debug { + println "hello world!" +} + workflow MAKE_REFERENCE { PREPARE_GENOME() } diff --git a/pyproject.toml b/pyproject.toml index 8956072d..775edfde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ Changelog = "https://github.com/CCBR/CHAMPAGNE/blob/main/docs/CHANGELOG.md" champagne = "." [tool.setuptools.package-data] -"*" = ["CITATION.cff", "LICENSE", "VERSION", "main.nf", "nextflow.config", "assets/**", "bin/**", "conf/**", "modules/**", "subworkflows/**", "tests/**"] +"*" = ["CITATION.cff", "LICENSE", "VERSION", "main.nf", "nextflow.config", "assets/**", "bin/**", "conf/**", "lib/**", "modules/**", "subworkflows/**", "tests/**"] [tool.setuptools.dynamic] version = {file = "VERSION"} From 6f235c09b01a51d0aaa9569741faf345b995de2b Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 17:42:43 -0400 Subject: [PATCH 2/9] docs: link to source --- lib/Utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Utils.groovy b/lib/Utils.groovy index 02a514b7..e93674db 100644 --- a/lib/Utils.groovy +++ b/lib/Utils.groovy @@ -1,5 +1,5 @@ class Utils { - // Run spooker and direct output to a log file + // adapted from https://groovy-lang.gitlab.io/101-scripts/basico/command_local-en.html public static String run(command_string) { def message = "" def out_str = new StringBuilder() From 89d528feedb93f92086e3db5f1ae6b7f7352f8ee Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 18:03:27 -0400 Subject: [PATCH 3/9] refactor: create spooker function to call Utils.run() --- lib/Utils.groovy | 8 ++++++++ main.nf | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/Utils.groovy b/lib/Utils.groovy index e93674db..f28a045d 100644 --- a/lib/Utils.groovy +++ b/lib/Utils.groovy @@ -1,4 +1,5 @@ class Utils { + // run a shell command and capture the output // adapted from https://groovy-lang.gitlab.io/101-scripts/basico/command_local-en.html public static String run(command_string) { def message = "" @@ -20,4 +21,11 @@ class Utils { } return message } + + // run spooker for the workflow + public static String spooker(workflow) { + def pipeline_name = "${workflow.manifest.name.tokenize('/')[-1]}" + def message = this.run("spooker ${workflow.launchDir} ${pipeline_name} | tee ${workflow.launchDir}/log/spooker.log") + return message + } } diff --git a/main.nf b/main.nf index 9263f3b2..de4e8e35 100644 --- a/main.nf +++ b/main.nf @@ -36,7 +36,7 @@ include { PPQT_PROCESS include { NORMALIZE_INPUT } from "./modules/local/deeptools.nf" workflow.onComplete { - message = Utils.run("spooker ${workflow.launchDir} | tee ${workflow.launchDir}/log/spooker.log") + def message = Utils.spooker(workflow) println "${message}" } From 902a2f15bbcd10876f38e4630bc3cf3b78502d8c Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 19:02:53 -0400 Subject: [PATCH 4/9] refactor: separate spooker call from writing to log --- lib/Utils.groovy | 42 ++++++++++++++++++++++-------------------- main.nf | 4 +++- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/lib/Utils.groovy b/lib/Utils.groovy index f28a045d..fb35f743 100644 --- a/lib/Utils.groovy +++ b/lib/Utils.groovy @@ -1,31 +1,33 @@ class Utils { // run a shell command and capture the output // adapted from https://groovy-lang.gitlab.io/101-scripts/basico/command_local-en.html - public static String run(command_string) { - def message = "" - def out_str = new StringBuilder() - def error_str = new StringBuilder() - try { - def command = command_string.execute() - command.consumeProcessOutput(out_str, error_str) - command.waitForOrKill(1000) + public static List run(command_string) { + def out = new StringBuilder() + def err = new StringBuilder() + def command = command_string.execute() + command.consumeProcessOutput(out, err) + command.waitFor() - if (!error_str.toString().equals("")){ - message = "Error executing `${command_string}`:\n${error_str}" - - } else { - message = "Executed `${command_string}`\n${out_str}" - } - } catch(IOException e) { - message = e - } - return message + return [out, err] } // run spooker for the workflow public static String spooker(workflow) { def pipeline_name = "${workflow.manifest.name.tokenize('/')[-1]}" - def message = this.run("spooker ${workflow.launchDir} ${pipeline_name} | tee ${workflow.launchDir}/log/spooker.log") - return message + def command_string = "spooker ${workflow.launchDir} ${pipeline_name}" + def out = new StringBuilder() + def err = new StringBuilder() + try { + def command = command_string.execute() + command.consumeProcessOutput(out, err) + command.waitFor() + } catch(IOError e) { + err = e + } + new FileWriter("${workflow.launchDir}/log/spooker.log").with { + write("${out}\n${err}") + flush() + } + return err } } diff --git a/main.nf b/main.nf index de4e8e35..3b24ca53 100644 --- a/main.nf +++ b/main.nf @@ -37,7 +37,9 @@ include { NORMALIZE_INPUT } from "./modules/local/deeptools.nf" workflow.onComplete { def message = Utils.spooker(workflow) - println "${message}" + if (message) { + println message + } } From 70c2b0568a68775b9e04127916e7da1567cada13 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 19:19:56 -0400 Subject: [PATCH 5/9] fix: catch IOException when spooker not in path --- lib/Utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Utils.groovy b/lib/Utils.groovy index fb35f743..2d93d74f 100644 --- a/lib/Utils.groovy +++ b/lib/Utils.groovy @@ -21,7 +21,7 @@ class Utils { def command = command_string.execute() command.consumeProcessOutput(out, err) command.waitFor() - } catch(IOError e) { + } catch(IOException e) { err = e } new FileWriter("${workflow.launchDir}/log/spooker.log").with { From ecca3d8a7b62ee492aab98ae2e66e031c8a2c0c0 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 19:21:20 -0400 Subject: [PATCH 6/9] chore: print message when spooker is running --- main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.nf b/main.nf index 3b24ca53..9d2e79ca 100644 --- a/main.nf +++ b/main.nf @@ -36,6 +36,7 @@ include { PPQT_PROCESS include { NORMALIZE_INPUT } from "./modules/local/deeptools.nf" workflow.onComplete { + println "Running spooker" def message = Utils.spooker(workflow) if (message) { println message From ae6d06da89786742a151d2850484a5cddaf3f974 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 19:22:50 -0400 Subject: [PATCH 7/9] chore: update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecdad8aa..cbb08c8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## development version + +- Run spooker on workflow completion (#126). + ## CHAMPAGNE 0.2.0 ### New features From 8109106a2d0bb2fcc1323c299338627b6115c218 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 19:25:47 -0400 Subject: [PATCH 8/9] chore: remove unused run() function because other functions that call it don't seem to wait for the process to actually complete before trying to use the output --- lib/Utils.groovy | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/Utils.groovy b/lib/Utils.groovy index 2d93d74f..db78a83f 100644 --- a/lib/Utils.groovy +++ b/lib/Utils.groovy @@ -1,16 +1,4 @@ class Utils { - // run a shell command and capture the output - // adapted from https://groovy-lang.gitlab.io/101-scripts/basico/command_local-en.html - public static List run(command_string) { - def out = new StringBuilder() - def err = new StringBuilder() - def command = command_string.execute() - command.consumeProcessOutput(out, err) - command.waitFor() - - return [out, err] - } - // run spooker for the workflow public static String spooker(workflow) { def pipeline_name = "${workflow.manifest.name.tokenize('/')[-1]}" From 17a65d293bb34f5c1f0be455933ee1d39037c412 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Oct 2023 19:26:11 -0400 Subject: [PATCH 9/9] chore: remove debug scaffolding --- main.nf | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/main.nf b/main.nf index 9d2e79ca..696ec0af 100644 --- a/main.nf +++ b/main.nf @@ -43,24 +43,6 @@ workflow.onComplete { } } - -process YEET { - input: - val text - - output: - stdout - - script: - """ - echo $text - """ - -} -workflow debug { - println "hello world!" -} - workflow MAKE_REFERENCE { PREPARE_GENOME() }