Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(task): dependencies #26467

Merged
merged 39 commits into from
Nov 19, 2024
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
cb60623
feat(task): dependencies
dsherret Oct 22, 2024
526834d
Merge branch 'main' into feat_task_dependencies
bartlomieju Nov 14, 2024
7f2e2bd
support for description
bartlomieju Nov 14, 2024
8d02b99
simplify
bartlomieju Nov 14, 2024
9e16466
further refactor
bartlomieju Nov 14, 2024
ba70860
add a todo
bartlomieju Nov 14, 2024
1bddb7c
Merge branch 'main' into feat_task_dependencies
bartlomieju Nov 16, 2024
b835b45
add basic tests
bartlomieju Nov 16, 2024
9e206db
update tests
bartlomieju Nov 16, 2024
8e2cb55
add cross package test
bartlomieju Nov 16, 2024
e8a5459
fmt and lint
bartlomieju Nov 16, 2024
32b04fe
add failing test for 'diamond' dependencies
bartlomieju Nov 16, 2024
52762f9
update schema file
bartlomieju Nov 16, 2024
caedab6
Merge branch 'main' into feat_task_dependencies
bartlomieju Nov 17, 2024
ed367a2
change diamond dep test for now
bartlomieju Nov 17, 2024
05432d8
Merge branch 'main' into feat_task_dependencies
bartlomieju Nov 17, 2024
2e73474
feat: run task chains in topological order
marvinhagemeister Nov 18, 2024
efd9c80
fix: make clippy happy
marvinhagemeister Nov 18, 2024
71209f7
Merge branch 'main' into feat_task_dependencies
bartlomieju Nov 18, 2024
b54306b
fix: don't parse script value as task
marvinhagemeister Nov 18, 2024
b5978df
fix: remove accidentally committed fixture
marvinhagemeister Nov 18, 2024
89c5542
feat: parallelize tasks if possible
marvinhagemeister Nov 18, 2024
efbd227
fix: make clippy happy
marvinhagemeister Nov 18, 2024
d5c4dc6
this doesn't work
bartlomieju Nov 19, 2024
27d86a4
wip
bartlomieju Nov 19, 2024
b47c130
a
bartlomieju Nov 19, 2024
530ac3a
Merge branch 'main' into feat_task_dependencies
bartlomieju Nov 19, 2024
2317822
show which tasks we depend on
bartlomieju Nov 19, 2024
f537a2b
remove a clone
bartlomieju Nov 19, 2024
8e4e7d1
cleanup
bartlomieju Nov 19, 2024
903da22
remove TODO
bartlomieju Nov 19, 2024
52c2305
simplify a bit
bartlomieju Nov 19, 2024
9c60d6f
fix a test
bartlomieju Nov 19, 2024
245e813
Use refcell
dsherret Nov 19, 2024
b35e17a
add missing newline
bartlomieju Nov 19, 2024
c7f8c4b
Show cycle detection, move state into method call, extract function f…
dsherret Nov 19, 2024
45017a5
Remove marked now that we keep track of the path
dsherret Nov 19, 2024
b4f8593
Merge branch 'main' into feat_task_dependencies
bartlomieju Nov 19, 2024
f00edf1
remove todo
bartlomieju Nov 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cli/schemas/config-file.v1.json
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,13 @@
"type": "string",
"required": true,
"description": "The task to execute"
},
"dependencies": {
"type": "array",
"items": {
"type": "string"
},
"description": "Tasks that should be executed before this task"
}
}
}
Expand Down
124 changes: 107 additions & 17 deletions cli/tools/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,31 @@ pub async fn execute_script(
env_vars,
cli_options,
};
task_runner.run_task(task_name).await

match task_runner.sort_tasks_topo(task_name) {
Ok(sorted) => task_runner.run_tasks(sorted).await,
Err(err) => match err {
TaskError::NotFound(name) => {
if task_flags.is_run {
return Err(anyhow!("Task not found: {}", name));
}

log::error!("Task not found: {}", name);
if log::log_enabled!(log::Level::Error) {
print_available_tasks(
&mut std::io::stderr(),
&task_runner.cli_options.start_dir,
&task_runner.tasks_config,
)?;
}
Ok(1)
}
TaskError::TaskDepCycle(name) => {
log::error!("Task cycle detected: {}", name);
Ok(1)
}
},
}
}

struct RunSingleOptions<'a> {
Expand All @@ -86,6 +110,12 @@ struct RunSingleOptions<'a> {
custom_commands: HashMap<String, Rc<dyn ShellCommand>>,
}

#[derive(Debug)]
enum TaskError {
NotFound(String),
TaskDepCycle(String),
}

struct TaskRunner<'a> {
tasks_config: WorkspaceTasksConfig,
task_flags: &'a TaskFlags,
Expand All @@ -96,26 +126,30 @@ struct TaskRunner<'a> {
}

impl<'a> TaskRunner<'a> {
async fn run_task(
async fn run_tasks(
&self,
task_name: &String,
task_names: Vec<String>,
) -> Result<i32, deno_core::anyhow::Error> {
let Some((dir_url, task_or_script)) = self.tasks_config.task(task_name)
else {
if self.task_flags.is_run {
return Err(anyhow!("Task not found: {}", task_name));
// TODO(bartlomieju): we might want to limit concurrency here - eg. `wireit` runs 2 tasks in parallel
// unless and env var is specified.
// TODO(marvinhagemeister): Run in sequence for now as it's not safe
// to run all tasks in parallel. We can only run tasks in parallel where
// all dependencies have been executed prior to that.
bartlomieju marked this conversation as resolved.
Show resolved Hide resolved
for task_name in &task_names {
let exit_code = self.run_task(task_name).await?;
if exit_code > 0 {
return Ok(exit_code);
}
}

log::error!("Task not found: {}", task_name);
if log::log_enabled!(log::Level::Error) {
print_available_tasks(
&mut std::io::stderr(),
&self.cli_options.start_dir,
&self.tasks_config,
)?;
}
return Ok(1);
};
Ok(0)
}

async fn run_task(
&self,
task_name: &String,
) -> Result<i32, deno_core::anyhow::Error> {
let (dir_url, task_or_script) = self.get_task(task_name.as_str()).unwrap();

match task_or_script {
TaskOrScript::Task(_tasks, definition) => {
Expand Down Expand Up @@ -232,6 +266,62 @@ impl<'a> TaskRunner<'a> {
.exit_code,
)
}

fn get_task(
&self,
task_name: &str,
) -> Result<(&Url, TaskOrScript), TaskError> {
let Some(result) = self.tasks_config.task(task_name) else {
return Err(TaskError::NotFound(task_name.to_string()));
};

Ok(result)
}

fn sort_tasks_topo(&self, name: &str) -> Result<Vec<String>, TaskError> {
let mut marked: Vec<String> = vec![];
let mut sorted: Vec<String> = vec![];

self.sort_visit(name, &mut marked, &mut sorted)?;

Ok(sorted)
}

fn sort_visit(
&self,
name: &str,
marked: &mut Vec<String>,
sorted: &mut Vec<String>,
) -> Result<(), TaskError> {
// Already sorted
if sorted.contains(&name.to_string()) {
return Ok(());
}

// Graph has a cycle
if marked.contains(&name.to_string()) {
return Err(TaskError::TaskDepCycle(name.to_string()));
}

marked.push(name.to_string());

let Some(def) = self.tasks_config.task(name) else {
return Err(TaskError::NotFound(name.to_string()));
};

match def.1 {
TaskOrScript::Task(_, actual_def) => {
for dep in &actual_def.dependencies {
self.sort_visit(dep, marked, sorted)?
}
}
TaskOrScript::Script(_, name) => self.sort_visit(name, marked, sorted)?,
}

sorted.push(name.to_string());

Ok(())
}
}

fn output_task(task_name: &str, script: &str) {
Expand Down
7 changes: 7 additions & 0 deletions tests/specs/npm/prefix_import_and_alias/__test__.jsonc
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"tempDir": true,
"steps": [{
"args": "run -A main.js",
"output": "main.out"
}]
}
5 changes: 5 additions & 0 deletions tests/specs/npm/prefix_import_and_alias/deno.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"imports": {
"@denotest/basic": "npm:@denotest/[email protected]"
}
}
Empty file.
49 changes: 49 additions & 0 deletions tests/specs/task/dependencies/__test__.jsonc
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
{
"tests": {
"basic1": {
"cwd": "basic1",
"tempDir": true,
"args": "task run",
"output": "./basic1.out"
},
"basic2": {
"cwd": "basic2",
"tempDir": true,
"args": "task run",
"output": "./basic2.out"
},
"cross_package": {
"cwd": "cross_package/package1",
"tempDir": true,
"args": "task run",
"output": "./cross_package.out",
"exitCode": 1
},
"diamond": {
"cwd": "diamond",
"tempDir": true,
"args": "task a",
"output": "./diamond.out"
},
"diamond_big": {
"cwd": "diamond_big",
"tempDir": true,
"args": "task a",
"output": "./diamond_big.out"
},
"cycle": {
"cwd": "cycle",
"tempDir": true,
"output": "./cycle.out",
"args": "task a",
"exitCode": 1
},
"cycle_2": {
"cwd": "cycle_2",
"tempDir": true,
"args": "task a",
"output": "./cycle_2.out",
"exitCode": 1
}
}
}
10 changes: 10 additions & 0 deletions tests/specs/task/dependencies/basic1.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
Task build1 deno run ../build1.js
Starting build1
build1 performing more work...
build1 finished
Task build2 deno run ../build2.js
Starting build2
build2 performing more work...
build2 finished
Task run deno run ../run.js
run finished
10 changes: 10 additions & 0 deletions tests/specs/task/dependencies/basic1/deno.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"tasks": {
"build1": "deno run ../build1.js",
"build2": "deno run ../build2.js",
"run": {
"command": "deno run ../run.js",
"dependencies": ["build1", "build2"]
}
}
}
10 changes: 10 additions & 0 deletions tests/specs/task/dependencies/basic2.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
Task build1 deno run ../build1.js
Starting build1
build1 performing more work...
build1 finished
Task build2 deno run ../build2.js
Starting build2
build2 performing more work...
build2 finished
Task run deno run ../run.js
run finished
13 changes: 13 additions & 0 deletions tests/specs/task/dependencies/basic2/deno.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"tasks": {
"build1": "deno run ../build1.js",
"build2": {
"command": "deno run ../build2.js",
"dependencies": ["build1"]
},
"run": {
"command": "deno run ../run.js",
"dependencies": ["build2"]
}
}
}
9 changes: 9 additions & 0 deletions tests/specs/task/dependencies/build1.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { randomTimeout } from "./util.js";

console.log("Starting build1");

await randomTimeout(500, 750);
console.log("build1 performing more work...");
await randomTimeout(500, 750);

console.log("build1 finished");
9 changes: 9 additions & 0 deletions tests/specs/task/dependencies/build2.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { randomTimeout } from "./util.js";

console.log("Starting build2");

await randomTimeout(250, 750);
console.log("build2 performing more work...");
await randomTimeout(250, 750);

console.log("build2 finished");
4 changes: 4 additions & 0 deletions tests/specs/task/dependencies/cross_package.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Task not found: ../package2:run
Available tasks:
- run
deno run.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"tasks": {
"run": {
"command": "deno run.js",
"dependencies": ["../package2:run"]
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"tasks": {
"run": "deno run.js"
}
}
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/cycle.out
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Task cycle detected: a
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/cycle/a.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
console.log("Running a");
8 changes: 8 additions & 0 deletions tests/specs/task/dependencies/cycle/deno.jsonc
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"tasks": {
"a": {
"command": "deno run a.js",
"dependencies": ["a"]
}
}
}
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/cycle_2.out
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Task cycle detected: a
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/cycle_2/a.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
console.log("Running a");
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/cycle_2/b.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
console.log("Running b");
12 changes: 12 additions & 0 deletions tests/specs/task/dependencies/cycle_2/deno.jsonc
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"tasks": {
"a": {
"command": "deno run a.js",
"dependencies": ["b"]
},
"b": {
"command": "deno run b.js",
"dependencies": ["a"]
}
}
}
8 changes: 8 additions & 0 deletions tests/specs/task/dependencies/diamond.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Task d deno run d.js
Running d
Task b deno run b.js
Running b
Task c deno run c.js
Running c
Task a deno run a.js
Running a
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/diamond/a.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
console.log("Running a");
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/diamond/b.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
console.log("Running b");
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/diamond/c.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
console.log("Running c");
1 change: 1 addition & 0 deletions tests/specs/task/dependencies/diamond/d.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
console.log("Running d");
22 changes: 22 additions & 0 deletions tests/specs/task/dependencies/diamond/deno.jsonc
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
// a
// / \
// b c
// \ /
// d
"tasks": {
"a": {
"command": "deno run a.js",
"dependencies": ["b", "c"]
},
"b": {
"command": "deno run b.js",
"dependencies": ["d"]
},
"c": {
"command": "deno run c.js",
"dependencies": ["d"]
},
"d": "deno run d.js"
}
}
Loading
Loading