From c147077c200e134e8cf113266abdf797f397ad5f Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Fri, 31 Jan 2025 19:26:32 -0500 Subject: [PATCH] [components] Add workspace.yaml scaffolding --- .../components/setting-up-a-deployment.md | 42 ++++++++-- .../dagster_dg/cli/code_location.py | 26 +++++- .../dagster-dg/dagster_dg/context.py | 6 ++ .../workspace.yaml.jinja | 10 +++ .../libraries/dagster-dg/dagster_dg/utils.py | 12 +++ .../cli_tests/test_code_location_commands.py | 79 ++++++++++++++----- .../cli_tests/test_deployment_commands.py | 1 + 7 files changed, 149 insertions(+), 27 deletions(-) create mode 100644 python_modules/libraries/dagster-dg/dagster_dg/templates/DEPLOYMENT_NAME_PLACEHOLDER/workspace.yaml.jinja diff --git a/docs/docs-beta/docs/guides/build/components/setting-up-a-deployment.md b/docs/docs-beta/docs/guides/build/components/setting-up-a-deployment.md index 5fcd573e2ff58..17391d9ee914c 100644 --- a/docs/docs-beta/docs/guides/build/components/setting-up-a-deployment.md +++ b/docs/docs-beta/docs/guides/build/components/setting-up-a-deployment.md @@ -31,6 +31,7 @@ $ cd my-deployment && tree . ├── code_locations └── pyproject.toml +└── workspace.yaml ``` Importantly, the `pyproject.toml` file contains an `is_deployment` setting @@ -43,6 +44,23 @@ marking this directory as a deployment: is_deployment = true ``` +The workspace.yaml file specifies the code locations to load when running +`dagster dev`. Because we don't have any code locations yet, it's empty except +for an example comment: + +```yaml +# This file contains the configuration for the workspace-- it should contain an entry in `load_from` +# for each code location. +# +# ##### EXAMPLE +# +# load_from: +# - python_file: +# relative_path: code_locations/my-code-location/my_code_location/definitions.py +# location_name: my_code_location +# executable_path: code_locations/my-code-location/.venv/bin/python +``` + To add a code location to the deployment, run: ```bash @@ -93,6 +111,17 @@ is_code_location = true is_component_lib = true ``` +We can also see that the `workspace.yaml` file has been updated to include the +new code location: + +```yaml +load_from: + - python_file: + relative_path: code_locations/code-location-1/code_location_1/definitions.py + location_name: code_location_1 + executable_path: code_locations/code-location-1/.venv/bin/python +``` + Let's enter this directory and search for registered component types: ```bash @@ -170,8 +199,8 @@ is because we are now using the environment of `code-location-2`, in which we have not installed `dagster-components[sling]`. For a final step, let's load up our two code locations with `dagster dev`. -We'll need a workspace.yaml to do this. Create a new file `workspace.yaml` in -the `my-deployment` directory: +Since `dg code-location scaffold` automatically updates it, our +`workspace.yaml` should already look like this: ```yaml load_from: @@ -189,7 +218,8 @@ And finally we'll run `dagster dev` to see your two code locations loaded up in UI. You may already have `dagster` installed in the ambient environment, in which case plain `dagster dev` will work. But in case you don't, we'll run `dagster dev` using `uv`, which will pull down and run `dagster` for you in -an isolated environment: +an isolated environment. Make sure you are in the code location root directory +and run (it will automatically pick up your `workspace.yaml`): ``` uv tool run --with=dagster-webserver dagster dev @@ -199,8 +229,6 @@ uv tool run --with=dagster-webserver dagster dev ![](/images/guides/build/projects-and-components/setting-up-a-deployment/two-code-locations.png) :::note -`dg` scaffolding functionality is currently under heavy development. In the -future we will construct this workspace.yaml file for you automatically in the -course of scaffolding code locations, and provide a `dg dev` command that -handles pulling down `dagster` for you in the background. +`dg` is currently under heavy development. In the future we will provide a `dg +dev` command that handles pulling down `dagster` for you in the background. ::: diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py index 5c2939bb19e57..f90445039d708 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py @@ -8,7 +8,7 @@ from dagster_dg.config import normalize_cli_config from dagster_dg.context import DgContext from dagster_dg.scaffold import scaffold_code_location -from dagster_dg.utils import DgClickCommand, DgClickGroup, exit_with_error +from dagster_dg.utils import DgClickCommand, DgClickGroup, exit_with_error, modify_yaml @click.group(name="code-location", cls=DgClickGroup) @@ -98,6 +98,30 @@ def code_location_scaffold_command( code_location_path, dg_context, editable_dagster_root, skip_venv=skip_venv ) + # Update workspace.yaml + if dg_context.is_deployment: + if not dg_context.workspace_yaml_path.exists(): + click.secho( + "Expected a workspace.yaml file in the deployment directory, but did not find one. Skipping workspace.yaml modification step.", + fg="yellow", + ) + else: + with modify_yaml(dg_context.workspace_yaml_path) as workspace_yaml: + workspace_yaml.setdefault("load_from", []) + entry = { + "relative_path": str( + code_location_path.relative_to(dg_context.deployment_root_path) + ), + "location_name": name, + } + if not skip_venv and dg_context.use_dg_managed_environment: + entry["executable_path"] = str( + (code_location_path / ".venv" / "bin" / "python").relative_to( + dg_context.deployment_root_path + ) + ) + workspace_yaml["load_from"].append(entry) + # ######################## # ##### LIST diff --git a/python_modules/libraries/dagster-dg/dagster_dg/context.py b/python_modules/libraries/dagster-dg/dagster_dg/context.py index 4ab489a5c174f..cb41c4fb25094 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/context.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/context.py @@ -159,6 +159,12 @@ def deployment_root_path(self) -> Path: raise DgError("Cannot find deployment configuration file") return deployment_config_path.parent + @property + def workspace_yaml_path(self) -> Path: + if not self.is_deployment: + raise DgError("`workspace_yaml_path` is only available in a deployment context") + return self.deployment_root_path / "workspace.yaml" + def has_code_location(self, name: str) -> bool: if not self.is_deployment: raise DgError( diff --git a/python_modules/libraries/dagster-dg/dagster_dg/templates/DEPLOYMENT_NAME_PLACEHOLDER/workspace.yaml.jinja b/python_modules/libraries/dagster-dg/dagster_dg/templates/DEPLOYMENT_NAME_PLACEHOLDER/workspace.yaml.jinja new file mode 100644 index 0000000000000..6b15e69462524 --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg/templates/DEPLOYMENT_NAME_PLACEHOLDER/workspace.yaml.jinja @@ -0,0 +1,10 @@ +# This file contains the configuration for the workspace-- it should contain an entry in `load_from` +# for each code location. +# +# ##### EXAMPLE +# +# load_from: +# - python_file: +# relative_path: code_locations/my-code-location/my_code_location/definitions.py +# location_name: my_code_location +# executable_path: code_locations/my-code-location/.venv/bin/python diff --git a/python_modules/libraries/dagster-dg/dagster_dg/utils.py b/python_modules/libraries/dagster-dg/dagster_dg/utils.py index 027a849988856..a33f6a8604e74 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/utils.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/utils.py @@ -14,6 +14,7 @@ import click import jinja2 +import yaml from typer.rich_utils import rich_format_help from typing_extensions import TypeAlias @@ -224,6 +225,17 @@ def _should_skip_file(path: str, excludes: list[str] = DEFAULT_FILE_EXCLUDE_PATT return False +@contextlib.contextmanager +def modify_yaml(path: Path) -> Iterator[dict[str, Any]]: + if not path.exists(): + raise DgError(f"File does not exist: {path}") + with open(path) as f: + # Return empty dict if file is empty + yaml_content = yaml.safe_load(f) or {} + yield yaml_content + path.write_text(yaml.dump(yaml_content)) + + def ensure_dagster_dg_tests_import() -> None: from dagster_dg import __file__ as dagster_dg_init_py diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py index 6702e4990742d..42fa7aab60781 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py @@ -3,6 +3,7 @@ import pytest import tomli +import yaml from dagster_dg.utils import discover_git_root, ensure_dagster_dg_tests_import, pushd ensure_dagster_dg_tests_import() @@ -28,12 +29,19 @@ # and returns the local version of the package. -def test_code_location_scaffold_inside_deployment_success(monkeypatch) -> None: +@pytest.mark.parametrize("with_workspace_yaml", [True, False]) +def test_code_location_scaffold_inside_deployment_success( + monkeypatch, with_workspace_yaml: bool +) -> None: # Remove when we are able to test without editable install dagster_git_repo_dir = discover_git_root(Path(__file__)) monkeypatch.setenv("DAGSTER_GIT_REPO_DIR", str(dagster_git_repo_dir)) with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): + # Delete workspace.yaml if we are testing without it + if not with_workspace_yaml: + Path("workspace.yaml").unlink() + result = runner.invoke("code-location", "scaffold", "foo-bar", "--use-editable-dagster") assert_runner_result(result) assert Path("code_locations/foo-bar").exists() @@ -47,6 +55,21 @@ def test_code_location_scaffold_inside_deployment_success(monkeypatch) -> None: assert Path("code_locations/foo-bar/.venv").exists() assert Path("code_locations/foo-bar/uv.lock").exists() + # Check workspace.yaml modified + if with_workspace_yaml: + workspace_yaml_path = Path("workspace.yaml") + assert workspace_yaml_path.exists() + workspace_yaml = yaml.safe_load(workspace_yaml_path.read_text()) + assert len(workspace_yaml["load_from"]) == 1 + assert workspace_yaml["load_from"][0] == { + "relative_path": "code_locations/foo-bar", + "location_name": "foo-bar", + "executable_path": "code_locations/foo-bar/.venv/bin/python", + } + else: + assert not Path("workspace.yaml").exists() + assert "Expected a workspace.yaml file" in result.output + # Restore when we are able to test without editable install # with open("code_locations/bar/pyproject.toml") as f: # toml = tomli.loads(f.read()) @@ -121,37 +144,55 @@ def test_code_location_scaffold_editable_dagster_success(mode: str, monkeypatch) def test_code_location_scaffold_skip_venv_success() -> None: - with ProxyRunner.test() as runner, runner.isolated_filesystem(): + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): result = runner.invoke("code-location", "scaffold", "--skip-venv", "foo-bar") assert_runner_result(result) - assert Path("foo-bar").exists() - assert Path("foo-bar/foo_bar").exists() - assert Path("foo-bar/foo_bar/lib").exists() - assert Path("foo-bar/foo_bar/components").exists() - assert Path("foo-bar/foo_bar_tests").exists() - assert Path("foo-bar/pyproject.toml").exists() + assert Path("code_locations/foo-bar").exists() + assert Path("code_locations/foo-bar/foo_bar").exists() + assert Path("code_locations/foo-bar/foo_bar/lib").exists() + assert Path("code_locations/foo-bar/foo_bar/components").exists() + assert Path("code_locations/foo-bar/foo_bar_tests").exists() + assert Path("code_locations/foo-bar/pyproject.toml").exists() # Check venv not created - assert not Path("foo-bar/.venv").exists() - assert not Path("foo-bar/uv.lock").exists() + assert not Path("code_locations/foo-bar/.venv").exists() + assert not Path("code_locations/foo-bar/uv.lock").exists() + + # Check workspace.yaml modified without executable_path + workspace_yaml_path = Path("workspace.yaml") + workspace_yaml = yaml.safe_load(workspace_yaml_path.read_text()) + assert len(workspace_yaml["load_from"]) == 1 + assert workspace_yaml["load_from"][0] == { + "relative_path": "code_locations/foo-bar", + "location_name": "foo-bar", + } def test_code_location_scaffold_no_use_dg_managed_environment_success() -> None: - with ProxyRunner.test() as runner, runner.isolated_filesystem(): + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): result = runner.invoke( "code-location", "scaffold", "--no-use-dg-managed-environment", "foo-bar" ) assert_runner_result(result) - assert Path("foo-bar").exists() - assert Path("foo-bar/foo_bar").exists() - assert Path("foo-bar/foo_bar/lib").exists() - assert Path("foo-bar/foo_bar/components").exists() - assert Path("foo-bar/foo_bar_tests").exists() - assert Path("foo-bar/pyproject.toml").exists() + assert Path("code_locations/foo-bar").exists() + assert Path("code_locations/foo-bar/foo_bar").exists() + assert Path("code_locations/foo-bar/foo_bar/lib").exists() + assert Path("code_locations/foo-bar/foo_bar/components").exists() + assert Path("code_locations/foo-bar/foo_bar_tests").exists() + assert Path("code_locations/foo-bar/pyproject.toml").exists() # Check venv not created - assert not Path("foo-bar/.venv").exists() - assert not Path("foo-bar/uv.lock").exists() + assert not Path("code_locations/foo-bar/.venv").exists() + assert not Path("code_locations/foo-bar/uv.lock").exists() + + # Check workspace.yaml modified without executable_path + workspace_yaml_path = Path("workspace.yaml") + workspace_yaml = yaml.safe_load(workspace_yaml_path.read_text()) + assert len(workspace_yaml["load_from"]) == 1 + assert workspace_yaml["load_from"][0] == { + "relative_path": "code_locations/foo-bar", + "location_name": "foo-bar", + } def test_code_location_scaffold_editable_dagster_no_env_var_no_value_fails(monkeypatch) -> None: diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_deployment_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_deployment_commands.py index 47801fd6592cc..6f9d341102673 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_deployment_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_deployment_commands.py @@ -18,6 +18,7 @@ def test_scaffold_deployment_command_success() -> None: assert_runner_result(result) assert Path("foo").exists() assert Path("foo/code_locations").exists() + assert Path("foo/workspace.yaml").exists() def test_scaffold_deployment_command_already_exists_fails() -> None: