[components] Add workspace.yaml scaffolding

docs/docs-beta/docs/guides/build/components/setting-up-a-deployment.md

@@ -31,6 +31,7 @@ $ cd my-deployment && tree
 .
 ├── code_locations
 └── pyproject.toml
+└── workspace.yaml
 ```
 
 Importantly, the `pyproject.toml` file contains an `is_deployment` setting
@@ -43,6 +44,23 @@ marking this directory as a deployment:
 is_deployment = true
 ```
 
+The workspace.yaml file specifies the code locations to load when running
+`dagster dev`. Because we don't have any code locations yet, it's empty except
+for an example comment:
+
+```yaml
+# This file contains the configuration for the workspace-- it should contain an entry in `load_from`
+# for each code location.
+#
+# ##### EXAMPLE
+# 
+# load_from:
+#   - python_file:
+#       relative_path: code_locations/my-code-location/my_code_location/definitions.py
+#       location_name: my_code_location
+#       executable_path: code_locations/my-code-location/.venv/bin/python
+```
+
 To add a code location to the deployment, run:
 
 ```bash
@@ -93,6 +111,17 @@ is_code_location = true
 is_component_lib = true
 ```
 
+We can also see that the `workspace.yaml` file has been updated to include the
+new code location:
+
+```yaml
+load_from:
+  - python_file:
+      relative_path: code_locations/code-location-1/code_location_1/definitions.py
+      location_name: code_location_1
+      executable_path: code_locations/code-location-1/.venv/bin/python
+```
+
 Let's enter this directory and search for registered component types:
 
 ```bash
@@ -170,8 +199,8 @@ is because we are now using the environment of `code-location-2`, in which we
 have not installed `dagster-components[sling]`.
 
 For a final step, let's load up our two code locations with `dagster dev`.
-We'll need a workspace.yaml to do this. Create a new file `workspace.yaml` in
-the `my-deployment` directory:
+Since `dg code-location scaffold` automatically updates it, our
+`workspace.yaml` should already look like this:
 
 ```yaml
 load_from:
@@ -189,7 +218,8 @@ And finally we'll run `dagster dev` to see your two code locations loaded up in
 UI. You may already have `dagster` installed in the ambient environment, in
 which case plain `dagster dev` will work. But in case you don't, we'll run
 `dagster dev` using `uv`, which will pull down and run `dagster` for you in
-an isolated environment:
+an isolated environment. Make sure you are in the code location root directory
+and run (it will automatically pick up your `workspace.yaml`):
 
 ```
 uv tool run --with=dagster-webserver dagster dev
@@ -199,8 +229,6 @@ uv tool run --with=dagster-webserver dagster dev
 ![](/images/guides/build/projects-and-components/setting-up-a-deployment/two-code-locations.png)
 
 :::note
-`dg` scaffolding functionality is currently under heavy development. In the
-future we will construct this workspace.yaml file for you automatically in the
-course of scaffolding code locations, and provide a `dg dev` command that
-handles pulling down `dagster` for you in the background.
+`dg` is currently under heavy development. In the future we will provide a `dg
+dev` command that handles pulling down `dagster` for you in the background.
 :::

python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py

@@ -8,7 +8,7 @@
 from dagster_dg.config import normalize_cli_config
 from dagster_dg.context import DgContext
 from dagster_dg.scaffold import scaffold_code_location
-from dagster_dg.utils import DgClickCommand, DgClickGroup, exit_with_error
+from dagster_dg.utils import DgClickCommand, DgClickGroup, exit_with_error, modify_yaml
 
 
 @click.group(name="code-location", cls=DgClickGroup)
@@ -98,6 +98,30 @@ def code_location_scaffold_command(
         code_location_path, dg_context, editable_dagster_root, skip_venv=skip_venv
     )
 
+    # Update workspace.yaml
+    if dg_context.is_deployment:
+        if not dg_context.workspace_yaml_path.exists():
+            click.secho(
+                "Expected a workspace.yaml file in the deployment directory, but did not find one. Skipping workspace.yaml modification step.",
+                fg="yellow",
+            )
+        else:
+            with modify_yaml(dg_context.workspace_yaml_path) as workspace_yaml:
+                workspace_yaml.setdefault("load_from", [])
+                entry = {
+                    "relative_path": str(
+                        code_location_path.relative_to(dg_context.deployment_root_path)
+                    ),
+                    "location_name": name,
+                }
+                if not skip_venv and dg_context.use_dg_managed_environment:
+                    entry["executable_path"] = str(
+                        (code_location_path / ".venv" / "bin" / "python").relative_to(
+                            dg_context.deployment_root_path
+                        )
+                    )
+                workspace_yaml["load_from"].append(entry)
+
 
 # ########################
 # ##### LIST

python_modules/libraries/dagster-dg/dagster_dg/context.py

@@ -159,6 +159,12 @@ def deployment_root_path(self) -> Path:
             raise DgError("Cannot find deployment configuration file")
         return deployment_config_path.parent
 
+    @property
+    def workspace_yaml_path(self) -> Path:
+        if not self.is_deployment:
+            raise DgError("`workspace_yaml_path` is only available in a deployment context")
+        return self.deployment_root_path / "workspace.yaml"
+
     def has_code_location(self, name: str) -> bool:
         if not self.is_deployment:
             raise DgError(

python_modules/libraries/dagster-dg/dagster_dg/templates/DEPLOYMENT_NAME_PLACEHOLDER/workspace.yaml.jinja

@@ -0,0 +1,10 @@
+# This file contains the configuration for the workspace-- it should contain an entry in `load_from`
+# for each code location.
+#
+# ##### EXAMPLE
+# 
+# load_from:
+#   - python_file:
+#       relative_path: code_locations/my-code-location/my_code_location/definitions.py
+#       location_name: my_code_location
+#       executable_path: code_locations/my-code-location/.venv/bin/python

python_modules/libraries/dagster-dg/dagster_dg/utils.py

@@ -14,6 +14,7 @@
 
 import click
 import jinja2
+import yaml
 from typer.rich_utils import rich_format_help
 from typing_extensions import TypeAlias
 
@@ -224,6 +225,17 @@ def _should_skip_file(path: str, excludes: list[str] = DEFAULT_FILE_EXCLUDE_PATT
     return False
 
 
+@contextlib.contextmanager
+def modify_yaml(path: Path) -> Iterator[dict[str, Any]]:
+    if not path.exists():
+        raise DgError(f"File does not exist: {path}")
+    with open(path) as f:
+        # Return empty dict if file is empty
+        yaml_content = yaml.safe_load(f) or {}
+    yield yaml_content
+    path.write_text(yaml.dump(yaml_content))
+
+
 def ensure_dagster_dg_tests_import() -> None:
     from dagster_dg import __file__ as dagster_dg_init_py
 

python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py

@@ -3,6 +3,7 @@
 
 import pytest
 import tomli
+import yaml
 from dagster_dg.utils import discover_git_root, ensure_dagster_dg_tests_import, pushd
 
 ensure_dagster_dg_tests_import()
@@ -28,12 +29,19 @@
 # and returns the local version of the package.
 
 
-def test_code_location_scaffold_inside_deployment_success(monkeypatch) -> None:
+@pytest.mark.parametrize("with_workspace_yaml", [True, False])
+def test_code_location_scaffold_inside_deployment_success(
+    monkeypatch, with_workspace_yaml: bool
+) -> None:
     # Remove when we are able to test without editable install
     dagster_git_repo_dir = discover_git_root(Path(__file__))
     monkeypatch.setenv("DAGSTER_GIT_REPO_DIR", str(dagster_git_repo_dir))
 
     with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner):
+        # Delete workspace.yaml if we are testing without it
+        if not with_workspace_yaml:
+            Path("workspace.yaml").unlink()
+
         result = runner.invoke("code-location", "scaffold", "foo-bar", "--use-editable-dagster")
         assert_runner_result(result)
         assert Path("code_locations/foo-bar").exists()
@@ -47,6 +55,21 @@ def test_code_location_scaffold_inside_deployment_success(monkeypatch) -> None:
         assert Path("code_locations/foo-bar/.venv").exists()
         assert Path("code_locations/foo-bar/uv.lock").exists()
 
+        # Check workspace.yaml modified
+        if with_workspace_yaml:
+            workspace_yaml_path = Path("workspace.yaml")
+            assert workspace_yaml_path.exists()
+            workspace_yaml = yaml.safe_load(workspace_yaml_path.read_text())
+            assert len(workspace_yaml["load_from"]) == 1
+            assert workspace_yaml["load_from"][0] == {
+                "relative_path": "code_locations/foo-bar",
+                "location_name": "foo-bar",
+                "executable_path": "code_locations/foo-bar/.venv/bin/python",
+            }
+        else:
+            assert not Path("workspace.yaml").exists()
+            assert "Expected a workspace.yaml file" in result.output
+
         # Restore when we are able to test without editable install
         # with open("code_locations/bar/pyproject.toml") as f:
         #     toml = tomli.loads(f.read())
@@ -121,37 +144,55 @@ def test_code_location_scaffold_editable_dagster_success(mode: str, monkeypatch)
 
 
 def test_code_location_scaffold_skip_venv_success() -> None:
-    with ProxyRunner.test() as runner, runner.isolated_filesystem():
+    with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner):
         result = runner.invoke("code-location", "scaffold", "--skip-venv", "foo-bar")
         assert_runner_result(result)
-        assert Path("foo-bar").exists()
-        assert Path("foo-bar/foo_bar").exists()
-        assert Path("foo-bar/foo_bar/lib").exists()
-        assert Path("foo-bar/foo_bar/components").exists()
-        assert Path("foo-bar/foo_bar_tests").exists()
-        assert Path("foo-bar/pyproject.toml").exists()
+        assert Path("code_locations/foo-bar").exists()
+        assert Path("code_locations/foo-bar/foo_bar").exists()
+        assert Path("code_locations/foo-bar/foo_bar/lib").exists()
+        assert Path("code_locations/foo-bar/foo_bar/components").exists()
+        assert Path("code_locations/foo-bar/foo_bar_tests").exists()
+        assert Path("code_locations/foo-bar/pyproject.toml").exists()
 
         # Check venv not created
-        assert not Path("foo-bar/.venv").exists()
-        assert not Path("foo-bar/uv.lock").exists()
+        assert not Path("code_locations/foo-bar/.venv").exists()
+        assert not Path("code_locations/foo-bar/uv.lock").exists()
+
+        # Check workspace.yaml modified without executable_path
+        workspace_yaml_path = Path("workspace.yaml")
+        workspace_yaml = yaml.safe_load(workspace_yaml_path.read_text())
+        assert len(workspace_yaml["load_from"]) == 1
+        assert workspace_yaml["load_from"][0] == {
+            "relative_path": "code_locations/foo-bar",
+            "location_name": "foo-bar",
+        }
 
 
 def test_code_location_scaffold_no_use_dg_managed_environment_success() -> None:
-    with ProxyRunner.test() as runner, runner.isolated_filesystem():
+    with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner):
         result = runner.invoke(
             "code-location", "scaffold", "--no-use-dg-managed-environment", "foo-bar"
         )
         assert_runner_result(result)
-        assert Path("foo-bar").exists()
-        assert Path("foo-bar/foo_bar").exists()
-        assert Path("foo-bar/foo_bar/lib").exists()
-        assert Path("foo-bar/foo_bar/components").exists()
-        assert Path("foo-bar/foo_bar_tests").exists()
-        assert Path("foo-bar/pyproject.toml").exists()
+        assert Path("code_locations/foo-bar").exists()
+        assert Path("code_locations/foo-bar/foo_bar").exists()
+        assert Path("code_locations/foo-bar/foo_bar/lib").exists()
+        assert Path("code_locations/foo-bar/foo_bar/components").exists()
+        assert Path("code_locations/foo-bar/foo_bar_tests").exists()
+        assert Path("code_locations/foo-bar/pyproject.toml").exists()
 
         # Check venv not created
-        assert not Path("foo-bar/.venv").exists()
-        assert not Path("foo-bar/uv.lock").exists()
+        assert not Path("code_locations/foo-bar/.venv").exists()
+        assert not Path("code_locations/foo-bar/uv.lock").exists()
+
+        # Check workspace.yaml modified without executable_path
+        workspace_yaml_path = Path("workspace.yaml")
+        workspace_yaml = yaml.safe_load(workspace_yaml_path.read_text())
+        assert len(workspace_yaml["load_from"]) == 1
+        assert workspace_yaml["load_from"][0] == {
+            "relative_path": "code_locations/foo-bar",
+            "location_name": "foo-bar",
+        }
 
 
 def test_code_location_scaffold_editable_dagster_no_env_var_no_value_fails(monkeypatch) -> None:

python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_deployment_commands.py

@@ -18,6 +18,7 @@ def test_scaffold_deployment_command_success() -> None:
         assert_runner_result(result)
         assert Path("foo").exists()
         assert Path("foo/code_locations").exists()
+        assert Path("foo/workspace.yaml").exists()
 
 
 def test_scaffold_deployment_command_already_exists_fails() -> None: