add windows testing

azure-pipelines.yml

@@ -9,7 +9,7 @@ parameters:
     type: object
     default:
       - "3.10"
-  - name: py3_env_suffixes
+  - name: dagster_core_py3_env_suffixes
     type: object
     default:
       - api_tests
@@ -26,23 +26,37 @@ jobs:
     strategy:
       matrix:
         ${{ each py_version in parameters.py3_versions }}:
-          ${{ each env_suffix in parameters.py3_env_suffixes }}:
+          ${{ each env_suffix in parameters.dagster_core_py3_env_suffixes }}:
             ${{ replace(py_version, '.', '') }}-windows-${{ env_suffix }}:
-              TOXENV: "py${{ replace(py_version, '.', '') }}-windows-${{ env_suffix }}"
-              python.version: "${{ py_version }}"
+              PACKAGE_ROOT: "python_modules\\dagster"
+              PATH_BACK_TO_REPO_ROOT: "..\\.."
+              TOX_ENV: "py${{ replace(py_version, '.', '') }}-windows-${{ env_suffix }}"
+              PYTHON_VERSION: "${{ py_version }}"
+
+        ${{ each py_version in parameters.py3_versions }}:
+          dagster-dg-${{ replace(py_version, '.', '') }}:
+            PACKAGE_ROOT: "python_modules\\libraries\\dagster-dg"
+            PATH_BACK_TO_REPO_ROOT: "..\\..\\.."
+            TOX_ENV: windows
+            PYTHON_VERSION: "${{ py_version }}"
     variables:
-      PYTHONLEGACYWINDOWSSTDIO: "1"
+      PYTHONUTF8: "1"
+    # Use PowerShell (`powershell`) instead of cmd shell (`script`) for better UTF-8 support.
     steps:
       - task: UsePythonVersion@0
         inputs:
-          versionSpec: "$(python.version)"
+          versionSpec: "$(PYTHON_VERSION)"
           architecture: "x64"
-      - script: pip install "tox<4.0.0" uv
+      - powershell: |
+          pip install "tox<4.0.0" uv
         displayName: "Install tox & uv"
-      - script: cd python_modules\dagster && tox -e %TOXENV% && cd ..\..
+      - powershell: |
+          cd $env:PACKAGE_ROOT
+          tox -e $env:TOX_ENV
+          cd $env:PATH_BACK_TO_REPO_ROOT
         displayName: "Run tests"
       - task: PublishTestResults@2
         inputs:
           testResultsFiles: "**/test_results.xml"
-          testRunTitle: "dagster $(TOXENV)"
+          testRunTitle: "$(PACKAGE_ROOT) $(TOX_ENV)"
         condition: succeededOrFailed()

python_modules/libraries/dagster-dg/dagster_dg/cache.py

@@ -1,9 +1,8 @@
 import hashlib
 import shutil
-import sys
 from collections.abc import Sequence
 from pathlib import Path
-from typing import Final, Literal, Optional, Union
+from typing import Final, Literal, Optional
 
 from typing_extensions import Self, TypeAlias
 
@@ -12,19 +11,19 @@
     DEFAULT_FILE_EXCLUDE_PATTERNS,
     hash_directory_metadata,
     hash_file_metadata,
+    is_macos,
+    is_windows,
 )
 
 _CACHE_CONTAINER_DIR_NAME: Final = "dg-cache"
 
-CachableDataType: TypeAlias = Union[
-    Literal["component_registry_data"], Literal["all_components_schema"]
-]
+CachableDataType: TypeAlias = Literal["component_registry_data", "all_components_schema"]
 
 
 def get_default_cache_dir() -> Path:
-    if sys.platform == "win32":
+    if is_windows():
         return Path.home() / "AppData" / "dg" / "cache"
-    elif sys.platform == "darwin":
+    elif is_macos():
         return Path.home() / "Library" / "Caches" / "dg"
     else:
         return Path.home() / ".cache" / "dg"

python_modules/libraries/dagster-dg/dagster_dg/completion.py

@@ -25,7 +25,7 @@ def install_completion(context: click.Context):
     shell, _ = shellingham.detect_shell()
     comp_class = click.shell_completion.get_completion_class(shell)
     if comp_class is None:
-        exit_with_error(f"Shell {shell} is not supported.")
+        exit_with_error(f"Shell `{shell}` is not supported.")
     else:
         comp_inst = comp_class(
             cli=context.command, ctx_args={}, prog_name="dg", complete_var="_DG_COMPLETE"

python_modules/libraries/dagster-dg/dagster_dg/config.py

@@ -1,4 +1,3 @@
-import sys
 from collections.abc import Mapping
 from dataclasses import dataclass, replace
 from pathlib import Path
@@ -9,17 +8,17 @@
 from click.core import ParameterSource
 
 from dagster_dg.error import DgError, DgValidationError
-from dagster_dg.utils import get_toml_value
+from dagster_dg.utils import get_toml_value, is_macos, is_windows
 
 T = TypeVar("T")
 
 DEFAULT_BUILTIN_COMPONENT_LIB = "dagster_components"
 
 
 def _get_default_cache_dir() -> Path:
-    if sys.platform == "win32":
+    if is_windows():
         return Path.home() / "AppData" / "dg" / "cache"
-    elif sys.platform == "darwin":
+    elif is_macos():
         return Path.home() / "Library" / "Caches" / "dg"
     else:
         return Path.home() / ".cache" / "dg"

python_modules/libraries/dagster-dg/dagster_dg/context.py

@@ -21,11 +21,12 @@
     get_executable_path,
     get_path_for_module,
     get_path_for_package,
-    get_uv_command_env,
     get_uv_run_executable_path,
+    get_venv_executable,
     is_executable_available,
     is_package_installed,
     pushd,
+    strip_activated_venv_from_env_vars,
 )
 
 # Deployment
@@ -130,7 +131,7 @@ def has_cache(self) -> bool:
         return self._cache is not None
 
     def get_cache_key(self, data_type: CachableDataType) -> tuple[str, str, str]:
-        path_parts = [str(part) for part in self.root_path.parts if part != "/"]
+        path_parts = [str(part) for part in self.root_path.parts if part != self.root_path.anchor]
         paths_to_hash = [
             self.root_path / "uv.lock",
             *([self.components_lib_path] if self.is_component_library else []),
@@ -214,7 +215,7 @@ def code_location_python_executable(self) -> Path:
             raise DgError(
                 "`code_location_python_executable` is only available in a code location context"
             )
-        return self.root_path / ".venv" / "bin" / "python"
+        return self.root_path / get_venv_executable(Path(".venv"))
 
     @cached_property
     def components_package_name(self) -> str:
@@ -302,7 +303,7 @@ def components_lib_path(self) -> Path:
     def external_components_command(self, command: list[str], log: bool = True) -> str:
         if self.use_dg_managed_environment:
             code_location_command_prefix = ["uv", "run", "dagster-components"]
-            env = get_uv_command_env()
+            env = strip_activated_venv_from_env_vars()
             executable_path = get_uv_run_executable_path("dagster-components")
         else:
             code_location_command_prefix = ["dagster-components"]
@@ -327,7 +328,7 @@ def ensure_uv_lock(self, path: Optional[Path] = None) -> None:
         path = path or self.root_path
         with pushd(path):
             if not (path / "uv.lock").exists():
-                subprocess.run(["uv", "sync"], check=True, env=get_uv_command_env())
+                subprocess.run(["uv", "sync"], check=True, env=strip_activated_venv_from_env_vars())
 
     @property
     def use_dg_managed_environment(self) -> bool:
@@ -344,3 +345,16 @@ def validate_registry_command_environment(self) -> None:
         """
         if not is_executable_available("dagster-components"):
             exit_with_error(MISSING_DAGSTER_COMPONENTS_ERROR_MESSAGE)
+
+    @cached_property
+    def venv_path(self) -> Path:
+        path = self.root_path
+        while path != path.parent:
+            if (path / ".venv").exists():
+                return path
+            path = path.parent
+        raise DgError("Cannot find .venv")
+
+    @cached_property
+    def dagster_components_executable(self) -> Path:
+        return get_venv_executable(self.venv_path, "dagster-components")

python_modules/libraries/dagster-dg/dagster_dg/scaffold.py

@@ -72,7 +72,7 @@ def get_pyproject_toml_dev_dependencies(use_editable_dagster: bool) -> str:
 
 def get_pyproject_toml_uv_sources(editable_dagster_root: Path) -> str:
     lib_lines = [
-        f'{path.name} = {{ path = "{path}", editable = true }}'
+        f"{path.name} = {{ path = '{path}', editable = true }}"
         for path in _gather_dagster_packages(editable_dagster_root)
     ]
     return "\n".join(
@@ -140,7 +140,7 @@ def scaffold_component_type(dg_context: DgContext, name: str) -> None:
     scaffold_subtree(
         path=root_path,
         name_placeholder="COMPONENT_TYPE_NAME_PLACEHOLDER",
-        templates_path=os.path.join(os.path.dirname(__file__), "templates", "COMPONENT_TYPE"),
+        templates_path=str(Path(__file__).parent / "templates" / "COMPONENT_TYPE"),
         project_name=name,
         component_type_class_name=camelcase(name),
         name=name,

python_modules/libraries/dagster-dg/dagster_dg/utils/__init__.py

@@ -31,6 +31,27 @@
 CLI_CONFIG_KEY = "config"
 
 
+def is_windows() -> bool:
+    return sys.platform == "win32"
+
+
+def is_macos() -> bool:
+    return sys.platform == "darwin"
+
+
+def get_venv_executable(venv_dir: Path, executable: str = "python") -> Path:
+    if is_windows():
+        return venv_dir / "Scripts" / f"{executable}.exe"
+    else:
+        return venv_dir / "bin" / executable
+
+
+def install_to_venv(venv_dir: Path, install_args: list[str]) -> None:
+    executable = get_venv_executable(venv_dir)
+    command = ["uv", "pip", "install", "--python", str(executable), *install_args]
+    subprocess.run(command, check=True)
+
+
 # Temporarily places a path at the front of sys.path, ensuring that any modules in that path are
 # importable.
 @contextlib.contextmanager
@@ -85,10 +106,16 @@ def is_executable_available(command: str) -> bool:
     return bool(shutil.which(command)) or bool(get_uv_run_executable_path(command))
 
 
+# Short for "normalize path"-- use this to get the platform-correct string representation of an
+# existing string path.
+def cross_platfrom_string_path(path: str):
+    return str(Path(path))
+
+
 # uv commands should be executed in an environment with no pre-existing VIRTUAL_ENV set. If this
 # variable is set (common during development) and does not match the venv resolved by uv, it prints
 # undesireable warnings.
-def get_uv_command_env() -> Mapping[str, str]:
+def strip_activated_venv_from_env_vars() -> Mapping[str, str]:
     return {k: v for k, v in os.environ.items() if not k == "VIRTUAL_ENV"}
 
 
@@ -100,6 +127,14 @@ def discover_git_root(path: Path) -> Path:
     raise ValueError("Could not find git root")
 
 
+def discover_venv(path: Path) -> Path:
+    while path != path.parent:
+        if (path / ".venv").exists():
+            return path
+        path = path.parent
+    raise ValueError("Could not find venv")
+
+
 @contextlib.contextmanager
 def pushd(path: Union[str, Path]) -> Iterator[None]:
     old_cwd = os.getcwd()
@@ -287,6 +322,11 @@ def exit_with_error(error_message: str) -> Never:
     sys.exit(1)
 
 
+# ########################
+# ##### ERROR MESSAGES
+# ########################
+
+
 def _format_error_message(message: str) -> str:
     # width=10000 unwraps any hardwrapping
     return textwrap.dedent(textwrap.fill(message, width=10000))

python_modules/libraries/dagster-dg/dagster_dg/utils/editor/__init__.py

@@ -2,17 +2,18 @@
 import os
 import shutil
 import subprocess
-import sys
 import zipfile
 from pathlib import Path
 
 import click
 
+from dagster_dg.utils import is_macos, is_windows
+
 
 def get_default_extension_dir() -> Path:
-    if sys.platform == "win32":
+    if is_windows():
         return Path.home() / "AppData" / "dg" / "vscode"
-    elif sys.platform == "darwin":
+    elif is_macos():
         return Path.home() / "Library" / "Application Support" / "dg" / "vscode"
     else:
         return Path.home() / ".local" / "share" / "dg" / "vscode"

python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/component_command_tests/test_component_commands.py

@@ -4,7 +4,11 @@
 from pathlib import Path
 
 import pytest
-from dagster_dg.utils import ensure_dagster_dg_tests_import, set_toml_value
+from dagster_dg.utils import (
+    cross_platfrom_string_path,
+    ensure_dagster_dg_tests_import,
+    set_toml_value,
+)
 
 ensure_dagster_dg_tests_import()
 
@@ -14,6 +18,7 @@
     isolated_example_code_location_foo_bar,
     isolated_example_deployment_foo,
     modify_pyproject_toml,
+    standardize_box_characters,
 )
 
 # ########################
@@ -26,6 +31,7 @@ def test_component_scaffold_dynamic_subcommand_generation() -> None:
         result = runner.invoke("component", "scaffold", "--help")
         assert_runner_result(result)
 
+        normalized_output = standardize_box_characters(result.output)
         # These are wrapped in a table so it's hard to check exact output.
         for line in [
             "╭─ Commands",
@@ -34,7 +40,7 @@ def test_component_scaffold_dynamic_subcommand_generation() -> None:
             "│ simple_asset@dagster_components.test",
             "│ simple_pipes_script_asset@dagster_components.test",
         ]:
-            assert line in result.output
+            assert standardize_box_characters(line) in normalized_output
 
 
 @pytest.mark.parametrize("in_deployment", [True, False])
@@ -245,14 +251,14 @@ def test_component_scaffold_succeeds_scaffolded_component_type() -> None:
 # ##### REAL COMPONENTS
 
 
-dbt_project_path = "../stub_code_locations/dbt_project_location/components/jaffle_shop"
+dbt_project_path = Path("../stub_code_locations/dbt_project_location/components/jaffle_shop")
 
 
 @pytest.mark.parametrize(
     "params",
     [
         ["--json-params", json.dumps({"project_path": str(dbt_project_path)})],
-        ["--project-path", dbt_project_path],
+        ["--project-path", str(dbt_project_path)],
     ],
 )
 def test_scaffold_dbt_project_instance(params) -> None:
@@ -277,7 +283,9 @@ def test_scaffold_dbt_project_instance(params) -> None:
         assert component_yaml_path.exists()
         assert "type: dbt_project@dagster_components" in component_yaml_path.read_text()
         assert (
-            "stub_code_locations/dbt_project_location/components/jaffle_shop"
+            cross_platfrom_string_path(
+                "stub_code_locations/dbt_project_location/components/jaffle_shop"
+            )
             in component_yaml_path.read_text()
         )