diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 694caaf..ca5b756 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,6 +33,8 @@ jobs: - run: git fetch origin main - run: pre-commit run --from-ref origin/main --to-ref HEAD - run: hatch fmt --check + - run: hatch version + - run: hatch build - run: hatch test --all coverage: @@ -41,10 +43,10 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.12 + python-version: 3.11 cache: pip - run: pip install hatch - - run: hatch test --cover + - run: hatch test -py 3.11 --cover - uses: codecov/codecov-action@v4.5.0 with: verbose: true diff --git a/.gitignore b/.gitignore index d5558f4..64c2e48 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ __pycache__ /dist #coverage reports -/.coverage +/.coverage* /coverage.xml #auto-generated diff --git a/README.md b/README.md index 9bb649b..a35a716 100644 --- a/README.md +++ b/README.md @@ -259,12 +259,15 @@ hatch test hatch test --all # sometimes working DBT project is required to verify user experience -hatch run scripts/generate.py +hatch run scripts/generate.py --help +# to generate DBT project with 100 models +hatch run scripts/generate.py 100 hatch run dbt build # to validate dbt-pumpkin output visually (on test project generated above) -hatch run +dbt=1.8 dbt-pumpkin synchronize -hatch run +dbt=1.9 dbt-pumpkin synchronize +hatch run +dbt=1.8 test:dbt-pumpkin synchronize +hatch run +dbt=1.9 test:dbt-pumpkin synchronize +hatch run test:dbt-pumpkin bootstrap --dry-run ``` ## Troubleshooting diff --git a/pyproject.toml b/pyproject.toml index 20661dd..f58aaa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,21 @@ artifacts = ["dbt_pumpkin/version.py"] path = "dbt_pumpkin/version.py" [tool.hatch.envs.default] +dependencies = [ + "pytest", + "dbt-core~=1.9.0", + "dbt-duckdb~=1.9.0" +] +python = "3.12" + +[tool.hatch.envs.default.env-vars] +EXPECTED_PYTHON_VERSION = "3.12" +EXPECTED_DBT_VERSION = "1.9" +DBT_PROJECT_DIR = ".dbt_project" +DBT_PROFILES_DIR = ".dbt_project" + +[tool.hatch.envs.test] +python = "3.11" dependencies = [ "pytest", "dbt-core~={matrix:dbt}.0", @@ -46,19 +61,14 @@ dependencies = [ ] installer = "uv" -[tool.hatch.envs.default.env-vars] -EXPECTED_PYTHON_VERSION = "{matrix:python}" +[tool.hatch.envs.test.env-vars] +EXPECTED_PYTHON_VERSION = "3.11" EXPECTED_DBT_VERSION = "{matrix:dbt}" DBT_PROJECT_DIR = ".dbt_project" DBT_PROFILES_DIR = ".dbt_project" -[[tool.hatch.envs.default.matrix]] -python = ["3.12"] -dbt = ["1.8", "1.9"] - -[[tool.hatch.envs.default.matrix]] -python = ["3.11"] -dbt = ["1.5", "1.6", "1.7"] +[[tool.hatch.envs.test.matrix]] +dbt = ["1.5", "1.6", "1.7", "1.8", "1.9"] [tool.hatch.envs.hatch-test] extra-dependencies = [ diff --git a/scripts/generate.py b/scripts/generate.py index bc4b9f4..782e153 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -1,42 +1,87 @@ # /// script # requires-python = ">=3.9" +# dependencies = [ +# "click", +# ] # /// - +import pathlib +import shutil +import textwrap from pathlib import Path -project_dir: Path = Path(__file__, "../../.dbt_project").resolve() - -print(f"Will generate DBT project at {project_dir}") - -project_dir.mkdir(parents=True, exist_ok=True) - -(project_dir / "dbt_project.yml").write_text("""\ -name: my_pumpkin -version: 1.0.0 -profile: test_pumpkin -models: - my_pumpkin: - +dbt-pumpkin-path: "_schema/{name}.yml" -""") - -(project_dir / "profiles.yml").write_text(f"""\ -test_pumpkin: - target: test - outputs: - test: - # Comment to stop formatting in 1 line - type: duckdb - path: {project_dir}/test.duckdb - threads: 8 -""") - -models_dir = project_dir / "models" -models_dir.mkdir(parents=True, exist_ok=True) - -for i in range(1, 1000): - model_path = models_dir / f"model_{i}.sql" - model_path.write_text("""\ - select 1 as id +import click + + +@click.command +@click.option( + "--project-dir", + envvar="DBT_PROJECT_DIR", + default=".dbt_project", + type=click.Path(path_type=pathlib.Path, resolve_path=True), +) +@click.option( + "--profiles-dir", + envvar="DBT_PROFILES_DIR", + default=".dbt_project", + type=click.Path(path_type=pathlib.Path, resolve_path=True), +) +@click.option("--keep", is_flag=True, default=False, show_default=True, help="Don't prune project dir") +@click.argument("models", default=10, type=int) +def cli(*_, project_dir: Path, profiles_dir: Path, keep: bool, models: int): + """ + Generates simple DBT project which can be used for manual testing of dbt-pumpkin output + """ + + print(f"Will generate DBT project at {project_dir}") + + if not keep and project_dir.exists(): + shutil.rmtree(project_dir) + + project_dir.mkdir(parents=True, exist_ok=True) + + (project_dir / "dbt_project.yml").write_text( + textwrap.dedent("""\ + name: my_pumpkin + version: 1.0.0 + profile: test_pumpkin + models: + my_pumpkin: + +dbt-pumpkin-path: "_schema/{name}.yml" + """) + ) + + print(f"Will generate DBT profiles at {profiles_dir}") + + (profiles_dir / "profiles.yml").write_text( + textwrap.dedent(f"""\ + test_pumpkin: + target: test + outputs: + test: + # Comment to stop formatting in 1 line + type: duckdb + path: {project_dir}/test.duckdb + threads: 8 """) + ) + + models_dir = project_dir / "models" + models_dir.mkdir(parents=True, exist_ok=True) + + for i in range(1, models + 1): + model_path = models_dir / f"model_{i}.sql" + model_path.write_text( + textwrap.dedent("""\ + select 1 as id + """) + ) + + print("Generated") + + +def main(): + cli() + -print("Generated") +if __name__ == "__main__": + main()