Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: hatch build #65

Merged
merged 3 commits into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ jobs:
- run: git fetch origin main
- run: pre-commit run --from-ref origin/main --to-ref HEAD
- run: hatch fmt --check
- run: hatch version
- run: hatch build
- run: hatch test --all

coverage:
Expand All @@ -41,10 +43,10 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: 3.12
python-version: 3.11
cache: pip
- run: pip install hatch
- run: hatch test --cover
- run: hatch test -py 3.11 --cover
- uses: codecov/[email protected]
with:
verbose: true
Expand Down
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ __pycache__
/dist

#coverage reports
/.coverage
/.coverage*
/coverage.xml

#auto-generated
Expand Down
9 changes: 6 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -259,12 +259,15 @@ hatch test
hatch test --all
# sometimes working DBT project is required to verify user experience
hatch run scripts/generate.py
hatch run scripts/generate.py --help
# to generate DBT project with 100 models
hatch run scripts/generate.py 100
hatch run dbt build
# to validate dbt-pumpkin output visually (on test project generated above)
hatch run +dbt=1.8 dbt-pumpkin synchronize
hatch run +dbt=1.9 dbt-pumpkin synchronize
hatch run +dbt=1.8 test:dbt-pumpkin synchronize
hatch run +dbt=1.9 test:dbt-pumpkin synchronize
hatch run test:dbt-pumpkin bootstrap --dry-run
```

## Troubleshooting
Expand Down
28 changes: 19 additions & 9 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,26 +39,36 @@ artifacts = ["dbt_pumpkin/version.py"]
path = "dbt_pumpkin/version.py"

[tool.hatch.envs.default]
dependencies = [
"pytest",
"dbt-core~=1.9.0",
"dbt-duckdb~=1.9.0"
]
python = "3.12"

[tool.hatch.envs.default.env-vars]
EXPECTED_PYTHON_VERSION = "3.12"
EXPECTED_DBT_VERSION = "1.9"
DBT_PROJECT_DIR = ".dbt_project"
DBT_PROFILES_DIR = ".dbt_project"

[tool.hatch.envs.test]
python = "3.11"
dependencies = [
"pytest",
"dbt-core~={matrix:dbt}.0",
"dbt-duckdb~={matrix:dbt}.0"
]
installer = "uv"

[tool.hatch.envs.default.env-vars]
EXPECTED_PYTHON_VERSION = "{matrix:python}"
[tool.hatch.envs.test.env-vars]
EXPECTED_PYTHON_VERSION = "3.11"
EXPECTED_DBT_VERSION = "{matrix:dbt}"
DBT_PROJECT_DIR = ".dbt_project"
DBT_PROFILES_DIR = ".dbt_project"

[[tool.hatch.envs.default.matrix]]
python = ["3.12"]
dbt = ["1.8", "1.9"]

[[tool.hatch.envs.default.matrix]]
python = ["3.11"]
dbt = ["1.5", "1.6", "1.7"]
[[tool.hatch.envs.test.matrix]]
dbt = ["1.5", "1.6", "1.7", "1.8", "1.9"]

[tool.hatch.envs.hatch-test]
extra-dependencies = [
Expand Down
115 changes: 80 additions & 35 deletions scripts/generate.py
Original file line number Diff line number Diff line change
@@ -1,42 +1,87 @@
# /// script
# requires-python = ">=3.9"
# dependencies = [
# "click",
# ]
# ///

import pathlib
import shutil
import textwrap
from pathlib import Path

project_dir: Path = Path(__file__, "../../.dbt_project").resolve()

print(f"Will generate DBT project at {project_dir}")

project_dir.mkdir(parents=True, exist_ok=True)

(project_dir / "dbt_project.yml").write_text("""\
name: my_pumpkin
version: 1.0.0
profile: test_pumpkin
models:
my_pumpkin:
+dbt-pumpkin-path: "_schema/{name}.yml"
""")

(project_dir / "profiles.yml").write_text(f"""\
test_pumpkin:
target: test
outputs:
test:
# Comment to stop formatting in 1 line
type: duckdb
path: {project_dir}/test.duckdb
threads: 8
""")

models_dir = project_dir / "models"
models_dir.mkdir(parents=True, exist_ok=True)

for i in range(1, 1000):
model_path = models_dir / f"model_{i}.sql"
model_path.write_text("""\
select 1 as id
import click


@click.command
@click.option(
"--project-dir",
envvar="DBT_PROJECT_DIR",
default=".dbt_project",
type=click.Path(path_type=pathlib.Path, resolve_path=True),
)
@click.option(
"--profiles-dir",
envvar="DBT_PROFILES_DIR",
default=".dbt_project",
type=click.Path(path_type=pathlib.Path, resolve_path=True),
)
@click.option("--keep", is_flag=True, default=False, show_default=True, help="Don't prune project dir")
@click.argument("models", default=10, type=int)
def cli(*_, project_dir: Path, profiles_dir: Path, keep: bool, models: int):
"""
Generates simple DBT project which can be used for manual testing of dbt-pumpkin output
"""

print(f"Will generate DBT project at {project_dir}")

if not keep and project_dir.exists():
shutil.rmtree(project_dir)

project_dir.mkdir(parents=True, exist_ok=True)

(project_dir / "dbt_project.yml").write_text(
textwrap.dedent("""\
name: my_pumpkin
version: 1.0.0
profile: test_pumpkin
models:
my_pumpkin:
+dbt-pumpkin-path: "_schema/{name}.yml"
""")
)

print(f"Will generate DBT profiles at {profiles_dir}")

(profiles_dir / "profiles.yml").write_text(
textwrap.dedent(f"""\
test_pumpkin:
target: test
outputs:
test:
# Comment to stop formatting in 1 line
type: duckdb
path: {project_dir}/test.duckdb
threads: 8
""")
)

models_dir = project_dir / "models"
models_dir.mkdir(parents=True, exist_ok=True)

for i in range(1, models + 1):
model_path = models_dir / f"model_{i}.sql"
model_path.write_text(
textwrap.dedent("""\
select 1 as id
""")
)

print("Generated")


def main():
cli()


print("Generated")
if __name__ == "__main__":
main()
Loading