From 1b35c9920118b9be73a17326005f546ac5869073 Mon Sep 17 00:00:00 2001 From: David Ochoa Date: Thu, 14 Dec 2023 14:02:06 +0000 Subject: [PATCH] feat: yamllint to ensure yaml linting (#338) * feat: yamllint support * feat: updates yamllint rules --- .github/dependabot.yaml | 7 +--- .github/workflows/pr.yaml | 3 +- .github/workflows/prerelease.yaml | 3 +- .github/workflows/release.yaml | 2 +- .pre-commit-config.yaml | 61 +++++++++++++++++-------------- .vscode/extensions.json | 4 +- .vscode/settings.json | 6 +-- .yamllint | 59 ++++++++++++++++++++++++++++++ config/datasets/gcp.yaml | 3 +- poetry.lock | 22 ++++++++++- pyproject.toml | 1 + src/airflow/docker-compose.yaml | 1 - 12 files changed, 128 insertions(+), 44 deletions(-) create mode 100644 .yamllint diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index e593653f6..5354d671d 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -1,8 +1,5 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. version: 2 updates: - package-ecosystem: "pip" # See documentation for possible values diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f18fec180..57f560613 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,6 +1,7 @@ name: Checks -on: [push] +"on": + push: jobs: test: diff --git a/.github/workflows/prerelease.yaml b/.github/workflows/prerelease.yaml index 50c2c3829..f3ded11fb 100644 --- a/.github/workflows/prerelease.yaml +++ b/.github/workflows/prerelease.yaml @@ -1,7 +1,6 @@ ---- name: Test & pre-release -on: +"on": push: branches: - release/* diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2f35e8339..1e75318ed 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,6 +1,6 @@ name: Release and Documentation -on: +"on": push: branches: - main diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c8276836..2a7d9b00f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,13 @@ repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.6 + hooks: + - id: ruff + args: + - --fix + - id: ruff-format + files: ^((otg|utils|tests)/.+)?[^/]+\.py$ + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: @@ -8,11 +17,20 @@ repos: - id: check-merge-conflict - id: check-case-conflict - id: check-json + exclude: (.vscode|.devcontainer) + - id: no-commit-to-branch + args: + - --branch=dev + - --branch=main - id: check-toml - id: check-yaml - id: check-ast - id: debug-statements - id: check-docstring-first + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.32.0 + hooks: + - id: yamllint - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 @@ -27,14 +45,12 @@ repos: - id: pycln args: [--all] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.6 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.0.3" hooks: - - id: ruff - args: - - --fix - - id: ruff-format - files: ^((otg|utils|tests)/.+)?[^/]+\.py$ + - id: prettier + args: ["--staged"] + - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook rev: v9.10.0 hooks: @@ -46,20 +62,17 @@ repos: rev: "v1.7.1" hooks: - id: mypy - args: [ - # --ignore-missing-imports is required to run mypy on virtual - # environments without installed dependencies - "--ignore-missing-imports", - "--check-untyped-defs", - "--disallow-any-generics", - "--disallow-incomplete-defs", - "--no-implicit-optional", - "--no-implicit-reexport", - "--strict-equality", - "--warn-redundant-casts", - "--warn-unused-ignores", - "--disallow-untyped-defs", - ] + args: + - "--ignore-missing-imports" + - "--check-untyped-defs" + - "--disallow-any-generics" + - "--disallow-incomplete-defs" + - "--no-implicit-optional" + - "--no-implicit-reexport" + - "--strict-equality" + - "--warn-redundant-casts" + - "--warn-unused-ignores" + - "--disallow-untyped-defs" - repo: https://github.com/econchick/interrogate rev: 1.5.0 @@ -82,9 +95,3 @@ repos: rev: 0.3.8 hooks: - id: pydoclint - - - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v3.0.3" - hooks: - - id: prettier - args: ["--staged"] diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 5cdde1833..1be2f117e 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -3,7 +3,9 @@ "charliermarsh.ruff", "ms-python.mypy-type-checker", "ms-python.python", - "esbenp.prettier-vscode" + "esbenp.prettier-vscode", + "redhat.vscode-yaml", + "fnando.linter" ], "unwantedRecommendations": ["ms-python.flake8"] } diff --git a/.vscode/settings.json b/.vscode/settings.json index ae198ee2e..aaafda97f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,8 +4,8 @@ "editor.formatOnPaste": false, "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll": false, - "source.organizeImports": true + "source.fixAll": "never", + "source.organizeImports": "explicit" } }, "python.terminal.launchArgs": ["-m", "IPython", "--no-autoindent"], @@ -15,7 +15,7 @@ "editor.detectIndentation": false, "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" } }, "json.format.keepLines": true, diff --git a/.yamllint b/.yamllint new file mode 100644 index 000000000..0148408de --- /dev/null +++ b/.yamllint @@ -0,0 +1,59 @@ +rules: + braces: + level: error + min-spaces-inside: 0 + max-spaces-inside: 1 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + brackets: + level: error + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + colons: + level: error + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: error + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: + level: error + require-starting-space: true + min-spaces-from-content: 1 + comments-indentation: + level: error + document-end: + level: error + present: false + document-start: + level: error + present: false + empty-lines: + level: error + max: 1 + max-start: 0 + max-end: 1 + hyphens: + level: error + max-spaces-after: 1 + indentation: + level: error + spaces: 2 + indent-sequences: true + check-multi-line-strings: false + key-duplicates: + level: error + line-length: disable + new-line-at-end-of-file: + level: error + new-lines: + level: error + type: unix + trailing-spaces: + level: error + truthy: + level: error diff --git a/config/datasets/gcp.yaml b/config/datasets/gcp.yaml index 3edd7dd78..c198b4d96 100644 --- a/config/datasets/gcp.yaml +++ b/config/datasets/gcp.yaml @@ -13,7 +13,8 @@ jung: gs://genetics-portal-raw/pchic_jung2019/jung2019_pchic_tableS3.csv thurman: gs://genetics-portal-input/v2g_input/thurman2012/genomewideCorrs_above0.7_promoterPlusMinus500kb_withGeneNames_32celltypeCategories.bed8.gz catalog_associations: ${datasets.inputs}/v2d/gwas_catalog_v1.0.2-associations_e110_r2023-11-24.tsv catalog_studies: - # To get a complete representation of all GWAS Catalog studies, we need to ingest the list of unpublished studies from a different file. + # To get a complete representation of all GWAS Catalog studies, we need to + # ingest the list of unpublished studies from a different file. - ${datasets.inputs}/v2d/gwas-catalog-v1.0.3-studies-r2023-11-24.tsv - ${datasets.inputs}/v2d/gwas-catalog-v1.0.3-unpublished-studies-r2023-11-24.tsv catalog_ancestries: diff --git a/poetry.lock b/poetry.lock index 323c00727..e90d031a5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "aiodns" @@ -8336,6 +8336,24 @@ files = [ {file = "xyzservices-2023.10.1.tar.gz", hash = "sha256:091229269043bc8258042edbedad4fcb44684b0473ede027b5672ad40dc9fa02"}, ] +[[package]] +name = "yamllint" +version = "1.33.0" +description = "A linter for YAML files." +optional = false +python-versions = ">=3.8" +files = [ + {file = "yamllint-1.33.0-py3-none-any.whl", hash = "sha256:28a19f5d68d28d8fec538a1db21bb2d84c7dc2e2ea36266da8d4d1c5a683814d"}, + {file = "yamllint-1.33.0.tar.gz", hash = "sha256:2dceab9ef2d99518a2fcf4ffc964d44250ac4459be1ba3ca315118e4a1a81f7d"}, +] + +[package.dependencies] +pathspec = ">=0.5.3" +pyyaml = "*" + +[package.extras] +dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] + [[package]] name = "yarl" version = "1.9.2" @@ -8441,4 +8459,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "3.10.8" -content-hash = "0cc308744411ad7154a4601091ff2010fd4fae757441989d615dbcbd2d18c256" +content-hash = "cdc4614e4aaa54ee2aefbaad8e4a7793aba08d30043a3f29224ba10f406a6c3a" diff --git a/pyproject.toml b/pyproject.toml index d51091861..158a4017b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ pydoclint = "^0.3.8" prettier = "^0.0.7" deptry = "^0.12.0" python-semantic-release = "^8.3.0" +yamllint = "^1.33.0" [tool.semantic_release.branches.prerelease] match = "(release/*)" diff --git a/src/airflow/docker-compose.yaml b/src/airflow/docker-compose.yaml index 75bc3d171..8e8523490 100644 --- a/src/airflow/docker-compose.yaml +++ b/src/airflow/docker-compose.yaml @@ -43,7 +43,6 @@ # Default: '' # # Feel free to modify this file to suit your needs. ---- version: "3.8" x-airflow-common: &airflow-common # In order to add custom dependencies or upgrade provider packages you can use your extended image.