From e9f84ae5a409c9cbcecd62a2a3b7ff75db70c6cc Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Fri, 7 Feb 2025 01:23:55 +0530 Subject: [PATCH] Moving yandex provider to new provider structure (#46525) * Moving yandex provider to new provider structure * fixup! Moving yandex provider to new provider structure --------- Co-authored-by: Jarek Potiuk --- .github/boring-cyborg.yml | 5 +- Dockerfile.ci | 11 +- dev/moving_providers/move_providers.py | 6 +- .../changelog.rst | 25 - .../commits.rst | 492 ------------------ .../configurations-ref.rst | 18 - .../connections/yandexcloud.rst | 95 ---- .../apache-airflow-providers-yandex/index.rst | 110 ---- .../installing-providers-from-sources.rst | 18 - .../operators/dataproc.rst | 37 -- .../operators/index.rst | 28 - .../operators/yq.rst | 28 - .../yandex-cloud-lockbox-secret-backend.rst | 298 ----------- .../security.rst | 18 - .../integration-logos/yandex/Yandex-Cloud.png | Bin 12347 -> 0 bytes .../yandex/.latest-doc-only-change.txt | 1 - .../airflow/providers/yandex/CHANGELOG.rst | 454 ---------------- providers/yandex/README.rst | 49 +- providers/yandex/docs/operators/dataproc.rst | 2 +- providers/yandex/docs/operators/yq.rst | 2 +- .../providers => }/yandex/provider.yaml | 11 +- providers/yandex/pyproject.toml | 79 +++ .../src/airflow/providers/yandex/LICENSE | 201 +++++++ .../src/airflow/providers/yandex/__init__.py | 0 .../providers/yandex/get_provider_info.py | 121 +++++ .../providers/yandex/hooks/__init__.py | 0 .../providers/yandex/hooks/dataproc.py | 0 .../airflow/providers/yandex/hooks/yandex.py | 0 .../src/airflow/providers/yandex/hooks/yq.py | 0 .../providers/yandex/links/__init__.py | 0 .../src/airflow/providers/yandex/links/yq.py | 0 .../providers/yandex/operators/__init__.py | 0 .../providers/yandex/operators/dataproc.py | 0 .../airflow/providers/yandex/operators/yq.py | 0 .../providers/yandex/secrets/__init__.py | 0 .../providers/yandex/secrets/lockbox.py | 4 +- .../providers/yandex/utils/__init__.py | 0 .../providers/yandex/utils/credentials.py | 0 .../providers/yandex/utils/defaults.py | 0 .../airflow/providers/yandex/utils/fields.py | 0 .../providers/yandex/utils/user_agent.py | 0 providers/yandex/tests/conftest.py | 32 ++ .../yandex/tests/provider_tests/__init__.py | 17 + .../tests/provider_tests}/yandex/__init__.py | 0 .../provider_tests/yandex/hooks}/__init__.py | 0 .../yandex/hooks/test_dataproc.py | 4 +- .../yandex/hooks/test_yandex.py | 4 +- .../provider_tests}/yandex/hooks/test_yq.py | 5 +- .../provider_tests/yandex/links}/__init__.py | 0 .../provider_tests}/yandex/links/test_yq.py | 0 .../yandex/operators}/__init__.py | 0 .../yandex/operators/test_dataproc.py | 4 +- .../yandex/operators/test_yq.py | 9 +- .../yandex/secrets}/__init__.py | 0 .../yandex/secrets/test_lockbox.py | 5 +- .../provider_tests/yandex/utils}/__init__.py | 0 .../yandex/utils/test_credentials.py | 0 .../yandex/utils/test_defaults.py | 0 .../yandex/utils/test_fields.py | 0 .../yandex/utils/test_user_agent.py | 4 +- .../tests/system/yandex}/__init__.py | 0 .../system/yandex/example_yandexcloud.py | 6 +- .../yandex/example_yandexcloud_dataproc.py | 0 ...xample_yandexcloud_dataproc_lightweight.py | 0 .../system/yandex/example_yandexcloud_yq.py | 0 pyproject.toml | 3 + scripts/ci/docker-compose/remove-sources.yml | 1 + scripts/ci/docker-compose/tests-sources.yml | 1 + scripts/docker/entrypoint_ci.sh | 11 +- 69 files changed, 541 insertions(+), 1678 deletions(-) delete mode 100644 docs/apache-airflow-providers-yandex/changelog.rst delete mode 100644 docs/apache-airflow-providers-yandex/commits.rst delete mode 100644 docs/apache-airflow-providers-yandex/configurations-ref.rst delete mode 100644 docs/apache-airflow-providers-yandex/connections/yandexcloud.rst delete mode 100644 docs/apache-airflow-providers-yandex/index.rst delete mode 100644 docs/apache-airflow-providers-yandex/installing-providers-from-sources.rst delete mode 100644 docs/apache-airflow-providers-yandex/operators/dataproc.rst delete mode 100644 docs/apache-airflow-providers-yandex/operators/index.rst delete mode 100644 docs/apache-airflow-providers-yandex/operators/yq.rst delete mode 100644 docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst delete mode 100644 docs/apache-airflow-providers-yandex/security.rst delete mode 100644 docs/integration-logos/yandex/Yandex-Cloud.png delete mode 100644 providers/src/airflow/providers/yandex/.latest-doc-only-change.txt delete mode 100644 providers/src/airflow/providers/yandex/CHANGELOG.rst rename providers/{src/airflow/providers => }/yandex/provider.yaml (92%) create mode 100644 providers/yandex/pyproject.toml create mode 100644 providers/yandex/src/airflow/providers/yandex/LICENSE rename providers/{ => yandex}/src/airflow/providers/yandex/__init__.py (100%) create mode 100644 providers/yandex/src/airflow/providers/yandex/get_provider_info.py rename providers/{ => yandex}/src/airflow/providers/yandex/hooks/__init__.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/hooks/dataproc.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/hooks/yandex.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/hooks/yq.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/links/__init__.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/links/yq.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/operators/__init__.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/operators/dataproc.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/operators/yq.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/secrets/__init__.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/secrets/lockbox.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/utils/__init__.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/utils/credentials.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/utils/defaults.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/utils/fields.py (100%) rename providers/{ => yandex}/src/airflow/providers/yandex/utils/user_agent.py (100%) create mode 100644 providers/yandex/tests/conftest.py create mode 100644 providers/yandex/tests/provider_tests/__init__.py rename providers/{tests/system => yandex/tests/provider_tests}/yandex/__init__.py (100%) rename providers/{tests/yandex => yandex/tests/provider_tests/yandex/hooks}/__init__.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/hooks/test_dataproc.py (99%) rename providers/{tests => yandex/tests/provider_tests}/yandex/hooks/test_yandex.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/hooks/test_yq.py (99%) rename providers/{tests/yandex/hooks => yandex/tests/provider_tests/yandex/links}/__init__.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/links/test_yq.py (100%) rename providers/{tests/yandex/links => yandex/tests/provider_tests/yandex/operators}/__init__.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/operators/test_dataproc.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/operators/test_yq.py (99%) rename providers/{tests/yandex/operators => yandex/tests/provider_tests/yandex/secrets}/__init__.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/secrets/test_lockbox.py (99%) rename providers/{tests/yandex/secrets => yandex/tests/provider_tests/yandex/utils}/__init__.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/utils/test_credentials.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/utils/test_defaults.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/utils/test_fields.py (100%) rename providers/{tests => yandex/tests/provider_tests}/yandex/utils/test_user_agent.py (100%) rename providers/{tests/yandex/utils => yandex/tests/system/yandex}/__init__.py (100%) rename providers/{ => yandex}/tests/system/yandex/example_yandexcloud.py (100%) rename providers/{ => yandex}/tests/system/yandex/example_yandexcloud_dataproc.py (100%) rename providers/{ => yandex}/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py (100%) rename providers/{ => yandex}/tests/system/yandex/example_yandexcloud_yq.py (100%) diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index adeb560a23ead..cf37c187c6ba9 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -304,10 +304,7 @@ labelPRBasedOnFilePath: - providers/weaviate/** provider:yandex: - - providers/src/airflow/providers/yandex/**/* - - docs/apache-airflow-providers-yandex/**/* - - providers/tests/yandex/**/* - - providers/tests/system/yandex/**/* + - providers/yandex/** provider:ydb: - providers/ydb/** diff --git a/Dockerfile.ci b/Dockerfile.ci index 27ec994edd5bf..baf31cab0314e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -930,17 +930,12 @@ function check_boto_upgrade() { echo echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud opensearch-py || true - # We need to include few dependencies to pass pip check with other dependencies: - # * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider) - # * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider) set -x # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ - "oss2>=2.14.0" "cryptography<43.0.0" "opensearch-py" + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore set +x - pip check } function check_downgrade_sqlalchemy() { diff --git a/dev/moving_providers/move_providers.py b/dev/moving_providers/move_providers.py index 13c1ef20cb586..da1213e4ffbc1 100755 --- a/dev/moving_providers/move_providers.py +++ b/dev/moving_providers/move_providers.py @@ -134,7 +134,11 @@ def _do_stuff( if from_path.exists(): shutil.move(from_path, to_path) console.print(f"\n[yellow]Moved {from_path} -> {to_path}\n") - if remove_empty_parent_dir and len([path for path in from_path.parent.iterdir()]) == 0: + if ( + remove_empty_parent_dir + and from_path.exists() + and len([path for path in from_path.parent.iterdir()]) == 0 + ): console.print(f"\n[yellow]Removed also empty parent dir {from_path.parent}\n") from_path.parent.rmdir() return diff --git a/docs/apache-airflow-providers-yandex/changelog.rst b/docs/apache-airflow-providers-yandex/changelog.rst deleted file mode 100644 index 9bcad616eb83d..0000000000000 --- a/docs/apache-airflow-providers-yandex/changelog.rst +++ /dev/null @@ -1,25 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE - OVERWRITTEN WHEN PREPARING PACKAGES. - - .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE - `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY - -.. include:: ../../providers/src/airflow/providers/yandex/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-yandex/commits.rst b/docs/apache-airflow-providers-yandex/commits.rst deleted file mode 100644 index 78d8068f45cbc..0000000000000 --- a/docs/apache-airflow-providers-yandex/commits.rst +++ /dev/null @@ -1,492 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE - OVERWRITTEN WHEN PREPARING PACKAGES. - - .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE - `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY - - .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! - -Package apache-airflow-providers-yandex ------------------------------------------------------- - -This package is for Yandex, including: - - - `Yandex.Cloud `__ - - -This is detailed commit list of changes for versions provider package: ``yandex``. -For high-level changelog, see :doc:`package information including changelog `. - - - -4.0.0 -..... - -Latest change: 2024-12-19 - -================================================================================================= =========== ======================================================================================== -Commit Committed Subject -================================================================================================= =========== ======================================================================================== -`35b927fe17 `_ 2024-12-19 ``Update path of example dags in docs (#45069)`` -`4b38bed76c `_ 2024-12-16 ``Bump min version of Providers to 2.9 (#44956)`` -`e786c78f52 `_ 2024-12-07 ``Remove Provider Deprecations in Yandex provider (#44754)`` -`1275fec92f `_ 2024-11-24 ``Use Python 3.9 as target version for Ruff & Black rules (#44298)`` -`4dfae23532 `_ 2024-11-15 ``Update DAG example links in multiple providers documents (#44034)`` -`a53d9f6d25 `_ 2024-11-14 ``Prepare docs for Nov 1st wave of providers (#44011)`` -`857ca4c06c `_ 2024-10-09 ``Split providers out of the main "airflow/" tree into a UV workspace project (#42505)`` -================================================================================================= =========== ======================================================================================== - -3.12.0 -...... - -Latest change: 2024-08-19 - -================================================================================================= =========== ======================================================================= -Commit Committed Subject -================================================================================================= =========== ======================================================================= -`75fb7acbac `_ 2024-08-19 ``Prepare docs for Aug 2nd wave of providers (#41559)`` -`fcbff15bda `_ 2024-08-12 ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)`` -`2daa5bd01a `_ 2024-08-04 ``providers/yandex: fix typing (#40997)`` -`d23881c648 `_ 2024-08-03 ``Prepare docs for Aug 1st wave of providers (#41230)`` -`09a7bd1d58 `_ 2024-07-09 ``Prepare docs 1st wave July 2024 (#40644)`` -`a62bd83188 `_ 2024-06-27 ``Enable enforcing pydocstyle rule D213 in ruff. (#40448)`` -================================================================================================= =========== ======================================================================= - -3.11.2 -...... - -Latest change: 2024-06-22 - -================================================================================================= =========== ========================================================================= -Commit Committed Subject -================================================================================================= =========== ========================================================================= -`6e5ae26382 `_ 2024-06-22 ``Prepare docs 2nd wave June 2024 (#40273)`` -`0d5bb60981 `_ 2024-06-17 ``Fix typos in Providers docs and Yandex hook (#40277)`` -`53e6739e67 `_ 2024-06-01 ``Limit yandex provider to avoid mypy errors (#39990)`` -`8173693a70 `_ 2024-05-31 ``Remove upper-binding in yandex after dataproc issue is fixed (#39974)`` -`b8a83b2293 `_ 2024-05-31 ``Workaround new yandexcloud breaking dataproc integration (#39964)`` -================================================================================================= =========== ========================================================================= - -3.11.1 -...... - -Latest change: 2024-05-26 - -================================================================================================= =========== ========================================================================= -Commit Committed Subject -================================================================================================= =========== ========================================================================= -`34500f3a2f `_ 2024-05-26 ``Prepare docs 3rd wave May 2024 (#39738)`` -`e0dd075d1b `_ 2024-05-15 `` AIP-21: yandexcloud: rename files, emit deprecation warning (#39618)`` -`defe4590e9 `_ 2024-05-11 ``yandex provider: bump version for yq http client package (#39548)`` -`2b1a2f8d56 `_ 2024-05-11 ``Reapply templates for all providers (#39554)`` -`2c05187b07 `_ 2024-05-10 ``Faster 'airflow_version' imports (#39552)`` -`05945a47f3 `_ 2024-05-09 ``add doc about Yandex Query operator (#39445)`` -`73918925ed `_ 2024-05-08 ``Simplify 'airflow_version' imports (#39497)`` -================================================================================================= =========== ========================================================================= - -3.11.0 -...... - -Latest change: 2024-05-01 - -================================================================================================= =========== ======================================================================= -Commit Committed Subject -================================================================================================= =========== ======================================================================= -`fe4605a10e `_ 2024-05-01 ``Prepare docs 1st wave May 2024 (#39328)`` -`ead9b00f7c `_ 2024-04-25 ``Bump minimum Airflow version in providers to Airflow 2.7.0 (#39240)`` -================================================================================================= =========== ======================================================================= - -3.10.0 -...... - -Latest change: 2024-04-13 - -================================================================================================= =========== ================================================================== -Commit Committed Subject -================================================================================================= =========== ================================================================== -`f9dcc82fb6 `_ 2024-04-13 ``Prepare docs 1st wave (RC2) April 2024 (#38995)`` -`5fa80b6aea `_ 2024-04-10 ``Prepare docs 1st wave (RC1) April 2024 (#38863)`` -`a9a6976dd2 `_ 2024-03-28 ``docs: yandex provider grammatical improvements (#38589)`` -`30817a5c6d `_ 2024-03-22 ``support iam token from metadata, simplify code (#38411)`` -`390bec1c82 `_ 2024-03-20 ``Add Yandex Query support from Yandex.Cloud (#37458)`` -`0a74928894 `_ 2024-03-18 ``Bump ruff to 0.3.3 (#38240)`` -`c0b849ad2b `_ 2024-03-11 ``Avoid use of 'assert' outside of the tests (#37718)`` -`83316b8158 `_ 2024-03-04 ``Prepare docs 1st wave (RC1) March 2024 (#37876)`` -`5a0be392e6 `_ 2024-02-16 ``Add comment about versions updated by release manager (#37488)`` -================================================================================================= =========== ================================================================== - -3.9.0 -..... - -Latest change: 2024-02-12 - -================================================================================================= =========== ================================================================================== -Commit Committed Subject -================================================================================================= =========== ================================================================================== -`bfb054e9e8 `_ 2024-02-12 ``Prepare docs 1st wave of Providers February 2024 (#37326)`` -`08036e5df5 `_ 2024-02-08 ``D401 Support in Providers (simple) (#37258)`` -`cea58c1111 `_ 2024-02-02 ``fix: using endpoint from connection if not specified (#37076)`` -`3ec781946a `_ 2024-02-01 ``Add secrets-backends section into the Yandex provider yaml definition (#37065)`` -`0e752383a8 `_ 2024-01-31 ``docs: update description in airflow provider.yaml (#37096)`` -================================================================================================= =========== ================================================================================== - -3.8.0 -..... - -Latest change: 2024-01-26 - -================================================================================================= =========== ==================================================================================================================== -Commit Committed Subject -================================================================================================= =========== ==================================================================================================================== -`cead3da4a6 `_ 2024-01-26 ``Add docs for RC2 wave of providers for 2nd round of Jan 2024 (#37019)`` -`0b680c9492 `_ 2024-01-26 ``Revert "Provide the logger_name param in providers hooks in order to override the logger name (#36675)" (#37015)`` -`12ccb5f0ac `_ 2024-01-25 ``feat: add Yandex Cloud Lockbox secrets backend (#36449)`` -`2b4da0101f `_ 2024-01-22 ``Prepare docs 2nd wave of Providers January 2024 (#36945)`` -`6ff96af480 `_ 2024-01-18 ``Fix stacklevel in warnings.warn into the providers (#36831)`` -`6bd450da1e `_ 2024-01-10 ``Provide the logger_name param in providers hooks in order to override the logger name (#36675)`` -`19ebcac239 `_ 2024-01-07 ``Prepare docs 1st wave of Providers January 2024 (#36640)`` -`6937ae7647 `_ 2023-12-30 ``Speed up autocompletion of Breeze by simplifying provider state (#36499)`` -================================================================================================= =========== ==================================================================================================================== - -3.7.1 -..... - -Latest change: 2023-12-23 - -================================================================================================= =========== ================================================================================== -Commit Committed Subject -================================================================================================= =========== ================================================================================== -`b15d5578da `_ 2023-12-23 ``Re-apply updated version numbers to 2nd wave of providers in December (#36380)`` -`f5883d6e7b `_ 2023-12-23 ``Prepare 2nd wave of providers in December (#36373)`` -`cd476acd8f `_ 2023-12-11 ``Follow BaseHook connection fields method signature in child classes (#36086)`` -================================================================================================= =========== ================================================================================== - -3.7.0 -..... - -Latest change: 2023-12-08 - -================================================================================================= =========== ======================================================================= -Commit Committed Subject -================================================================================================= =========== ======================================================================= -`999b70178a `_ 2023-12-08 ``Prepare docs 1st wave of Providers December 2023 (#36112)`` -`d0918d77ee `_ 2023-12-07 ``Bump minimum Airflow version in providers to Airflow 2.6.0 (#36017)`` -`0b23d5601c `_ 2023-11-24 ``Prepare docs 2nd wave of Providers November 2023 (#35836)`` -`99534e47f3 `_ 2023-11-19 ``Use reproducible builds for provider packages (#35693)`` -`99df205f42 `_ 2023-11-16 ``Fix and reapply templates for provider documentation (#35686)`` -================================================================================================= =========== ======================================================================= - -3.6.0 -..... - -Latest change: 2023-11-08 - -================================================================================================= =========== ================================================================== -Commit Committed Subject -================================================================================================= =========== ================================================================== -`1b059c57d6 `_ 2023-11-08 ``Prepare docs 1st wave of Providers November 2023 (#35537)`` -`706878ec35 `_ 2023-11-04 ``Remove empty lines in generated changelog (#35436)`` -`052e26ad47 `_ 2023-11-04 ``Change security.rst to use includes in providers (#35435)`` -`09880741cb `_ 2023-11-03 ``Add configuration files for yandex (#35420)`` -`0b850a97e8 `_ 2023-11-03 ``Yandex dataproc deduce default service account (#35059)`` -`d1c58d86de `_ 2023-10-28 ``Prepare docs 3rd wave of Providers October 2023 - FIX (#35233)`` -`3592ff4046 `_ 2023-10-28 ``Prepare docs 3rd wave of Providers October 2023 (#35187)`` -`dd7ba3cae1 `_ 2023-10-19 ``Pre-upgrade 'ruff==0.0.292' changes in providers (#35053)`` -================================================================================================= =========== ================================================================== - -3.5.0 -..... - -Latest change: 2023-10-13 - -================================================================================================= =========== =============================================================== -Commit Committed Subject -================================================================================================= =========== =============================================================== -`e9987d5059 `_ 2023-10-13 ``Prepare docs 1st wave of Providers in October 2023 (#34916)`` -`0c8e30e43b `_ 2023-10-05 ``Bump min airflow version of providers (#34728)`` -================================================================================================= =========== =============================================================== - -3.4.0 -..... - -Latest change: 2023-08-26 - -================================================================================================= =========== ====================================================================== -Commit Committed Subject -================================================================================================= =========== ====================================================================== -`c077d19060 `_ 2023-08-26 ``Prepare docs for Aug 2023 3rd wave of Providers (#33730)`` -`2ae1c10bfa `_ 2023-08-23 ``add support for Yandex Dataproc cluster labels (#29811)`` -`2b43fa473f `_ 2023-08-22 ``Resume yandex provider (#33574)`` -`73b90c48b1 `_ 2023-07-21 ``Allow configuration to be contributed by providers (#32604)`` -`3878fe6fab `_ 2023-07-05 ``Remove spurious headers for provider changelogs (#32373)`` -`09d4718d3a `_ 2023-06-27 ``Improve provider documentation and README structure (#32125)`` -`8b146152d6 `_ 2023-06-20 ``Add note about dropping Python 3.7 for providers (#32015)`` -`a59076eaee `_ 2023-06-02 ``Add D400 pydocstyle check - Providers (#31427)`` -`abea189022 `_ 2023-05-18 ``Use '__version__' in providers not 'version' (#31393)`` -`0a30706aa7 `_ 2023-05-03 ``Use 'AirflowProviderDeprecationWarning' in providers (#30975)`` -`eef5bc7f16 `_ 2023-05-03 ``Add full automation for min Airflow version for providers (#30994)`` -`a7eb32a5b2 `_ 2023-04-30 ``Bump minimum Airflow version in providers (#30917)`` -`b4d6e83686 `_ 2023-04-19 ``Suspend Yandex provider due to protobuf limitation (#30667)`` -`d23a3bbed8 `_ 2023-04-04 ``Add mechanism to suspend providers (#30422)`` -================================================================================================= =========== ====================================================================== - -3.3.0 -..... - -Latest change: 2023-03-03 - -================================================================================================= =========== ===================================================================== -Commit Committed Subject -================================================================================================= =========== ===================================================================== -`fcd3c0149f `_ 2023-03-03 ``Prepare docs for 03/2023 wave of Providers (#29878)`` -`1768872a00 `_ 2023-02-22 ``support Yandex SDK feature "endpoint" (#29635)`` -`2b92c3c74d `_ 2023-01-05 ``Fix providers documentation formatting (#28754)`` -`c8e348dcb0 `_ 2022-12-05 ``Add automated version replacement in example dag indexes (#28090)`` -================================================================================================= =========== ===================================================================== - -3.2.0 -..... - -Latest change: 2022-11-26 - -================================================================================================= =========== ==================================================================================== -Commit Committed Subject -================================================================================================= =========== ==================================================================================== -`25bdbc8e67 `_ 2022-11-26 ``Updated docs for RC3 wave of providers (#27937)`` -`2e20e9f7eb `_ 2022-11-24 ``Prepare for follow-up relase for November providers (#27774)`` -`12c3c39d1a `_ 2022-11-15 ``pRepare docs for November 2022 wave of Providers (#27613)`` -`78b8ea2f22 `_ 2022-10-24 ``Move min airflow version to 2.3.0 for all providers (#27196)`` -`2a34dc9e84 `_ 2022-10-23 ``Enable string normalization in python formatting - providers (#27205)`` -`837e463ae8 `_ 2022-10-22 ``Allow no extra prefix in yandex hook (#27040)`` -`f8db64c35c `_ 2022-09-28 ``Update docs for September Provider's release (#26731)`` -`06acf40a43 `_ 2022-09-13 ``Apply PEP-563 (Postponed Evaluation of Annotations) to non-core airflow (#26289)`` -================================================================================================= =========== ==================================================================================== - -3.1.0 -..... - -Latest change: 2022-08-10 - -================================================================================================= =========== =============================================================================== -Commit Committed Subject -================================================================================================= =========== =============================================================================== -`e5ac6c7cfb `_ 2022-08-10 ``Prepare docs for new providers release (August 2022) (#25618)`` -`a61e0c1df7 `_ 2022-07-29 ``YandexCloud provider: Support new Yandex SDK features for DataProc (#25158)`` -`d2459a241b `_ 2022-07-13 ``Add documentation for July 2022 Provider's release (#25030)`` -`0de31bd73a `_ 2022-06-29 ``Move provider dependencies to inside provider folders (#24672)`` -`510a6bab45 `_ 2022-06-28 ``Remove 'hook-class-names' from provider.yaml (#24702)`` -`08b675cf66 `_ 2022-06-13 ``Fix links to sources for examples (#24386)`` -================================================================================================= =========== =============================================================================== - -3.0.0 -..... - -Latest change: 2022-06-09 - -================================================================================================= =========== ================================================================================== -Commit Committed Subject -================================================================================================= =========== ================================================================================== -`dcdcf3a2b8 `_ 2022-06-09 ``Update release notes for RC2 release of Providers for May 2022 (#24307)`` -`717a7588bc `_ 2022-06-07 ``Update package description to remove double min-airflow specification (#24292)`` -`aeabe994b3 `_ 2022-06-07 ``Prepare docs for May 2022 provider's release (#24231)`` -`333e98759b `_ 2022-06-07 ``Fix link broken after #24082 (#24276)`` -`027b707d21 `_ 2022-06-05 ``Add explanatory note for contributors about updating Changelog (#24229)`` -`65ad2aed26 `_ 2022-06-01 ``Migrate Yandex example DAGs to new design AIP-47 (#24082)`` -================================================================================================= =========== ================================================================================== - -2.2.3 -..... - -Latest change: 2022-03-22 - -================================================================================================= =========== ============================================================== -Commit Committed Subject -================================================================================================= =========== ============================================================== -`d7dbfb7e26 `_ 2022-03-22 ``Add documentation for bugfix release of Providers (#22383)`` -================================================================================================= =========== ============================================================== - -2.2.2 -..... - -Latest change: 2022-03-14 - -================================================================================================= =========== ==================================================================== -Commit Committed Subject -================================================================================================= =========== ==================================================================== -`16adc035b1 `_ 2022-03-14 ``Add documentation for Classifier release for March 2022 (#22226)`` -================================================================================================= =========== ==================================================================== - -2.2.1 -..... - -Latest change: 2022-03-07 - -================================================================================================= =========== ========================================================================== -Commit Committed Subject -================================================================================================= =========== ========================================================================== -`f5b96315fe `_ 2022-03-07 ``Add documentation for Feb Providers release (#22056)`` -`6126c4e40f `_ 2022-03-07 ``Fix spelling (#22054)`` -`d94fa37830 `_ 2022-02-08 ``Fixed changelog for January 2022 (delayed) provider's release (#21439)`` -`6c3a67d4fc `_ 2022-02-05 ``Add documentation for January 2021 providers release (#21257)`` -`cb73053211 `_ 2022-01-27 ``Add optional features in providers. (#21074)`` -`602abe8394 `_ 2022-01-20 ``Remove ':type' lines now sphinx-autoapi supports typehints (#20951)`` -================================================================================================= =========== ========================================================================== - -2.2.0 -..... - -Latest change: 2021-12-31 - -================================================================================================= =========== ============================================================================================================ -Commit Committed Subject -================================================================================================= =========== ============================================================================================================ -`f77417eb0d `_ 2021-12-31 ``Fix K8S changelog to be PyPI-compatible (#20614)`` -`97496ba2b4 `_ 2021-12-31 ``Update documentation for provider December 2021 release (#20523)`` -`d56e7b56bb `_ 2021-12-30 ``Fix template_fields type to have MyPy friendly Sequence type (#20571)`` -`a0821235fb `_ 2021-12-30 ``Use typed Context EVERYWHERE (#20565)`` -`6e51608f28 `_ 2021-12-16 ``Fix mypy for providers: elasticsearch, oracle, yandex (#20344)`` -`41c49c7ff6 `_ 2021-12-14 ``YandexCloud provider: Support new Yandex SDK features: log_group_id, user-agent, maven packages (#20103)`` -`9a469d813f `_ 2021-11-30 ``Capitalize names in docs (#19893)`` -`853576d901 `_ 2021-11-30 ``Update documentation for November 2021 provider's release (#19882)`` -`d9567eb106 `_ 2021-10-29 ``Prepare documentation for October Provider's release (#19321)`` -`f5ad26dcdd `_ 2021-10-21 ``Fixup string concatenations (#19099)`` -`840ea3efb9 `_ 2021-09-30 ``Update documentation for September providers release (#18613)`` -`ef037e7021 `_ 2021-09-29 ``Static start_date and default arg cleanup for misc. provider example DAGs (#18597)`` -`e25eea052f `_ 2021-09-19 ``Inclusive Language (#18349)`` -`1cb456cba1 `_ 2021-09-12 ``Add official download page for providers (#18187)`` -`046f02e5a7 `_ 2021-09-09 ``fix misspelling (#18121)`` -================================================================================================= =========== ============================================================================================================ - -2.1.0 -..... - -Latest change: 2021-08-30 - -================================================================================================= =========== ============================================================================= -Commit Committed Subject -================================================================================================= =========== ============================================================================= -`0a68588479 `_ 2021-08-30 ``Add August 2021 Provider's documentation (#17890)`` -`be75dcd39c `_ 2021-08-23 ``Update description about the new ''connection-types'' provider meta-data`` -`76ed2a49c6 `_ 2021-08-19 ``Import Hooks lazily individually in providers manager (#17682)`` -`e3089dd5d0 `_ 2021-08-02 ``Add autoscaling subcluster support and remove defaults (#17033)`` -`87f408b1e7 `_ 2021-07-26 ``Prepares docs for Rc2 release of July providers (#17116)`` -`0dbd0f420c `_ 2021-07-26 ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` -`b916b75079 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` -`866a601b76 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` -================================================================================================= =========== ============================================================================= - -2.0.0 -..... - -Latest change: 2021-06-18 - -================================================================================================= =========== ======================================================================= -Commit Committed Subject -================================================================================================= =========== ======================================================================= -`bbc627a3da `_ 2021-06-18 ``Prepares documentation for rc2 release of Providers (#16501)`` -`cbf8001d76 `_ 2021-06-16 ``Synchronizes updated changelog after buggfix release (#16464)`` -`1fba5402bb `_ 2021-06-15 ``More documentation update for June providers release (#16405)`` -`9c94b72d44 `_ 2021-06-07 ``Updated documentation for June 2021 provider release (#16294)`` -`1e647029e4 `_ 2021-06-01 ``Rename the main branch of the Airflow repo to be 'main' (#16149)`` -`37681bca00 `_ 2021-05-07 ``Auto-apply apply_default decorator (#15667)`` -`807ad32ce5 `_ 2021-05-01 ``Prepares provider release after PIP 21 compatibility (#15576)`` -`40a2476a5d `_ 2021-04-28 ``Adds interactivity when generating provider documentation. (#15518)`` -`a7ca1b3b0b `_ 2021-03-26 ``Fix Sphinx Issues with Docstrings (#14968)`` -`e172bd0e16 `_ 2021-03-22 ``Update docstrings to adhere to sphinx standards (#14918)`` -`68e4c4dcb0 `_ 2021-03-20 ``Remove Backport Providers (#14886)`` -`6e6526a0f6 `_ 2021-03-13 ``Update documentation for broken package releases (#14734)`` -================================================================================================= =========== ======================================================================= - -1.0.1 -..... - -Latest change: 2021-02-04 - -================================================================================================= =========== ========================================================= -Commit Committed Subject -================================================================================================= =========== ========================================================= -`88bdcfa0df `_ 2021-02-04 ``Prepare to release a new wave of providers. (#14013)`` -`ac2f72c98d `_ 2021-02-01 ``Implement provider versioning tools (#13767)`` -`3fd5ef3555 `_ 2021-01-21 ``Add missing logos for integrations (#13717)`` -`295d66f914 `_ 2020-12-30 ``Fix Grammar in PIP warning (#13380)`` -`6cf76d7ac0 `_ 2020-12-18 ``Fix typo in pip upgrade command :( (#13148)`` -`f6448b4e48 `_ 2020-12-15 ``Add link to PyPI Repository to provider docs (#13064)`` -================================================================================================= =========== ========================================================= - -1.0.0 -..... - -Latest change: 2020-12-09 - -================================================================================================= =========== ====================================================================================================================================================================== -Commit Committed Subject -================================================================================================= =========== ====================================================================================================================================================================== -`32971a1a2d `_ 2020-12-09 ``Updates providers versions to 1.0.0 (#12955)`` -`b40dffa085 `_ 2020-12-08 ``Rename remaing modules to match AIP-21 (#12917)`` -`9b39f24780 `_ 2020-12-08 ``Add support for dynamic connection form fields per provider (#12558)`` -`bd90136aaf `_ 2020-11-30 ``Move operator guides to provider documentation packages (#12681)`` -`de3b1e687b `_ 2020-11-28 ``Move connection guides to provider documentation packages (#12653)`` -`ef4af21351 `_ 2020-11-22 ``Move providers docs to separate package + Spell-check in a common job with docs-build (#12527)`` -`f2569de7d1 `_ 2020-11-22 ``Add example DAGs to provider docs (#12528)`` -`c34ef853c8 `_ 2020-11-20 ``Separate out documentation building per provider (#12444)`` -`0080354502 `_ 2020-11-18 ``Update provider READMEs for 1.0.0b2 batch release (#12449)`` -`ae7cb4a1e2 `_ 2020-11-17 ``Update wrong commit hash in backport provider changes (#12390)`` -`6889a333cf `_ 2020-11-15 ``Improvements for operators and hooks ref docs (#12366)`` -`7825e8f590 `_ 2020-11-13 ``Docs installation improvements (#12304)`` -`85a18e13d9 `_ 2020-11-09 ``Point at pypi project pages for cross-dependency of provider packages (#12212)`` -`59eb5de78c `_ 2020-11-09 ``Update provider READMEs for up-coming 1.0.0beta1 releases (#12206)`` -`b2a28d1590 `_ 2020-11-09 ``Moves provider packages scripts to dev (#12082)`` -`4e8f9cc8d0 `_ 2020-11-03 ``Enable Black - Python Auto Formmatter (#9550)`` -`8c42cf1b00 `_ 2020-11-03 ``Use PyUpgrade to use Python 3.6 features (#11447)`` -`5a439e84eb `_ 2020-10-26 ``Prepare providers release 0.0.2a1 (#11855)`` -`872b1566a1 `_ 2020-10-25 ``Generated backport providers readmes/setup for 2020.10.29 (#11826)`` -`349b0811c3 `_ 2020-10-20 ``Add D200 pydocstyle check (#11688)`` -`16e7129719 `_ 2020-10-13 ``Added support for provider packages for Airflow 2.0 (#11487)`` -`0a0e1af800 `_ 2020-10-03 ``Fix Broken Markdown links in Providers README TOC (#11249)`` -`ca4238eb4d `_ 2020-10-02 ``Fixed month in backport packages to October (#11242)`` -`5220e4c384 `_ 2020-10-02 ``Prepare Backport release 2020.09.07 (#11238)`` -`5093245d6f `_ 2020-09-30 ``Strict type coverage for Oracle and Yandex provider (#11198)`` -`9549274d11 `_ 2020-09-09 ``Upgrade black to 20.8b1 (#10818)`` -`fdd9b6f65b `_ 2020-08-25 ``Enable Black on Providers Packages (#10543)`` -`3696c34c28 `_ 2020-08-24 ``Fix typo in the word "release" (#10528)`` -`ee7ca128a1 `_ 2020-08-22 ``Fix broken Markdown refernces in Providers README (#10483)`` -`f6734b3b85 `_ 2020-08-12 ``Enable Sphinx spellcheck for doc generation (#10280)`` -`cdec301254 `_ 2020-08-07 ``Add correct signature to all operators and sensors (#10205)`` -`aeea71274d `_ 2020-08-02 ``Remove 'args' parameter from provider operator constructors (#10097)`` -`7d24b088cd `_ 2020-07-25 ``Stop using start_date in default_args in example_dags (2) (#9985)`` -`d0e7db4024 `_ 2020-06-19 ``Fixed release number for fresh release (#9408)`` -`12af6a0800 `_ 2020-06-19 ``Final cleanup for 2020.6.23rc1 release preparation (#9404)`` -`c7e5bce57f `_ 2020-06-19 ``Prepare backport release candidate for 2020.6.23rc1 (#9370)`` -`40bf8f28f9 `_ 2020-06-18 ``Detect automatically the lack of reference to the guide in the operator descriptions (#9290)`` -`f6bd817a3a `_ 2020-06-16 ``Introduce 'transfers' packages (#9320)`` -`0b0e4f7a4c `_ 2020-05-26 ``Preparing for RC3 relase of backports (#9026)`` -`00642a46d0 `_ 2020-05-26 ``Fixed name of 20 remaining wrongly named operators. (#8994)`` -`1d36b0303b `_ 2020-05-23 ``Fix references in docs (#8984)`` -`375d1ca229 `_ 2020-05-19 ``Release candidate 2 for backport packages 2020.05.20 (#8898)`` -`12c5e5d8ae `_ 2020-05-17 ``Prepare release candidate for backport packages (#8891)`` -`f3521fb0e3 `_ 2020-05-16 ``Regenerate readme files for backport package release (#8886)`` -`92585ca4cb `_ 2020-05-15 ``Added automated release notes generation for backport operators (#8807)`` -`59a4f26699 `_ 2020-04-17 ``stop rendering some class docs in wrong place (#8095)`` -`3320e432a1 `_ 2020-02-24 ``[AIRFLOW-6817] Lazy-load 'airflow.DAG' to keep user-facing API untouched (#7517)`` -`4d03e33c11 `_ 2020-02-22 ``[AIRFLOW-6817] remove imports from 'airflow/__init__.py', replaced implicit imports with explicit imports, added entry to 'UPDATING.MD' - squashed/rebased (#7456)`` -`9cbd7de6d1 `_ 2020-02-18 ``[AIRFLOW-6792] Remove _operator/_hook/_sensor in providers package and add tests (#7412)`` -`ee1ab7697c `_ 2020-02-14 ``[AIRFLOW-6531] Initial Yandex.Cloud Dataproc support (#7252)`` -================================================================================================= =========== ====================================================================================================================================================================== diff --git a/docs/apache-airflow-providers-yandex/configurations-ref.rst b/docs/apache-airflow-providers-yandex/configurations-ref.rst deleted file mode 100644 index 5885c9d91b6e8..0000000000000 --- a/docs/apache-airflow-providers-yandex/configurations-ref.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-yandex/connections/yandexcloud.rst b/docs/apache-airflow-providers-yandex/connections/yandexcloud.rst deleted file mode 100644 index b1d8b4074c295..0000000000000 --- a/docs/apache-airflow-providers-yandex/connections/yandexcloud.rst +++ /dev/null @@ -1,95 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. _yandex_cloud_connection: - -Yandex.Cloud Connection -======================= - -The Yandex.Cloud connection type enables the authentication in Yandex.Cloud services. - -Configuring the Connection --------------------------- - -Service account auth JSON - JSON object as a string. - - Example: ``{"id": "...", "service_account_id": "...", "private_key": "..."}`` - -Service account auth JSON file path - Path to the file containing service account auth JSON. - - Example: ``/home/airflow/authorized_key.json`` - -OAuth Token - User account OAuth token as a string. - - Example: ``y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc`` - -SSH public key (optional) - The key will be placed to all created Compute nodes, allowing you to have a root shell there. - -Folder ID (optional) - A folder is an entity to separate different projects within the cloud. - - If specified, this ID will be used by default when creating nodes and clusters. - - See `this guide `__ for details. - -Endpoint (optional) - Use this setting to configure your API endpoint. - - Leave blank to use default `endpoints `__. - -Default Connection IDs ----------------------- - -All hooks and operators related to Yandex.Cloud use the ``yandexcloud_default`` connection by default. - -Authenticating to Yandex.Cloud ------------------------------- - -Using authorized keys to authorize as a service account -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Before you start, make sure you have `created `__ -a Yandex Cloud `service account `__. - -First, you need to create an `authorized key `__ -for your service account and save the generated JSON file with both public and private key parts. - -Then, you need to specify the key in the ``Service account auth JSON`` field. - -Alternatively, you can specify the path to the JSON file in the ``Service account auth JSON file path`` field. - -Using an OAuth token to authorize as a user account -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -First, you need to create -an `OAuth token `__ for your user account. -Your token will look like this: ``y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc``. - -Then you need to specify your token in the ``OAuth Token`` field. - -Using metadata service -~~~~~~~~~~~~~~~~~~~~~~ - -If you do not specify any credentials, the connection will attempt to use -the `metadata service `__ for authentication. - -To do this, you need to `link `__ -your service account with your VM. diff --git a/docs/apache-airflow-providers-yandex/index.rst b/docs/apache-airflow-providers-yandex/index.rst deleted file mode 100644 index 249542f23edc3..0000000000000 --- a/docs/apache-airflow-providers-yandex/index.rst +++ /dev/null @@ -1,110 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -``apache-airflow-providers-yandex`` -=================================== - - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Basics - - Home - Changelog - Security - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Guides - - Configuration - Connection types - Lockbox Secret Backend - Operators - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: References - - Python API <_api/airflow/providers/yandex/index> - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: System tests - - System Tests <_api/tests/system/yandex/index> - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Resources - - Example DAGs - PyPI Repository - Installing from sources - -.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! - - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Commits - - Detailed list of commits - - -apache-airflow-providers-yandex package ------------------------------------------------------- - -This package is for Yandex, including: - - - `Yandex.Cloud `__ - - -Release: 4.0.0 - -Provider package ----------------- - -This package is for the ``yandex`` provider. -All classes for this package are included in the ``airflow.providers.yandex`` python package. - -Installation ------------- - -You can install this package on top of an existing Airflow 2 installation via -``pip install apache-airflow-providers-yandex``. -For the minimum Airflow version supported, see ``Requirements`` below. - -Requirements ------------- - -The minimum Apache Airflow version supported by this provider package is ``2.9.0``. - -======================= ================== -PIP package Version required -======================= ================== -``apache-airflow`` ``>=2.9.0`` -``yandexcloud`` ``>=0.308.0`` -``yandex-query-client`` ``>=0.1.4`` -======================= ================== diff --git a/docs/apache-airflow-providers-yandex/installing-providers-from-sources.rst b/docs/apache-airflow-providers-yandex/installing-providers-from-sources.rst deleted file mode 100644 index b4e730f4ff21a..0000000000000 --- a/docs/apache-airflow-providers-yandex/installing-providers-from-sources.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-yandex/operators/dataproc.rst b/docs/apache-airflow-providers-yandex/operators/dataproc.rst deleted file mode 100644 index b7188e2ea52f6..0000000000000 --- a/docs/apache-airflow-providers-yandex/operators/dataproc.rst +++ /dev/null @@ -1,37 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - -Yandex.Cloud Data Proc Operators -================================ - -`Yandex Data Proc `__ is a service -that helps you deploy Apache Hadoop®* and Apache Spark™ clusters in the Yandex Cloud infrastructure. - -With Data Proc, you can manage the cluster size and node capacity, -as well as work with various Apache® services, -such as Spark, HDFS, YARN, Hive, HBase, Oozie, Sqoop, Flume, Tez, and Zeppelin. - -Apache Hadoop is used for storing and analyzing structured and unstructured big data. - -Apache Spark is a tool for quick data processing -that can be integrated with Apache Hadoop and other storage systems. - -Using the operators -^^^^^^^^^^^^^^^^^^^ -To learn how to use Data Proc operators, -see `example DAGs `_. diff --git a/docs/apache-airflow-providers-yandex/operators/index.rst b/docs/apache-airflow-providers-yandex/operators/index.rst deleted file mode 100644 index 12b05418e100f..0000000000000 --- a/docs/apache-airflow-providers-yandex/operators/index.rst +++ /dev/null @@ -1,28 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - - -Yandex.Cloud Operators -====================== - - -.. toctree:: - :maxdepth: 1 - :glob: - - * diff --git a/docs/apache-airflow-providers-yandex/operators/yq.rst b/docs/apache-airflow-providers-yandex/operators/yq.rst deleted file mode 100644 index 23bd4ac336160..0000000000000 --- a/docs/apache-airflow-providers-yandex/operators/yq.rst +++ /dev/null @@ -1,28 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - -Yandex Query Operators -====================== -`Yandex Query `__ is a service in the Yandex Cloud to process data from different sources such as -`Object Storage `__, `MDB ClickHouse `__, -`MDB PostgreSQL `__, `Yandex DataStreams `__ using SQL scripts. - -Using the operators -^^^^^^^^^^^^^^^^^^^ -To learn how to use Yandex Query operator, -see `example DAG `__. diff --git a/docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst b/docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst deleted file mode 100644 index f30346b24da80..0000000000000 --- a/docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst +++ /dev/null @@ -1,298 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - -Yandex.Cloud Lockbox Secret Backend -=================================== - -This topic describes how to configure Apache Airflow to use `Yandex Lockbox `__ -as a secret backend and how to manage secrets. - -Getting started ---------------- - -Before you start, make sure you have installed the ``yandex`` provider in your Apache Airflow installation: - -.. code-block:: bash - - pip install apache-airflow-providers-yandex - -Enabling the Yandex Lockbox secret backend ------------------------------------------- - -To enable Yandex Lockbox as a secret backend, -specify :py:class:`~airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend` -as your ``backend`` in the ``[secrets]`` section of the ``airflow.cfg`` file. - -Here is a sample configuration: - -.. code-block:: ini - - [secrets] - backend = airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend - -You can also set this with an environment variable: - -.. code-block:: bash - - export AIRFLOW__SECRETS__BACKEND=airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend - -You can verify whether the configuration options have been set up correctly -using the ``airflow config get-value`` command: - -.. code-block:: console - - $ airflow config get-value secrets backend - airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend - -Backend parameters ------------------- - -The next step is to configure backend parameters using the ``backend_kwargs`` options -that allow you to provide the following parameters: - -* ``yc_oauth_token``: Specifies the user account OAuth token to connect to Yandex Lockbox. The parameter value should look like ``y3_xx123``. -* ``yc_sa_key_json``: Specifies the service account key in JSON. The parameter value should look like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. -* ``yc_sa_key_json_path``: Specifies the service account key in JSON file path. The parameter value should look like ``/home/airflow/authorized_key.json``, while the file content should have the following format: ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. -* ``yc_connection_id``: Specifies the connection ID to connect to Yandex Lockbox. The default value is ``yandexcloud_default``. -* ``folder_id``: Specifies the folder ID to search for Yandex Lockbox secrets in. If set to ``None`` (``null`` in JSON), the requests will use the connection ``folder_id``, if specified. -* ``connections_prefix``: Specifies the prefix of the secret to read to get connections. If set to ``None`` (``null`` in JSON), the requests for connections will not be sent to Yandex Lockbox. The default value is ``airflow/connections``. -* ``variables_prefix``: Specifies the prefix of the secret to read to get variables. If set to ``None`` (``null`` in JSON), the requests for variables will not be sent to Yandex Lockbox. The default value is ``airflow/variables``. -* ``config_prefix``: Specifies the prefix of the secret to read to get configurations. If set to ``None`` (``null`` in JSON), the requests for variables will not be sent to Yandex Lockbox. The default value is ``airflow/config``. -* ``sep``: Specifies the separator to concatenate ``secret_prefix`` and ``secret_id``. The default value is ``/``. -* ``endpoint``: Specifies the API endpoint. If set to ``None`` (``null`` in JSON), the requests will use the connection endpoint, if specified; otherwise, they will use the default endpoint. - -Make sure to provide all options as a JSON dictionary. - -For example, if you want to set ``connections_prefix`` to ``"example-connections-prefix"`` -and ``variables_prefix`` to ``"example-variables-prefix"``, -your configuration file should look like this: - -.. code-block:: ini - - [secrets] - backend = airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend - backend_kwargs = {"connections_prefix": "example-connections-prefix", "variables_prefix": "example-variables-prefix"} - -Setting up credentials ----------------------- - -You need to specify credentials or the ID of the ``yandexcloud`` connection to connect to Yandex Lockbox. - -The credentials will be used with the following priority: - -* OAuth token -* Service account key in JSON from file -* Service account key in JSON -* Yandex Cloud connection - -If you do not specify any credentials, the system will use the default connection ID: ``yandexcloud_default``. - -Using an OAuth token to authorize as a user account -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -First, you need to create -an `OAuth token `__ for your user account. -Your token will look like this: ``y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc``. - -Then, you need to specify the ``folder_id`` and your token in ``backend_kwargs``: - -.. code-block:: ini - - [secrets] - backend_kwargs = {"folder_id": "b1g66mft1vo1n4vbn57j", "yc_oauth_token": "y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc"} - -Using authorized keys to authorize as a service account -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Before you start, make sure you have `created `__ -a Yandex Cloud `service account `__ -with the ``lockbox.viewer`` and ``lockbox.payloadViewer`` permissions. - -First, you need to create an `authorized key `__ -for your service account and save the generated JSON file with both public and private key parts. - -Then, you need to specify the ``folder_id`` and key in ``backend_kwargs``: - -.. code-block:: ini - - [secrets] - backend_kwargs = {"folder_id": "b1g66mft1vo1n4vbn57j", "yc_sa_key_json": {"id": "...", "service_account_id": "...", "private_key": "..."}"} - -Alternatively, you can specify the path to the JSON file in ``backend_kwargs``: - -.. code-block:: ini - - [secrets] - backend_kwargs = {"folder_id": "b1g66mft1vo1n4vbn57j", "yc_sa_key_json_path": "/home/airflow/authorized_key.json"} - -Using Yandex Cloud connection for authorization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -First, you need to create :ref:`Yandex Cloud connection `. - -Then, you need to specify the ``connection_id`` in ``backend_kwargs``: - -.. code-block:: ini - - [secrets] - backend_kwargs = {"yc_connection_id": "my_yc_connection"} - -If you do not specify any credentials, -Lockbox Secret Backend will try to use the default connection ID: ``yandexcloud_default``. - -Lockbox Secret Backend will try to use the default folder ID from your connection. -You can also specify the ``folder_id`` in the ``backend_kwargs``: - -.. code-block:: ini - - [secrets] - backend_kwargs = {"folder_id": "b1g66mft1vo1n4vbn57j", "yc_connection_id": "my_yc_connection"} - -Storing and retrieving connections ----------------------------------- - -To store a connection, you need to `create a secret `__ -with a name in the following format: ``{connections_prefix}{sep}{connection_name}``. - -The payload must contain a text value with any key. - -Storing a connection as a URI -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The main way to save connections is using a :ref:`connection URI representation `, such as -``mysql://myname:mypassword@myhost.com?this_param=some+val&that_param=other+val%2A``. - -Here is an example of creating a secret with the ``yc`` CLI: - -.. code-block:: console - - $ yc lockbox secret create \ - --name airflow/connections/mysqldb \ - --payload '[{"key": "value", "text_value": "mysql://myname:mypassword@myhost.com?this_param=some+val&that_param=other+val%2A"}]' - done (1s) - name: airflow/connections/mysqldb - -Storing a connection as JSON -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Another way to store connections is using JSON format: - -.. code-block:: json - - { - "conn_type": "mysql", - "host": "host.com", - "login": "myname", - "password": "mypassword", - "extra": { - "this_param": "some val", - "that_param": "other val*" - } - } - -Here is an example of creating a secret with the ``yc`` CLI: - -.. code-block:: console - - $ yc lockbox secret create \ - --name airflow/connections/my_sql_db_json \ - --payload '[{"key": "value", "text_value": "{\"conn_type\": \"mysql\", \"host\": \"host.com\", \"login\": \"myname\", \"password\": \"mypassword\", \"extra\": {\"this_param\": \"some val\", \"that_param\": \"other val*\"}}"}]' - done (1s) - name: airflow/connections/my_sql_db_json - -Retrieving connection -~~~~~~~~~~~~~~~~~~~~~ - -To check the connection is correctly read from the Lockbox Secret Backend, you can use ``airflow connections get``: - -.. code-block:: console - - $ airflow connections get mysqldb -o json - [{"id": null, "conn_id": "mysqldb", "conn_type": "mysql", "description": null, "host": "host.com", "schema": "", "login": "myname", "password": "mypassword", "port": null, "is_encrypted": "False", "is_extra_encrypted": "False", "extra_dejson": {"this_param": "some val", "that_param": "other val*"}, "get_uri": "mysql://myname:mypassword@myhost.com/?this_param=some+val&that_param=other+val%2A"}] - -Storing and retrieving variables --------------------------------- - -To store a variable, you need to `create a secret `__ -with a name in the following format: ``{variables_prefix}{sep}{variable_name}``. -The payload must contain a text value with any key. - -Here is how a variable value may look like: ``some_secret_data``. - -Here is an example of creating a secret with the ``yc`` CLI: - -.. code-block:: console - - $ yc lockbox secret create \ - --name airflow/variables/my_variable \ - --payload '[{"key": "value", "text_value": "some_secret_data"}]' - done (1s) - name: airflow/variables/my_variable - -To check the variable is correctly read from the Lockbox Secret Backend, you can use ``airflow variables get``: - -.. code-block:: console - - $ airflow variables get my_variable - some_secret_data - -Storing and retrieving configs ------------------------------- - -Lockbox Secret Backend is also suitable for storing sensitive configurations. - -For example, we will provide you with a secret for ``sentry.sentry_dsn`` -and use ``sentry_dsn_value`` as the config value name. - -To store a config, you need to `create a secret `__ -with a name in the following format: ``{config_prefix}{sep}{config_value_name}``. -The payload must contain a text value with any key. - -Here is an example of creating a secret with the ``yc`` CLI: - -.. code-block:: console - - $ yc lockbox secret create \ - --name airflow/config/sentry_dsn_value \ - --payload '[{"key": "value", "text_value": "https://public@sentry.example.com/1"}]' - done (1s) - name: airflow/config/sentry_dsn_value - -Then, we need to specify the config value name as ``{key}_secret`` in the Apache Airflow configuration: - -.. code-block:: ini - - [sentry] - sentry_dsn_secret = sentry_dsn_value - -To check the config value is correctly read from the Lockbox Secret Backend, you can use ``airflow config get-value``: - -.. code-block:: console - - $ airflow config get-value sentry sentry_dsn - https://public@sentry.example.com/1 - -Cleaning up your secret ------------------------ - -You can easily delete your secret with the ``yc`` CLI: - -.. code-block:: console - - $ yc lockbox secret delete --name airflow/connections/mysqldb - name: airflow/connections/mysqldb diff --git a/docs/apache-airflow-providers-yandex/security.rst b/docs/apache-airflow-providers-yandex/security.rst deleted file mode 100644 index afa13dac6fc9b..0000000000000 --- a/docs/apache-airflow-providers-yandex/security.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: ../exts/includes/security.rst diff --git a/docs/integration-logos/yandex/Yandex-Cloud.png b/docs/integration-logos/yandex/Yandex-Cloud.png deleted file mode 100644 index 33e8e1b71b2c920f442e2ee66e810a9c5c34559a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12347 zcmZvjbyOTN7wBPleQG zFT^i6U3c?%{*91r7olwqDRX5yNvSUi$72BN-N|+5vv)%{OuR=>Jw4By@9UTRFQCXS#~axda^W+CUz~0OCaHN8(yZ-yApIY9LtUxxJ7GzmRT>`k6|f zMt;m{(vTRJftD5?E};@%LBTsS5iKb}BSaq9;R=l=f%&7NI9Rb__QfK(DTg7T0UyJ{vZd&bALOPBx zVl;?5llO~zAnCqmpn3YV6a40Fim7#jK01S=z*54d#i$S^Pi{82Q`}rk)F$Rj%!rD3 z*^eW*HpJcza+sjAjrVn6&APgZA|#_V%z!zRg(ePfB5LpUpAPeojq)ME9+D6+^K$b3$_!!3X>KGedQfGT>5GNE(5n+i%;y{gslE24Z}fQ_qLHvsCPxw4KBW)UDop|BWb$K z$J1xkg~)kxb9t$9hA%w&ffL1jIeV==GBg(P0JQ*VVlh;<2tPJ_60kfcXQ}%JN#4#9 z6@v4xiSt9JFsE|X;0gIG=EtqQ<1T(d}()aZQ zR;{!9(G3eqOwP);WFwSX*5bxb0p-(Uq9bF(53u?B8jND_Yqjisf4@1PG)8EQ!|fHW zq*F$&%|%7p@EaFugZ^oQCG%K5Fbo}`!rDX^9XgBO{V?ifL$W44QqWDbK?%7rGsm+NO-GH z+Kg}P;z2U(l;7vi7jXrwwc@oDS25x-7w-`v%IH-;bQf-ARU}-rag)`^8$z+d;m|G+ zs&JamZx8|g5GGl)QFc5q8dR{JR@!Yba`zhv$mQ1VBeC=y#o>DrvStmeC7WlGQ7VjS zPby+8Q)PIkfKD5axF%}2Q3>aSXl{+Uwz#mrFyB!s#jioTYJ~JEK#d!*=qo_G*^pwJ z#!4U;uQPzHZ(`H>R@w9;W-lOSZ#L84jrJ*`p2tNvo`6n!RChtLq#ch=xFkmG`W?Sa zN;3mhzq-VZp8}!evvQBcc9dBy_ooVlUK@fzF6uq+lR%dME@@xszaXwk$SpS-bxd**QXZMJ{hC3%J!gzKllj)U=*M&TR6mFNOtEq@W4E^Lpk`$)PX6Y7l zL9q9j$pc+TE|K;(0&?P4)oqqsHZU=i=&t1UqC_AIzDS2XQE5fLu1N=560*AUKy0XmvaAQ5Ag-7t3!bfq1gDZPpR%HGgj)QQJ&OPXE z=XUbmpliE?b{LFGmBZtI%{SCHM=@um`TdBwJZxT(EBhAe6H2W zK;=>)5>}eFOUqd%gk7aujY)e{S=drF177+#Tw$j~F+DDfqk(9vd5IqABj$s@)$S&EqcU zG_47Yc^OoomG~CEWWwY2F$H?;jW~>cs?t(oY47Bhn+%;F1BL%&6lhmwc zqb}1R+>zq4%ge2L`7_3yhw~(!ciVT*mAf~}Rqv>wiHOu|^pClk#eTv?_e2_hI&@ z0`7mB>*Y{{{ek0y5=5b@otco)BVot)Wnw#n? z$+^{qV^L)yyW^k|PmDOJf)4k%>{SySVFzyKTMmj=KYo=_%^VUp+^3ktOWs_*l3iUW(Uf%Yk$(10xS@1WT zGmFpwnI`OlOFQISv)}J_yZx|LzH|?OEX4%M?Toumv1{s8orYIfx&X@e$gJZ1(=v{b8-eBDcJ-1|4rx zlH6Y__6}BlxfK4x;kM*7a#QU3>m$s~{GP*5AwJ9zsdk2#!DgecR5*@v(bh9crXABI|Zjxi*4y_nS?w z3w5nkkyA|>;AJKYNGH(+>tqink=`!3HZMEDhluM*kw9_+<~<2&W_S-u2?qC=Vkmx{ za(F*bC-j(x4dHG-%g{_&Vdh+9xPBZJ5=P|YEIHuIL8o1E;>Plx?}uF$$|mhDE~*3D z+F&0}Dm6#&$6YBaT6!5H zjJed(hRS1;!qNB&%Hepl6N&);w8He*PEmI%bjMN+H>e6_ZGU{hA$O2`^{e)VgxzlH zA)aiW{f6lb+WD_X; z9V=km)6Y&zKq?N=F_3P!#iSlhXV{&KEqWZX9`3pRy08%ldJ+8)&^weVPm` z#AXq%p)Ru}@R=u0J+L%9rifs0pyUr5v3nOz!5rpa(SGI%ZEu|jR~W5mb&Fc7CsH@_ zu6DKyZse9-3Ljj=8l5Z`6|z1fe73>HS7+BY<=8CWC56}ZQcQTyna{F{YN@kLZ-GqE zxP0Nvt>61Gv3|NA`vp4=xmQl7;sg~A*xP)TQyLV*o0ylpF}i*{N9#-~Q2 zL7lc6p_5haW?eRd!Rp`@oD7+eF!MW11XNHCPHhGsNS0U{ZWkTqaXCEf>wz2 z9Sw^j>mdCe_OVq|^3f#8rAADF&uPjgDL((K$K$a7n`#!_C zR5o_i#fkne4T<(3RP)zM867sRCaz4%G%MC$f0^0rhdb@Tp*)^J0?v{~?9IbrO?gKy zF1#6b0?@)yGn}vyLP~`t0~O71dLrqcKs`3oFkJ)vhU5ZQ(E-7j>hc~yw(7h*GXzeT zUR!g3ac8J|hG&c9g==W5^l}}0Z%THFSvVy-Ez%JC2$DJ_lIAhatl<=~fybb*iVR5& z>#cDP^R5O0?xS?u%$-%=n=rkxsiip*PyMFBZ*LV%NBmMW6m$E+IW&U)w--L!i8K^) zD@v7nC1Q>s7)Gm5P7~1t1ibhcEz`IgQz*s-ns*r#d#BP2|2zyj%~#KxCd?5hGr6i(T*oIVo*_&2;5+c$s( zBu$w}3)!AVqwf5@UoD(xtk~7j=1iugzd!<2IACLLMt7M!zNVx=w}BscU(y%=e%A`a zuKqn2OxcP;-ylRQ$BFc4`gM(nZd=Eo1l2o4HleLg%C)`}6Y>|zsdVzo0FJOJ|`MxKC zsQskeTRr57URtjDA(la*d;80l1taBj&DSdv^+qBZ1Z&=n^e$^LQKMuE6wNrGJa}(v zwSm>J8H4V_iuLXNM!BAZWCp9P1%j0u_y-#Cq8xJmBfucki9B}At~MyUQf~5c&`>vk zwjxru7@ew{O8kX&s1&lDlva7NSJ8#>CJsYJ^cX6m!vLx)<6h^KFs$7j8?g&rxHa!v znfnGHzcmkd5>8McrnDFk+UN6xRA2=IE~=`_HD`4aF&RyhO7^M!do#GUTfsdV%TlDO zaCcD_MC-dV0!MJNgyf7;gj2uz7mxC3q$M!8%32K+u_JuPjzb3cEm{GGhfKtszgSMY z6qm}xhdo|>$whC!`M@%b{3Pl?KXkuK(ycq4SOeYE#1Hy)GgfqZt}*zAQcLYHVx$Oy z0T%NF2|0pT~oS8iE`wIl?Vcc5AWc8r)a;g z92ww-YEkOqq3#m>4D&a!DH1sV_m}4$^Yb_AfwH_!!Ua$}{rX$lw+mG)tol+X5v0sz z>GmdrCGb!}MK|_T3HbfgVTJ<2`_0iHM!NmGtk0{#rTSK`oZjPrlVUHby8!U<^G{F1hYL`DZ<3;+b@$`u+_t+2xN?_}rp-Py%c9hFs--4VU@@_fPy`QMmH~OUX8l{M8z( zQ(|&UNnrO5poFPF+#aVAQ=%Y3qCF{Mo1=RUlYgj?a70-el0couhz!R?Po*g3EH{z` zSI0cpske57(EI_tdX3?6!#|8#j7F>_XY+E`sVe?uP)+UJ7M^*4%Z(hwhraFEny1qOs zk4iYkar?x`3kTr6`lvH6Q4yhpHpe5?7uI22Udw}&$+bg?E5Q;ZTO2IytprF(OI0uJ zzCicoH&zNioTI(Fa7bzqV@+COYDhb22tq}Lc~L>{a?UVWsMZ)AvA*r0tI2oQ^QOHI z23K$bLIaON={GLdMxjYHlb0~lMj1^hneP05lZS!_t&`ZH8%R2^Zpp%7tCAtS5*3Z_ zdfe^~Z4dZeigyA5-rEkX-<>%V{Pd;oqwY%I9QrkOK8nTK@m)XE#wMl+Gg9@eL^l#G zvJsLYI2DWpFBQD(AUGWOk?(@vI5w7c*u#`vt6~NMkst^kBl_z4k+XV24t<#U3|yEW zvd?65Af_SC?c$z({`U-e$J8~~>sxKJ^eJd6@r{UuN!-{ddFY7Bw!r8;FZhY7U@mUB z=y*56O6_=ksSuNzeLc+};fsZ(T&Q_k_c85!D*~c0FWh#SK04Zy*&(+raESxSJ;Qfaf8O$FaALTZ3vujf;KI$-CbymfxXJ^ENpfq>^LberDJH z-JBAiR4ZBP{xKdRbC0#7H6m%bnks1Mc{3~)xI01Gl07=eD+^+KFS8a*sNP-|qiY*{ z>?V}e&4n68i0O+8S$H^bWHAg**t!wl{Pk!^9`KOdbQ}7GWMGqJ>wpEZ;?nBqN9{sx z`KX>_B>qspjp+Ug_+wp^38y3TmaosEafdahR>Kk+W*yY5xRXM5hc#9|g7wZ|Z#Gxu z%a@m+h&mYrJPcYCC3m1md#>^lIsWr?Uc<#yK=p}u-lOWpwv5QW+Dqkth6$Ww=P}k2 zU}!{>uMb#BZ0=j`Uv!wjEs+TjP!1vpTi^X>+IXK5mVI~in84emnwd7^M@29)s7 zws&s(Hi0mh`z=e%Q3fGL1m$lRa zg*}X?qk^XG;EfT>?#3d%YA6cOU`vlem!|m*h{O@o_NWA@P zf;e#)HM4OcBhm)a*iO0b(MNo1HAK5Dd_EZhcsayb-S0575epl@zuR^gv>4fCsc5MA z)3QKDrPE^Wl@Sou-n)7*EbNX0VLd(PV}!@ z_T;y+xOT+It7&!~mIvz2iro^Dg)a6WS~ldY*70=CLU{!QEL9*jVIS^!qbTf`@83f&W8c^Ph8X|tphRM8rOtS$bK9G1|G8m=Pc7H9c6j_qRzf6p| z;l{8B%F)y;lctqHcC8E~L49>E-}>|RojjH6!@^%d|zp2G7!4GtVfrxJOsL-JHM zX+{dpWhP<`t`{E>V^(-kqFl==n$~5YkyGXz#9ge zaig8)(U(@S1N|dY3bt0jzl@y0WWP`|Ie>Gs4ox?9wv(xY^_RCyBafq&a5AtNhIG;g z72o_CKtnLMj`+Nul^149z|kS5Pb-}Gr)ZZl{wzvTgJmDL;OBR z0&{vQIpzu0y4srPi8}<@A(=-}Lw891ysaP8r#d+6vBJwo?EDG8Yne1>VVUHL!oBZ) z=SiM`k~(gso}&c0P>VqDMy#lz-P_gj?r$0A6~mj1q~J=5L?5KV-Z9r zs6u5}hUP4xHv5R$H@u$4&TA*~I+#J;wA+f*I1+A>V^IgINHwl3k#}HAF@8c!65D0> zG@-Q!paM&xE}<&0T;lh`17*^QcNK0j>A8NFiG>*EPk)($K3dIehznlW;0~evxIX zB6$^RrW*N{uzP?Ov3u6_=OF==ALdg$#6NM%hI&UFf~ZyVH~OtMdy5C1tyT9zg2c9Z zpM9zq=YyvzqFZJy4({6@`J2ir>{+U~X|ZH_1>drZ`ZZ@%ztjzsH8M{^7RynJ9$R#{ zqPRx{SKqh#8pGO9^~;fqShl*JPY|Xizo)3mr?BF~l?X?D>vd>EGd70P;8ohq9`)Rw z8Y8Sp^DZwCWeKz8licPv7QSVD!+gt%F)#}0QWVN9TPHw#Kj-#4|MB>%C@t&-&c!#r zrpwdQs2lLOGnIDti+a1{F~8*xkqFJluN5gFSbglD;!n(Hny>!kAZU>d9p;+hC1gs< z#6B9R0b&`4bxuO8j>sewuNv|7brVBVK?sA|hEv8Tt{w2Pu=N9fM$97QIk-Y?&1N0|d zdzRaM)9Y-E6JOSs%|X~Sk~a6hwp&`5YRB>`S}>fiIS9zp;MNhJZ#1=kdr<8GJpO`k zT%0{$Awa}-3hfGs<`*FWtWpC6y=|UE)wi{3C^WaFbM=4y(Vwi~o;`=QpMJSq~fuK(f!O*7XW_lAwC8 z2n*1jqFfE<>wfh~0$y0m3p3t3=v&RP*5%B|~e9WU1;BRg(Ufl6`~kEK}b^YDVw_E3rs_Ba=Bw5(~dhLVcsv zSGIB;JVK_}Ys}SO1XoFfU-dL9CM~41d4#8815pL-PM)kw-3K?(&#%HkO<^p~J2rJ}G^>lw47u?@|Z`mi z@OkL&vfGnCh}2fErm{xyZn~Bm$C@lkUHmdQn4wY*#cdbv~I_+j|mLz1i+qPFo z%+l{M;o^wP{qb;&BJYhg1%&&HyL8mJ^T#DW?okH#B3o<2w?x=CB6_9*?IMdpOH3YZ zvW5-jIK8kZH%wn|5W6-Ou)g$p#0HfTh;VOeH}T~5&guPwbLov*qH zUGQe{uSQ&Ye1_Q9zu+PN();RVd%h%^@c}y7=x=oIim2Y;!e4y{z%7+FAs49xf3wbT z*E&Ut+&bXdx@({bqRhQc88Hz81HIBw!xqtPk|ll^A;nyb_M7yNPWc|z7yF)ZnhR-I zsbAEv8dNe7mBIy{aMnmZ+A>PIrE%~FNts)wV(aMm#fqAxWt4%~JM=eH4RXgO# zilR#wSyM7rilV%03lU*TTJ%c);4Ko-|1OK*s4$k9 zOnq|S4T1MCu8ZNx(jbVo%7Ss|fBT3ItUkn2Le(&*dwq9M;dTuAm($jPp?0iAej$dEc(*is}^+fhqC!7ctybApS$ zmO`6%Xfu_qyvSdw+~Lf>uhbovr2J~S4MwJJy)h`hDgSnS`e^LBxe%Li);wLn9tqRd zj2cE;z1$jJ$U3c5Vn~2gOLgTE_AJb*x;K(IL~%ut@okFF_Q#uTQs!OGGWMgYBf6P7 zqbkOkO4?2NGwY>a^zr>MzdKV}`rHZA5#g6k-%q}p=^!4olf6?G*vn>o$@ZQj;TWyX zpr43F&r8WUCxPl|7Riu(TOWhO3}pBtRBwTMlK&1Lb_-_C4HW}O`rmY__ajASlD#~_R z2a(hdvc2B@wf8$&zO!a#(@c+dZ+dS{r=s4G;<2a?t`u2QP{6;WwQ|@$HcMF8Qb`gI z;GXHyPgx~#KR_#Hlg?`utbz>&vL~0AA_mFgeJi(9>5UC2bw?m}CzAr3LuXdrC!L1Q zx+IfAB4BYQKD$a;?D}*cH-E2 z8p}o$HX}{EW={!*a_`baeuJ5)>gg1qBsghAwOP4c?s$f}pOUe$3ItF86KVDUe*Z&| zpy}B8r152kWInSclW8*b9u={;Pq@)BmViTIp==RaFF<8?2V-`zpvsOk`ddwM<_hXq za=j6z^FerApU44YYW8Qr+01vSZ;;5BjXr#iSul*Ab!-F47jpwItv60S!5 z%keGTTVIjj;~5k@9#c~EUedz}hQn@9Dtqc|f2cfw(;123D(h#CL!ES+^VgZQy^&ZL zmRNLcRe^9UJ1h#cacO+x_K}J}(;V2M5p&_5ERRm2^jPMlee+I9>EapQX@?SUr5W)( z1(7F}gIq_%$C|$ce?+@`&V4NHP{`(AU4K&E%I<=u3JEJKGOMg{T+$x!c1;@1v`O9Q zKAJ^1>sN0rvMw!xekq66)@CjLROw1kZF#^QonBljNmZ}6pEET~W?i=*uJP zj>hi`H69~aP0+L0B~ox3KjD7rBAq@$XcL8rHlsB=AYEnz*=CXHGQbxa zR?DJn%&$)@jl2nydyUbq9rNK|U>+{eGN`4ROm^D_gU1 z19dxpHucL{WX#um-MgL(uHb0OW~VwqiCIK25p>%^`O1%$Zlz3km63TLCE8^7Z-FG~ zJAc@sg=jDl2UbH8W@??}I|*8v*{XX_PY_mDVWuRu&1R;*IB-_&rG#X_TJmY05l~HC z=jvm8Cr8^CMB8a*VrlWE`}cux0M9nmmhNQlKFZN7eqf?CVM}0kA2Aq_? zAlx_uVrdl&P?Z9zkdia`e!+bv_O-zVS>^wR5QIL`Cj!3aX7j|w;l2;>CJu%ll{xtP z#>_!^xTZ^aX+3p{t9exljr?ukKDP(M9H`%08}o7p0%OLx0k18ag619hzTwPHVpteF z)h$jbXPh^EX$m0t4sj;IO^*6G`t~>_d>a=lE#9>Anz3Nrf+BCYSC-OK?ou&GO}_J` z`Sm5WI5|ho?>7YL$xX6m##9t6$LueJ^8hua8i~HvGx99$=1g9y#h;(%0M@axDSp_- zAj2R3MSx{5vpIf3@~eZnf#?!-j%|D4j&Mr9PIaUlPOClNE0NHL>stGEcF5jsuENIf zEXC4xm;(QRTHoaVLjL&aK-yzv_B)f*S87SwI){9vB22LN9vf~wvCKSD?dd{WD*W3P z)t1H%HTwy<*XZ!-JF_+%iumo;Q45{KzTmaI3!8cco|c*OrpgX9WHF1fs{{(YuOgJ0 z@hz@!x3*ZfeRXeNjKdi}5~U9j5>4hHawgW8uofIX%4VsRpTBZ6UT0A9+f#W!Y}VtM z7g}?3ImxWylG|-Xlomy4!{DdPvAqQov!^*%oyYvR^5IPuuk95XbAS7#47$3W?al-N|BuESVQniw=qtPoQi7~6uiO_XY)g}79KEm?W zrd*CNy2kF_6}g@jq^DrhA7a70!Bh@r}fn>3X;1$eAK4?vUoaEO}OE`yw z7HQV!>vU+_XKS_i@oY8#yQEI_YZ4u}{ZYSTkUaDf0wVi8718#TSdPqdHWtL?B z@E3M_Y7P8A3Zd9Lbw@KAVQ;_ba`U1$kTZXSP@zbM5hsy}t0@><4`@Buf$6d6sJL2^ zwGZCC=gV*_k9~_kxADxm$%JFnQm;)F*~V%6L&hBNguPj5414`Rk0W^E{F=Fyu#0%J z`#z0MT;XuGcmy*Gqgm0le8Gz%3Ij?a{oK z;D2`JY_7MFe)y=1{tkz4CD@>WcjNjA5Q4q6)kk6TVVKw-IZ31zqr(8 z0DFu61qceqP(NIApGLsY1XXCAPj?g_A47o+ABP5XpD_T--kYatVYjEarzfv=rTjPA zf*uVa0)Ks(5gfjQmuyZj$wTFl0&L;9~8$NuJL`5PTu8*51dB7`fj#8<8^j0 zNyH_SJvsw?}2I#Bfk$x7LN0rF3bGmSu>Z6S0$N#G0$R& z=ZLrcE9x%1tvJPLk@Yp)zf+-rzzkl>?`f3%#5S}o{KG{Sd)A4WqHx>HGe62&FGWL> z2lu!X1N!^s8nL&$ZCUPA=~8v(PFw-dA+TBb879dzWFud<%wM3PC&8L2VX&JxQLM~Z zjJB1N5Ibao!T^sE%F5zdYyvVwSiU|Np(0|88B*S2rLxUpXTPw{^fPlWXCr95y+!x( zLKA(OSaaO`R^v+CUzj@Lb|@KsaZpcD+6pODs~gtl{n$lo=h%`_$ktn_voy`xa($@8 zz)*v+_fBrjtT#$O!$6qT_cJLBclAwgj|MUE&ozV(zL!|>lPFz%uBl5IuU(b_LuQ?A z?*1?j&0*3s6HqZ=%-InbaRidEh5@CxzCRdJ(Fs5Wy}IRGNUog&D@J{JPv>}*VCTK;b7-OSP~ zyH{0y9EqssS2c07`uQm&vU1k<8rIFo#~GBePEL5I$GA5rFmn>y)mt@udshxLgkgoT z&w|Ax*Y~cRS0KP2k?vDmp$_g~9@$0>J!oq1AZ|TlY8{3CblLnggYNT!88?)89Dj{ZO}y#t>wBoy^Dzh~Ek}ccj7V?E zbfy?%)PMVKrEbKL|BLY$I}p*5(n%CE5zSS1X;`yD1Q%LsS)H?4Z#g)JIhgZl!|bz* z_m?mjq+f=)qK^2d6<=jAxc19u%K5ra4{D6g^i+TWk+pTu+VVur94ZRs%6=?fR*Zv{ zBwyIO-qmKP{6%cogN?_c|$6QOw~ zJMPUJNQ~F|p441qiEeEf)Lxgqc~kX=1?62Uadov5T~#G=?J!zPhxEQD*abU=QJEsw z&F8m^Dslxr);ejF3VSg(inLsY$ZSAM{pZaa?9HD+1`+1!#MX>be29y=G=cm3rg`UW z&QKbZ=ijB6*`_E;Kp-)f3o;*zh30acR~zUF)C%<1qk+i$2k>EQuQ} z8Gat$nHSU&`*Qh=Fhl}3^Kr1R9v6?-0|K9ze}aX+`pHfW3TGaRL|Z18HP>Ue*I1p{y^h(tq6Hv3%JGJ_OgtwnOCPgR7K%P z7q;QD3%T_hzdK+Oety^jQRQSfCmX11h39*VaFDvK|Ds!a`+7Q1ql`Q#q{u6Kg67S~Ia+9F`A;)AL??jkXl z6+t=174nt^Ms^&~hCY+`pb1?R(Qxj^*%#Wybgch^;gvhD)Hk1#%>WF>Sm5e=#mK;i z{$ITO`v1$6$6A11(ewYZ;^seJW&caC|3Li*s@wnPA%E>Ze*OQ3(f=R8{}Sl`CEO1~ z{sHa(2d|Fwzq$9i{{ZZL?f=kq;(tW@SDb&${0q*vdp#1xFI@Ale*?fN%Bsm!Ntp%z E4`_. - -Breaking changes -~~~~~~~~~~~~~~~~ - -.. warning:: - All deprecated classes, parameters and features have been removed from the {provider_name} provider package. - The following breaking changes were introduced: - - * removed ``YandexCloudBaseHook.provider_user_agent`` . Use ``utils.user_agent.provider_user_agent`` instead. - * removed ``connection_id`` parameter from ``YandexCloudBaseHook``. Use ``yandex_conn_id`` parameter. - * removed ``yandex.hooks.yandexcloud_dataproc`` module. - * removed ``yandex.operators.yandexcloud_dataproc`` module. - * removed implicit passing of ``yandex_conn_id`` in ``DataprocBaseOperator``. Please pass it as a parameter. - -* ``Remove Provider Deprecations in Yandex provider (#44754)`` - -Misc -~~~~ - -* ``Bump minimum Airflow version in providers to Airflow 2.9.0 (#44956)`` -* ``Update DAG example links in multiple providers documents (#44034)`` - - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Use Python 3.9 as target version for Ruff & Black rules (#44298)`` - * ``Prepare docs for Nov 1st wave of providers (#44011)`` - * ``Split providers out of the main "airflow/" tree into a UV workspace project (#42505)`` - -.. Review and move the new changes to one of the sections above: - * ``Update path of example dags in docs (#45069)`` - -3.12.0 -...... - -.. note:: - This release of provider is only available for Airflow 2.8+ as explained in the - `Apache Airflow providers support policy `_. - -Bug Fixes -~~~~~~~~~ - -* ``providers/yandex: fix typing (#40997)`` - -Misc -~~~~ - -* ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Prepare docs for Aug 1st wave of providers (#41230)`` - * ``Prepare docs 1st wave July 2024 (#40644)`` - * ``Enable enforcing pydocstyle rule D213 in ruff. (#40448)`` - -3.11.2 -...... - -Bug Fixes -~~~~~~~~~ - -* ``Exclude yandex versions 0.289.0, 0.290.0 (#39974)`` - -Misc -~~~~ - -* ``Fix typos in Providers docs and Yandex hook (#40277)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Limit yandex provider to avoid mypy errors (#39990)`` - * ``Workaround new yandexcloud breaking dataproc integration (#39964)`` - -3.11.1 -...... - -Misc -~~~~ - -* `` AIP-21: yandexcloud: rename files, emit deprecation warning (#39618)`` -* ``yandex provider: bump version for yq http client package (#39548)`` -* ``Faster 'airflow_version' imports (#39552)`` -* ``add doc about Yandex Query operator (#39445)`` -* ``Simplify 'airflow_version' imports (#39497)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Reapply templates for all providers (#39554)`` - -3.11.0 -...... - -.. note:: - This release of provider is only available for Airflow 2.7+ as explained in the - `Apache Airflow providers support policy `_. - -Misc -~~~~ - -* ``Bump minimum Airflow version in providers to Airflow 2.7.0 (#39240)`` - -3.10.0 -...... - -Features -~~~~~~~~ - -* ``Add Yandex Query support from Yandex.Cloud (#37458)`` - -Misc -~~~~ - -* ``support iam token from metadata, simplify code (#38411)`` -* ``Avoid use of 'assert' outside of the tests (#37718)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Prepare docs 1st wave (RC1) April 2024 (#38863)`` - * ``docs: yandex provider grammatical improvements (#38589)`` - * ``Bump ruff to 0.3.3 (#38240)`` - * ``Prepare docs 1st wave (RC1) March 2024 (#37876)`` - * ``Add comment about versions updated by release manager (#37488)`` - -3.9.0 -..... - -Features -~~~~~~~~ - -* ``Add secrets-backends section into the Yandex provider yaml definition (#37065)`` - -Bug Fixes -~~~~~~~~~ - -* ``fix: using endpoint from connection if not specified (#37076)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``D401 Support in Providers (simple) (#37258)`` - * ``docs: update description in airflow provider.yaml (#37096)`` - -3.8.0 -..... - -Features -~~~~~~~~ - -* ``feat: add Yandex Cloud Lockbox secrets backend (#36449)`` - - -Bug Fixes -~~~~~~~~~ - -* ``Fix stacklevel in warnings.warn into the providers (#36831)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Prepare docs 1st wave of Providers January 2024 (#36640)`` - * ``Speed up autocompletion of Breeze by simplifying provider state (#36499)`` - * ``Provide the logger_name param in providers hooks in order to override the logger name (#36675)`` - * ``Revert "Provide the logger_name param in providers hooks in order to override the logger name (#36675)" (#37015)`` - * ``Prepare docs 2nd wave of Providers January 2024 (#36945)`` - -3.7.1 -..... - -Bug Fixes -~~~~~~~~~ - -* ``Follow BaseHook connection fields method signature in child classes (#36086)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - -3.7.0 -..... - -.. note:: - This release of provider is only available for Airflow 2.6+ as explained in the - `Apache Airflow providers support policy `_. - -Misc -~~~~ - -* ``Bump minimum Airflow version in providers to Airflow 2.6.0 (#36017)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Fix and reapply templates for provider documentation (#35686)`` - * ``Prepare docs 2nd wave of Providers November 2023 (#35836)`` - * ``Use reproducible builds for provider packages (#35693)`` - -3.6.0 -..... - -Features -~~~~~~~~ - -* ``Yandex dataproc deduce default service account (#35059)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Prepare docs 3rd wave of Providers October 2023 - FIX (#35233)`` - * ``Prepare docs 3rd wave of Providers October 2023 (#35187)`` - * ``Pre-upgrade 'ruff==0.0.292' changes in providers (#35053)`` - -3.5.0 -..... - -.. note:: - This release of provider is only available for Airflow 2.5+ as explained in the - `Apache Airflow providers support policy `_. - -Misc -~~~~ - -* ``Bump min airflow version of providers (#34728)`` - -3.4.0 -..... - -.. note:: - This release dropped support for Python 3.7 - -Features -~~~~~~~~ - -* ``add support for Yandex Dataproc cluster labels (#29811)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Add note about dropping Python 3.7 for providers (#32015)`` - * ``Add D400 pydocstyle check - Providers (#31427)`` - * ``Add full automation for min Airflow version for providers (#30994)`` - * ``Add mechanism to suspend providers (#30422)`` - * ``Resume yandex provider (#33574)`` - * ``Remove spurious headers for provider changelogs (#32373)`` - * ``Improve provider documentation and README structure (#32125)`` - * ``Use '__version__' in providers not 'version' (#31393)`` - * ``Use 'AirflowProviderDeprecationWarning' in providers (#30975)`` - * ``Bump minimum Airflow version in providers (#30917)`` - * ``Suspend Yandex provider due to protobuf limitation (#30667)`` - -3.3.0 -..... - -Features -~~~~~~~~ - -* ``support Yandex SDK feature "endpoint" (#29635)`` - -3.2.0 -..... - -.. note:: - This release of provider is only available for Airflow 2.3+ as explained in the - Apache Airflow providers support policy `_. - -Features -~~~~~~~~ - -* In YandexCloudBaseHook, non-prefixed extra fields are supported and are preferred (#27040). E.g. ``folder_id`` will be preferred if ``extra__yandexcloud__folder_id`` is also present. - -Misc -~~~~ - -* ``Move min airflow version to 2.3.0 for all providers (#27196)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Enable string normalization in python formatting - providers (#27205)`` - * ``Update docs for September Provider's release (#26731)`` - * ``Apply PEP-563 (Postponed Evaluation of Annotations) to non-core airflow (#26289)`` - * ``pRepare docs for November 2022 wave of Providers (#27613)`` - * ``Prepare for follow-up release for November providers (#27774)`` - -3.1.0 -..... - -Features -~~~~~~~~ - -* ``YandexCloud provider: Support new Yandex SDK features for DataProc (#25158)`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Add documentation for July 2022 Provider's release (#25030)`` - * ``Move provider dependencies to inside provider folders (#24672)`` - * ``Remove 'hook-class-names' from provider.yaml (#24702)`` - -3.0.0 -..... - -Breaking changes -~~~~~~~~~~~~~~~~ - -.. note:: - This release of provider is only available for Airflow 2.2+ as explained in the Apache Airflow - Apache Airflow providers support policy `_. - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Migrate Yandex example DAGs to new design AIP-47 (#24082)`` - * ``Add explanatory note for contributors about updating Changelog (#24229)`` - * ``Prepare docs for May 2022 provider's release (#24231)`` - * ``Update package description to remove double min-airflow specification (#24292)`` - -2.2.3 -..... - -Bug Fixes -~~~~~~~~~ - -* ``Fix mistakenly added install_requires for all providers (#22382)`` - -2.2.2 -..... - -Misc -~~~~~ - -* ``Add Trove classifiers in PyPI (Framework :: Apache Airflow :: Provider)`` - -2.2.1 -..... - -Misc -~~~~ - -* ``Support for Python 3.10`` - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Fixed changelog for January 2022 (delayed) provider's release (#21439)`` - * ``Add documentation for January 2021 providers release (#21257)`` - * ``Add optional features in providers. (#21074)`` - * ``Remove ':type' lines now sphinx-autoapi supports typehints (#20951)`` - * ``Fix spelling (#22054)`` - -2.2.0 -..... - -Features -~~~~~~~~ - -* ``YandexCloud provider: Support new Yandex SDK features: log_group_id, user-agent, maven packages (#20103)`` - - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Fix mypy for providers: elasticsearch, oracle, yandex (#20344)`` - * ``Fixup string concatenations (#19099)`` - * ``Update documentation for November 2021 provider's release (#19882)`` - * ``Prepare documentation for October Provider's release (#19321)`` - * ``Update documentation for September providers release (#18613)`` - * ``Static start_date and default arg cleanup for misc. provider example DAGs (#18597)`` - * ``Inclusive Language (#18349)`` - * ``Use typed Context EVERYWHERE (#20565)`` - * ``Fix template_fields type to have MyPy friendly Sequence type (#20571)`` - * ``Update documentation for provider December 2021 release (#20523)`` - -2.1.0 -..... - -Misc -~~~~ - -* ``Optimise connection importing for Airflow 2.2.0`` - - -Features -~~~~~~~~ - -* ``Add autoscaling subcluster support and remove defaults (#17033)`` - - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Update description about the new ''connection-types'' provider meta-data (#17767)`` - * ``Import Hooks lazily individually in providers manager (#17682)`` - * ``Prepares docs for Rc2 release of July providers (#17116)`` - * ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` - * ``Prepare documentation for July release of providers. (#17015)`` - * ``Removes pylint from our toolchain (#16682)`` - -2.0.0 -..... - -Breaking changes -~~~~~~~~~~~~~~~~ - -* ``Auto-apply apply_default decorator (#15667)`` - -.. warning:: Due to apply_default decorator removal, this version of the provider requires Airflow 2.1.0+. - If your Airflow version is < 2.1.0, and you want to install this provider version, first upgrade - Airflow to at least version 2.1.0. Otherwise your Airflow package version will be upgraded - automatically and you will have to manually run ``airflow upgrade db`` to complete the migration. - -.. Below changes are excluded from the changelog. Move them to - appropriate section above if needed. Do not delete the lines(!): - * ``Adds interactivity when generating provider documentation. (#15518)`` - * ``Prepares provider release after PIP 21 compatibility (#15576)`` - * ``Update docstrings to adhere to sphinx standards (#14918)`` - * ``Remove Backport Providers (#14886)`` - * ``Update documentation for broken package releases (#14734)`` - * ``Updated documentation for June 2021 provider release (#16294)`` - * ``Fix Sphinx Issues with Docstrings (#14968)`` - * ``More documentation update for June providers release (#16405)`` - * ``Synchronizes updated changelog after buggfix release (#16464)`` - -1.0.1 -..... - -Updated documentation and readme files. - -1.0.0 -..... - -Initial version of the provider. diff --git a/providers/yandex/README.rst b/providers/yandex/README.rst index ef6c09aeba288..756fe2d21fb1f 100644 --- a/providers/yandex/README.rst +++ b/providers/yandex/README.rst @@ -1,3 +1,4 @@ + .. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -15,4 +16,50 @@ specific language governing permissions and limitations under the License. -This content will be overridden by pre-commit hook + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +Package ``apache-airflow-providers-yandex`` + +Release: ``4.0.0`` + + +This package is for Yandex, including: + + - `Yandex.Cloud `__ + + +Provider package +---------------- + +This is a provider package for ``yandex`` provider. All classes for this provider package +are in ``airflow.providers.yandex`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-yandex`` + +The package supports the following python versions: 3.9,3.10,3.11,3.12 + +Requirements +------------ + +======================= ================== +PIP package Version required +======================= ================== +``apache-airflow`` ``>=2.9.0`` +``yandexcloud`` ``>=0.308.0`` +``yandex-query-client`` ``>=0.1.4`` +======================= ================== + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/yandex/docs/operators/dataproc.rst b/providers/yandex/docs/operators/dataproc.rst index 03dfd3acae817..b7188e2ea52f6 100644 --- a/providers/yandex/docs/operators/dataproc.rst +++ b/providers/yandex/docs/operators/dataproc.rst @@ -34,4 +34,4 @@ that can be integrated with Apache Hadoop and other storage systems. Using the operators ^^^^^^^^^^^^^^^^^^^ To learn how to use Data Proc operators, -see `example DAGs `_. +see `example DAGs `_. diff --git a/providers/yandex/docs/operators/yq.rst b/providers/yandex/docs/operators/yq.rst index 23bd4ac336160..08a90bb817220 100644 --- a/providers/yandex/docs/operators/yq.rst +++ b/providers/yandex/docs/operators/yq.rst @@ -25,4 +25,4 @@ Yandex Query Operators Using the operators ^^^^^^^^^^^^^^^^^^^ To learn how to use Yandex Query operator, -see `example DAG `__. +see `example DAG `__. diff --git a/providers/src/airflow/providers/yandex/provider.yaml b/providers/yandex/provider.yaml similarity index 92% rename from providers/src/airflow/providers/yandex/provider.yaml rename to providers/yandex/provider.yaml index 22e06e21478bf..792fb195868c6 100644 --- a/providers/src/airflow/providers/yandex/provider.yaml +++ b/providers/yandex/provider.yaml @@ -52,29 +52,24 @@ versions: - 1.0.1 - 1.0.0 -dependencies: - - apache-airflow>=2.9.0 - - yandexcloud>=0.308.0 - - yandex-query-client>=0.1.4 - integrations: - integration-name: Yandex.Cloud external-doc-url: https://cloud.yandex.com/ - logo: /integration-logos/yandex/Yandex-Cloud.png + logo: /docs/integration-logos/Yandex-Cloud.png tags: [service] - integration-name: Yandex.Cloud Dataproc external-doc-url: https://cloud.yandex.com/dataproc how-to-guide: - /docs/apache-airflow-providers-yandex/operators/dataproc.rst - logo: /integration-logos/yandex/Yandex-Cloud.png + logo: /docs/integration-logos/Yandex-Cloud.png tags: [service] - integration-name: Yandex.Cloud YQ external-doc-url: https://cloud.yandex.com/en/services/query how-to-guide: - /docs/apache-airflow-providers-yandex/operators/yq.rst - logo: /integration-logos/yandex/Yandex-Cloud.png + logo: /docs/integration-logos/Yandex-Cloud.png tags: [service] operators: diff --git a/providers/yandex/pyproject.toml b/providers/yandex/pyproject.toml new file mode 100644 index 0000000000000..375951ae5d876 --- /dev/null +++ b/providers/yandex/pyproject.toml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.10.1"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-yandex" +version = "4.0.0" +description = "Provider package apache-airflow-providers-yandex for Apache Airflow" +readme = "README.rst" +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "yandex", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] +requires-python = "~=3.9" + +# The dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +dependencies = [ + "apache-airflow>=2.9.0", + "yandexcloud>=0.308.0", + "yandex-query-client>=0.1.4", +] + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.0.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.0.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Twitter" = "https://x.com/ApacheAirflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.yandex.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.yandex" + +[tool.pytest.ini_options] +ignore = "tests/system/" diff --git a/providers/yandex/src/airflow/providers/yandex/LICENSE b/providers/yandex/src/airflow/providers/yandex/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/yandex/src/airflow/providers/yandex/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/src/airflow/providers/yandex/__init__.py b/providers/yandex/src/airflow/providers/yandex/__init__.py similarity index 100% rename from providers/src/airflow/providers/yandex/__init__.py rename to providers/yandex/src/airflow/providers/yandex/__init__.py diff --git a/providers/yandex/src/airflow/providers/yandex/get_provider_info.py b/providers/yandex/src/airflow/providers/yandex/get_provider_info.py new file mode 100644 index 0000000000000..c2f62621f04c9 --- /dev/null +++ b/providers/yandex/src/airflow/providers/yandex/get_provider_info.py @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-yandex", + "name": "Yandex", + "description": "This package is for Yandex, including:\n\n - `Yandex.Cloud `__\n", + "state": "ready", + "source-date-epoch": 1734537491, + "versions": [ + "4.0.0", + "3.12.0", + "3.11.2", + "3.11.1", + "3.11.0", + "3.10.0", + "3.9.0", + "3.8.0", + "3.7.1", + "3.7.0", + "3.6.0", + "3.5.0", + "3.4.0", + "3.3.0", + "3.2.0", + "3.1.0", + "3.0.0", + "2.2.3", + "2.2.2", + "2.2.1", + "2.2.0", + "2.1.0", + "2.0.0", + "1.0.1", + "1.0.0", + ], + "integrations": [ + { + "integration-name": "Yandex.Cloud", + "external-doc-url": "https://cloud.yandex.com/", + "logo": "/docs/integration-logos/Yandex-Cloud.png", + "tags": ["service"], + }, + { + "integration-name": "Yandex.Cloud Dataproc", + "external-doc-url": "https://cloud.yandex.com/dataproc", + "how-to-guide": ["/docs/apache-airflow-providers-yandex/operators/dataproc.rst"], + "logo": "/docs/integration-logos/Yandex-Cloud.png", + "tags": ["service"], + }, + { + "integration-name": "Yandex.Cloud YQ", + "external-doc-url": "https://cloud.yandex.com/en/services/query", + "how-to-guide": ["/docs/apache-airflow-providers-yandex/operators/yq.rst"], + "logo": "/docs/integration-logos/Yandex-Cloud.png", + "tags": ["service"], + }, + ], + "operators": [ + { + "integration-name": "Yandex.Cloud Dataproc", + "python-modules": ["airflow.providers.yandex.operators.dataproc"], + }, + { + "integration-name": "Yandex.Cloud YQ", + "python-modules": ["airflow.providers.yandex.operators.yq"], + }, + ], + "hooks": [ + {"integration-name": "Yandex.Cloud", "python-modules": ["airflow.providers.yandex.hooks.yandex"]}, + { + "integration-name": "Yandex.Cloud Dataproc", + "python-modules": ["airflow.providers.yandex.hooks.dataproc"], + }, + {"integration-name": "Yandex.Cloud YQ", "python-modules": ["airflow.providers.yandex.hooks.yq"]}, + ], + "connection-types": [ + { + "hook-class-name": "airflow.providers.yandex.hooks.yandex.YandexCloudBaseHook", + "connection-type": "yandexcloud", + } + ], + "secrets-backends": ["airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend"], + "extra-links": ["airflow.providers.yandex.links.yq.YQLink"], + "config": { + "yandex": { + "description": "This section contains settings for Yandex Cloud integration.", + "options": { + "sdk_user_agent_prefix": { + "description": "Prefix for User-Agent header in Yandex.Cloud SDK requests\n", + "version_added": "3.6.0", + "type": "string", + "example": None, + "default": "", + } + }, + } + }, + "dependencies": ["apache-airflow>=2.9.0", "yandexcloud>=0.308.0", "yandex-query-client>=0.1.4"], + } diff --git a/providers/src/airflow/providers/yandex/hooks/__init__.py b/providers/yandex/src/airflow/providers/yandex/hooks/__init__.py similarity index 100% rename from providers/src/airflow/providers/yandex/hooks/__init__.py rename to providers/yandex/src/airflow/providers/yandex/hooks/__init__.py diff --git a/providers/src/airflow/providers/yandex/hooks/dataproc.py b/providers/yandex/src/airflow/providers/yandex/hooks/dataproc.py similarity index 100% rename from providers/src/airflow/providers/yandex/hooks/dataproc.py rename to providers/yandex/src/airflow/providers/yandex/hooks/dataproc.py diff --git a/providers/src/airflow/providers/yandex/hooks/yandex.py b/providers/yandex/src/airflow/providers/yandex/hooks/yandex.py similarity index 100% rename from providers/src/airflow/providers/yandex/hooks/yandex.py rename to providers/yandex/src/airflow/providers/yandex/hooks/yandex.py diff --git a/providers/src/airflow/providers/yandex/hooks/yq.py b/providers/yandex/src/airflow/providers/yandex/hooks/yq.py similarity index 100% rename from providers/src/airflow/providers/yandex/hooks/yq.py rename to providers/yandex/src/airflow/providers/yandex/hooks/yq.py diff --git a/providers/src/airflow/providers/yandex/links/__init__.py b/providers/yandex/src/airflow/providers/yandex/links/__init__.py similarity index 100% rename from providers/src/airflow/providers/yandex/links/__init__.py rename to providers/yandex/src/airflow/providers/yandex/links/__init__.py diff --git a/providers/src/airflow/providers/yandex/links/yq.py b/providers/yandex/src/airflow/providers/yandex/links/yq.py similarity index 100% rename from providers/src/airflow/providers/yandex/links/yq.py rename to providers/yandex/src/airflow/providers/yandex/links/yq.py diff --git a/providers/src/airflow/providers/yandex/operators/__init__.py b/providers/yandex/src/airflow/providers/yandex/operators/__init__.py similarity index 100% rename from providers/src/airflow/providers/yandex/operators/__init__.py rename to providers/yandex/src/airflow/providers/yandex/operators/__init__.py diff --git a/providers/src/airflow/providers/yandex/operators/dataproc.py b/providers/yandex/src/airflow/providers/yandex/operators/dataproc.py similarity index 100% rename from providers/src/airflow/providers/yandex/operators/dataproc.py rename to providers/yandex/src/airflow/providers/yandex/operators/dataproc.py diff --git a/providers/src/airflow/providers/yandex/operators/yq.py b/providers/yandex/src/airflow/providers/yandex/operators/yq.py similarity index 100% rename from providers/src/airflow/providers/yandex/operators/yq.py rename to providers/yandex/src/airflow/providers/yandex/operators/yq.py diff --git a/providers/src/airflow/providers/yandex/secrets/__init__.py b/providers/yandex/src/airflow/providers/yandex/secrets/__init__.py similarity index 100% rename from providers/src/airflow/providers/yandex/secrets/__init__.py rename to providers/yandex/src/airflow/providers/yandex/secrets/__init__.py diff --git a/providers/src/airflow/providers/yandex/secrets/lockbox.py b/providers/yandex/src/airflow/providers/yandex/secrets/lockbox.py similarity index 100% rename from providers/src/airflow/providers/yandex/secrets/lockbox.py rename to providers/yandex/src/airflow/providers/yandex/secrets/lockbox.py index d65131ab2cb1a..0381486bab76e 100644 --- a/providers/src/airflow/providers/yandex/secrets/lockbox.py +++ b/providers/yandex/src/airflow/providers/yandex/secrets/lockbox.py @@ -21,14 +21,14 @@ from functools import cached_property from typing import Any +import yandexcloud + import yandex.cloud.lockbox.v1.payload_pb2 as payload_pb import yandex.cloud.lockbox.v1.payload_service_pb2 as payload_service_pb import yandex.cloud.lockbox.v1.payload_service_pb2_grpc as payload_service_pb_grpc import yandex.cloud.lockbox.v1.secret_pb2 as secret_pb import yandex.cloud.lockbox.v1.secret_service_pb2 as secret_service_pb import yandex.cloud.lockbox.v1.secret_service_pb2_grpc as secret_service_pb_grpc -import yandexcloud - from airflow.models import Connection from airflow.providers.yandex.utils.credentials import get_credentials from airflow.providers.yandex.utils.defaults import default_conn_name diff --git a/providers/src/airflow/providers/yandex/utils/__init__.py b/providers/yandex/src/airflow/providers/yandex/utils/__init__.py similarity index 100% rename from providers/src/airflow/providers/yandex/utils/__init__.py rename to providers/yandex/src/airflow/providers/yandex/utils/__init__.py diff --git a/providers/src/airflow/providers/yandex/utils/credentials.py b/providers/yandex/src/airflow/providers/yandex/utils/credentials.py similarity index 100% rename from providers/src/airflow/providers/yandex/utils/credentials.py rename to providers/yandex/src/airflow/providers/yandex/utils/credentials.py diff --git a/providers/src/airflow/providers/yandex/utils/defaults.py b/providers/yandex/src/airflow/providers/yandex/utils/defaults.py similarity index 100% rename from providers/src/airflow/providers/yandex/utils/defaults.py rename to providers/yandex/src/airflow/providers/yandex/utils/defaults.py diff --git a/providers/src/airflow/providers/yandex/utils/fields.py b/providers/yandex/src/airflow/providers/yandex/utils/fields.py similarity index 100% rename from providers/src/airflow/providers/yandex/utils/fields.py rename to providers/yandex/src/airflow/providers/yandex/utils/fields.py diff --git a/providers/src/airflow/providers/yandex/utils/user_agent.py b/providers/yandex/src/airflow/providers/yandex/utils/user_agent.py similarity index 100% rename from providers/src/airflow/providers/yandex/utils/user_agent.py rename to providers/yandex/src/airflow/providers/yandex/utils/user_agent.py diff --git a/providers/yandex/tests/conftest.py b/providers/yandex/tests/conftest.py new file mode 100644 index 0000000000000..068fe6bbf5ae9 --- /dev/null +++ b/providers/yandex/tests/conftest.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pathlib + +import pytest + +pytest_plugins = "tests_common.pytest_plugin" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + deprecations_ignore_path = pathlib.Path(__file__).parent.joinpath("deprecations_ignore.yml") + dep_path = [deprecations_ignore_path] if deprecations_ignore_path.exists() else [] + config.inicfg["airflow_deprecations_ignore"] = ( + config.inicfg.get("airflow_deprecations_ignore", []) + dep_path # type: ignore[assignment,operator] + ) diff --git a/providers/yandex/tests/provider_tests/__init__.py b/providers/yandex/tests/provider_tests/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/yandex/tests/provider_tests/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/tests/system/yandex/__init__.py b/providers/yandex/tests/provider_tests/yandex/__init__.py similarity index 100% rename from providers/tests/system/yandex/__init__.py rename to providers/yandex/tests/provider_tests/yandex/__init__.py diff --git a/providers/tests/yandex/__init__.py b/providers/yandex/tests/provider_tests/yandex/hooks/__init__.py similarity index 100% rename from providers/tests/yandex/__init__.py rename to providers/yandex/tests/provider_tests/yandex/hooks/__init__.py diff --git a/providers/tests/yandex/hooks/test_dataproc.py b/providers/yandex/tests/provider_tests/yandex/hooks/test_dataproc.py similarity index 99% rename from providers/tests/yandex/hooks/test_dataproc.py rename to providers/yandex/tests/provider_tests/yandex/hooks/test_dataproc.py index c960e573c1fd6..212d1e3a9c05a 100644 --- a/providers/tests/yandex/hooks/test_dataproc.py +++ b/providers/yandex/tests/provider_tests/yandex/hooks/test_dataproc.py @@ -23,8 +23,8 @@ yandexlcloud = pytest.importorskip("yandexcloud") -from airflow.models import Connection -from airflow.providers.yandex.hooks.dataproc import DataprocHook +from airflow.models import Connection # noqa: E402 +from airflow.providers.yandex.hooks.dataproc import DataprocHook # noqa: E402 # Airflow connection with type "yandexcloud" must be created CONNECTION_ID = "yandexcloud_default" diff --git a/providers/tests/yandex/hooks/test_yandex.py b/providers/yandex/tests/provider_tests/yandex/hooks/test_yandex.py similarity index 100% rename from providers/tests/yandex/hooks/test_yandex.py rename to providers/yandex/tests/provider_tests/yandex/hooks/test_yandex.py index 92188cd07f319..7907c6a434001 100644 --- a/providers/tests/yandex/hooks/test_yandex.py +++ b/providers/yandex/tests/provider_tests/yandex/hooks/test_yandex.py @@ -21,12 +21,12 @@ import pytest -yandexcloud = pytest.importorskip("yandexcloud") - from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook from tests_common.test_utils.config import conf_vars +yandexcloud = pytest.importorskip("yandexcloud") + class TestYandexHook: @mock.patch("airflow.hooks.base.BaseHook.get_connection") diff --git a/providers/tests/yandex/hooks/test_yq.py b/providers/yandex/tests/provider_tests/yandex/hooks/test_yq.py similarity index 99% rename from providers/tests/yandex/hooks/test_yq.py rename to providers/yandex/tests/provider_tests/yandex/hooks/test_yq.py index 3defe65d6c8e5..c93a5f53895f6 100644 --- a/providers/tests/yandex/hooks/test_yq.py +++ b/providers/yandex/tests/provider_tests/yandex/hooks/test_yq.py @@ -21,15 +21,14 @@ from unittest import mock import pytest - -yandexcloud = pytest.importorskip("yandexcloud") - import responses from responses import matchers from airflow.models import Connection from airflow.providers.yandex.hooks.yq import YQHook +yandexcloud = pytest.importorskip("yandexcloud") + OAUTH_TOKEN = "my_oauth_token" IAM_TOKEN = "my_iam_token" SERVICE_ACCOUNT_AUTH_KEY_JSON = """{"id":"my_id", "service_account_id":"my_sa1", "private_key":"my_pk"}""" diff --git a/providers/tests/yandex/hooks/__init__.py b/providers/yandex/tests/provider_tests/yandex/links/__init__.py similarity index 100% rename from providers/tests/yandex/hooks/__init__.py rename to providers/yandex/tests/provider_tests/yandex/links/__init__.py diff --git a/providers/tests/yandex/links/test_yq.py b/providers/yandex/tests/provider_tests/yandex/links/test_yq.py similarity index 100% rename from providers/tests/yandex/links/test_yq.py rename to providers/yandex/tests/provider_tests/yandex/links/test_yq.py diff --git a/providers/tests/yandex/links/__init__.py b/providers/yandex/tests/provider_tests/yandex/operators/__init__.py similarity index 100% rename from providers/tests/yandex/links/__init__.py rename to providers/yandex/tests/provider_tests/yandex/operators/__init__.py diff --git a/providers/tests/yandex/operators/test_dataproc.py b/providers/yandex/tests/provider_tests/yandex/operators/test_dataproc.py similarity index 100% rename from providers/tests/yandex/operators/test_dataproc.py rename to providers/yandex/tests/provider_tests/yandex/operators/test_dataproc.py index 649631547de75..cde731d2d407b 100644 --- a/providers/tests/yandex/operators/test_dataproc.py +++ b/providers/yandex/tests/provider_tests/yandex/operators/test_dataproc.py @@ -21,8 +21,6 @@ import pytest -yandexcloud = pytest.importorskip("yandexcloud") - from airflow.models.dag import DAG from airflow.providers.yandex.operators.dataproc import ( DataprocCreateClusterOperator, @@ -33,6 +31,8 @@ DataprocDeleteClusterOperator, ) +yandexcloud = pytest.importorskip("yandexcloud") + # Airflow connection with type "yandexcloud" CONNECTION_ID = "yandexcloud_default" diff --git a/providers/tests/yandex/operators/test_yq.py b/providers/yandex/tests/provider_tests/yandex/operators/test_yq.py similarity index 99% rename from providers/tests/yandex/operators/test_yq.py rename to providers/yandex/tests/provider_tests/yandex/operators/test_yq.py index 3c415f6ac6b65..127e4eb837972 100644 --- a/providers/tests/yandex/operators/test_yq.py +++ b/providers/yandex/tests/provider_tests/yandex/operators/test_yq.py @@ -21,11 +21,6 @@ from unittest.mock import MagicMock, call, patch import pytest - -from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS - -yandexcloud = pytest.importorskip("yandexcloud") - import responses from responses import matchers @@ -33,6 +28,10 @@ from airflow.models.dag import DAG from airflow.providers.yandex.operators.yq import YQExecuteQueryOperator +from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS + +yandexcloud = pytest.importorskip("yandexcloud") + OAUTH_TOKEN = "my_oauth_token" FOLDER_ID = "my_folder_id" diff --git a/providers/tests/yandex/operators/__init__.py b/providers/yandex/tests/provider_tests/yandex/secrets/__init__.py similarity index 100% rename from providers/tests/yandex/operators/__init__.py rename to providers/yandex/tests/provider_tests/yandex/secrets/__init__.py diff --git a/providers/tests/yandex/secrets/test_lockbox.py b/providers/yandex/tests/provider_tests/yandex/secrets/test_lockbox.py similarity index 99% rename from providers/tests/yandex/secrets/test_lockbox.py rename to providers/yandex/tests/provider_tests/yandex/secrets/test_lockbox.py index 7b9ab19d3e87a..148b09931d1c1 100644 --- a/providers/tests/yandex/secrets/test_lockbox.py +++ b/providers/yandex/tests/provider_tests/yandex/secrets/test_lockbox.py @@ -20,9 +20,6 @@ from unittest.mock import MagicMock, Mock, patch import pytest - -yandexcloud = pytest.importorskip("yandexcloud") - import yandex.cloud.lockbox.v1.payload_pb2 as payload_pb import yandex.cloud.lockbox.v1.secret_pb2 as secret_pb import yandex.cloud.lockbox.v1.secret_service_pb2 as secret_service_pb @@ -30,6 +27,8 @@ from airflow.providers.yandex.secrets.lockbox import LockboxSecretBackend from airflow.providers.yandex.utils.defaults import default_conn_name +yandexcloud = pytest.importorskip("yandexcloud") + class TestLockboxSecretBackend: @patch("airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend._get_secret_value") diff --git a/providers/tests/yandex/secrets/__init__.py b/providers/yandex/tests/provider_tests/yandex/utils/__init__.py similarity index 100% rename from providers/tests/yandex/secrets/__init__.py rename to providers/yandex/tests/provider_tests/yandex/utils/__init__.py diff --git a/providers/tests/yandex/utils/test_credentials.py b/providers/yandex/tests/provider_tests/yandex/utils/test_credentials.py similarity index 100% rename from providers/tests/yandex/utils/test_credentials.py rename to providers/yandex/tests/provider_tests/yandex/utils/test_credentials.py diff --git a/providers/tests/yandex/utils/test_defaults.py b/providers/yandex/tests/provider_tests/yandex/utils/test_defaults.py similarity index 100% rename from providers/tests/yandex/utils/test_defaults.py rename to providers/yandex/tests/provider_tests/yandex/utils/test_defaults.py diff --git a/providers/tests/yandex/utils/test_fields.py b/providers/yandex/tests/provider_tests/yandex/utils/test_fields.py similarity index 100% rename from providers/tests/yandex/utils/test_fields.py rename to providers/yandex/tests/provider_tests/yandex/utils/test_fields.py diff --git a/providers/tests/yandex/utils/test_user_agent.py b/providers/yandex/tests/provider_tests/yandex/utils/test_user_agent.py similarity index 100% rename from providers/tests/yandex/utils/test_user_agent.py rename to providers/yandex/tests/provider_tests/yandex/utils/test_user_agent.py index 8e017a6e674a8..58cd4d3ed3968 100644 --- a/providers/tests/yandex/utils/test_user_agent.py +++ b/providers/yandex/tests/provider_tests/yandex/utils/test_user_agent.py @@ -20,10 +20,10 @@ import pytest -yandexcloud = pytest.importorskip("yandexcloud") - from airflow.providers.yandex.utils.user_agent import provider_user_agent +yandexcloud = pytest.importorskip("yandexcloud") + def test_provider_user_agent(): user_agent = provider_user_agent() diff --git a/providers/tests/yandex/utils/__init__.py b/providers/yandex/tests/system/yandex/__init__.py similarity index 100% rename from providers/tests/yandex/utils/__init__.py rename to providers/yandex/tests/system/yandex/__init__.py diff --git a/providers/tests/system/yandex/example_yandexcloud.py b/providers/yandex/tests/system/yandex/example_yandexcloud.py similarity index 100% rename from providers/tests/system/yandex/example_yandexcloud.py rename to providers/yandex/tests/system/yandex/example_yandexcloud.py index 3cb7226208baa..1e1d4ae417d9a 100644 --- a/providers/tests/system/yandex/example_yandexcloud.py +++ b/providers/yandex/tests/system/yandex/example_yandexcloud.py @@ -18,6 +18,9 @@ from datetime import datetime +from google.protobuf.json_format import MessageToDict +from yandexcloud.operations import OperationError + import yandex.cloud.dataproc.v1.cluster_pb2 as cluster_pb import yandex.cloud.dataproc.v1.cluster_service_pb2 as cluster_service_pb import yandex.cloud.dataproc.v1.cluster_service_pb2_grpc as cluster_service_grpc_pb @@ -26,9 +29,6 @@ import yandex.cloud.dataproc.v1.job_service_pb2 as job_service_pb import yandex.cloud.dataproc.v1.job_service_pb2_grpc as job_service_grpc_pb import yandex.cloud.dataproc.v1.subcluster_pb2 as subcluster_pb -from google.protobuf.json_format import MessageToDict -from yandexcloud.operations import OperationError - from airflow import DAG from airflow.decorators import task from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook diff --git a/providers/tests/system/yandex/example_yandexcloud_dataproc.py b/providers/yandex/tests/system/yandex/example_yandexcloud_dataproc.py similarity index 100% rename from providers/tests/system/yandex/example_yandexcloud_dataproc.py rename to providers/yandex/tests/system/yandex/example_yandexcloud_dataproc.py diff --git a/providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py b/providers/yandex/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py similarity index 100% rename from providers/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py rename to providers/yandex/tests/system/yandex/example_yandexcloud_dataproc_lightweight.py diff --git a/providers/tests/system/yandex/example_yandexcloud_yq.py b/providers/yandex/tests/system/yandex/example_yandexcloud_yq.py similarity index 100% rename from providers/tests/system/yandex/example_yandexcloud_yq.py rename to providers/yandex/tests/system/yandex/example_yandexcloud_yq.py diff --git a/pyproject.toml b/pyproject.toml index 764c76dd1c639..52856e41ae4c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -722,6 +722,7 @@ dev = [ "apache-airflow-providers-trino", "apache-airflow-providers-vertica", "apache-airflow-providers-weaviate", + "apache-airflow-providers-yandex", "apache-airflow-providers-ydb", "apache-airflow-providers-zendesk", "apache-airflow-task-sdk", @@ -815,6 +816,7 @@ apache-airflow-providers-teradata = { workspace = true } apache-airflow-providers-trino = { workspace = true } apache-airflow-providers-vertica = { workspace = true } apache-airflow-providers-weaviate = { workspace = true } +apache-airflow-providers-yandex = { workspace = true } apache-airflow-providers-ydb = { workspace = true } apache-airflow-providers-zendesk = { workspace = true } apache-airflow-task-sdk = { workspace = true } @@ -906,6 +908,7 @@ members = [ "providers/trino", "providers/vertica", "providers/weaviate", + "providers/yandex", "providers/ydb", "providers/zendesk", "task_sdk", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index 8d8205667e979..13a9d7d18fc85 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -116,6 +116,7 @@ services: - ../../../empty:/opt/airflow/providers/trino/src - ../../../empty:/opt/airflow/providers/vertica/src - ../../../empty:/opt/airflow/providers/weaviate/src + - ../../../empty:/opt/airflow/providers/yandex/src - ../../../empty:/opt/airflow/providers/ydb/src - ../../../empty:/opt/airflow/providers/zendesk/src # END automatically generated volumes by generate-volumes-for-sources pre-commit diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index 6e2f23013e726..80d3e6aa58a4c 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -123,6 +123,7 @@ services: - ../../../providers/trino/tests:/opt/airflow/providers/trino/tests - ../../../providers/vertica/tests:/opt/airflow/providers/vertica/tests - ../../../providers/weaviate/tests:/opt/airflow/providers/weaviate/tests + - ../../../providers/yandex/tests:/opt/airflow/providers/yandex/tests - ../../../providers/ydb/tests:/opt/airflow/providers/ydb/tests - ../../../providers/zendesk/tests:/opt/airflow/providers/zendesk/tests # END automatically generated volumes by generate-volumes-for-sources pre-commit diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 826e05e045378..8b1cbdd622205 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -250,17 +250,12 @@ function check_boto_upgrade() { echo echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo - # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud opensearch-py || true - # We need to include few dependencies to pass pip check with other dependencies: - # * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider) - # * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider) set -x # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ - "oss2>=2.14.0" "cryptography<43.0.0" "opensearch-py" + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore set +x - pip check } # Download minimum supported version of sqlalchemy to run tests with it