From a1c8717479c5fba1f014e77f7f195e21b920a3f2 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 30 Jan 2025 16:30:12 +0100 Subject: [PATCH] Remove requests limits and restore opensearch tests for latest boto (#46289) The requests limits were removing opensearch during "latest boto" tests but this is not needed any more, since apache.beam latest version already supports requests 2.32.*. That also unblocks moving opensearch to a separate provider project. --- Dockerfile.ci | 9 +-------- scripts/docker/entrypoint_ci.sh | 9 +-------- tests/always/test_example_dags.py | 3 +-- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 23b743165d3b0..396c04b3fd94e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -935,17 +935,10 @@ function check_boto_upgrade() { # We need to include few dependencies to pass pip check with other dependencies: # * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider) # * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider) - # * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider) - # * yandexcloud requirements for requests does not match those of apache.beam and latest botocore - # Both requests and yandexcloud exclusion above might be removed after - # https://github.com/apache/beam/issues/32080 is addressed - # This is already addressed and planned for 2.59.0 release. - # When you remove yandexcloud and opensearch from the above list, you can also remove the - # optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always". set -x # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ - "oss2>=2.14.0" "cryptography<43.0.0" "requests!=2.32.*,<3.0.0,>=2.24.0" + "oss2>=2.14.0" "cryptography<43.0.0" "opensearch-py" set +x pip check } diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index df676df0dde41..826e05e045378 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -255,17 +255,10 @@ function check_boto_upgrade() { # We need to include few dependencies to pass pip check with other dependencies: # * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider) # * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider) - # * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider) - # * yandexcloud requirements for requests does not match those of apache.beam and latest botocore - # Both requests and yandexcloud exclusion above might be removed after - # https://github.com/apache/beam/issues/32080 is addressed - # This is already addressed and planned for 2.59.0 release. - # When you remove yandexcloud and opensearch from the above list, you can also remove the - # optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always". set -x # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ - "oss2>=2.14.0" "cryptography<43.0.0" "requests!=2.32.*,<3.0.0,>=2.24.0" + "oss2>=2.14.0" "cryptography<43.0.0" "opensearch-py" set +x pip check } diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index f4b0cc7c75f62..c827de6090687 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -44,8 +44,7 @@ # The format of the dictionary is as follows: # key: the regexp matching the file to be excluded, # value: a dictionary containing package distributions with an optional version specifier, e.g., >=2.3.4 - ".*example_bedrock_retrieve_and_generate.py": {"opensearch-py": None}, - ".*example_opensearch.py": {"opensearch-py": None}, + # yandexcloud is automatically removed in case botocore is upgraded to latest r".*example_yandexcloud.*\.py": {"yandexcloud": None}, } IGNORE_AIRFLOW_PROVIDER_DEPRECATION_WARNING: tuple[str, ...] = (