From 1fa335fa572486d1e00615a1a21c0163a0bf284a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 7 Nov 2024 20:06:47 -0600 Subject: [PATCH 1/3] pants-plugins/uses_services: Add env_vars_hint message --- pants-plugins/uses_services/exceptions.py | 5 +++++ pants-plugins/uses_services/redis_rules.py | 22 +++++++++------------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index ba8becdb4d..09a2a63796 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -34,6 +34,8 @@ class ServiceSpecificMessages: service_start_cmd_generic: str + env_vars_hint: str = "" + class ServiceMissingError(Exception): """Error raised when a test uses a service but that service is missing.""" @@ -162,6 +164,9 @@ def generate( """ ) + if messages.env_vars_hint: + instructions += f"\n\n{messages.env_vars_hint}" + return cls( service=service, platform=platform, diff --git a/pants-plugins/uses_services/redis_rules.py b/pants-plugins/uses_services/redis_rules.py index 5863bb1aa3..b53f3bd4d7 100644 --- a/pants-plugins/uses_services/redis_rules.py +++ b/pants-plugins/uses_services/redis_rules.py @@ -143,13 +143,6 @@ async def redis_is_running( if is_running: return RedisIsRunning() - env_vars_hint = dedent( - """ - You can also export the ST2TESTS_REDIS_HOST and ST2TESTS_REDIS_PORT - env vars to automatically use any redis host, local or remote, - while running unit and integration tests. - """ - ) # redis is not running, so raise an error with instructions. raise ServiceMissingError.generate( platform=platform, @@ -162,21 +155,24 @@ async def redis_is_running( """\ sudo yum -y install redis # Don't forget to start redis. - """ - ) - + env_vars_hint, + ), service_start_cmd_deb="systemctl start redis", not_installed_clause_deb="this is one way to install it:", install_instructions_deb=dedent( """\ sudo apt-get install -y mongodb redis # Don't forget to start redis. - """ - ) - + env_vars_hint, + ), service_start_cmd_generic="systemctl start redis", + env_vars_hint=dedent( + """\ + You can also export the ST2TESTS_REDIS_HOST and ST2TESTS_REDIS_PORT + env vars to automatically use any redis host, local or remote, + while running unit and integration tests. + """ + ), ), ) From 5c384d9abcb12679e242a46bfba59a12a0349f72 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 11 Nov 2024 14:32:37 -0600 Subject: [PATCH 2/3] tests: respect ST2_DATABASE__* env vars --- pants-plugins/uses_services/mongo_rules.py | 64 ++++++++++++++++--- .../uses_services/mongo_rules_test.py | 16 ++++- .../uses_services/scripts/is_mongo_running.py | 4 +- pants.toml | 7 ++ 4 files changed, 80 insertions(+), 11 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index b537ff7aff..17b6835386 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations from dataclasses import dataclass from textwrap import dedent @@ -27,6 +28,8 @@ VenvPexProcess, rules as pex_rules, ) +from pants.core.goals.test import TestExtraEnv +from pants.engine.env_vars import EnvironmentVars from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.rules import collect_rules, Get, MultiGet, rule from pants.engine.process import FallibleProcessResult, ProcessCacheScope @@ -55,21 +58,50 @@ class UsesMongoRequest: # for unit tests: st2tests/st2tests/config.py # for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf # (changed by setting ST2_CONFIG_PATH env var inside the tests) - # TODO: for unit tests: modify code to pull db connect settings from env vars - # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot db_name - # and either add env vars for db connect settings or modify conf files as well - - # with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables. + # These can also be updated via the ST2_DATABASE__* env vars (which oslo_config reads). + # Integration tests should pass these changes onto subprocesses via the same env vars. db_host: str = "127.0.0.1" # localhost in test_db.DbConnectionTestCase db_port: int = 27017 # db_name is "st2" in test_db.DbConnectionTestCase db_name: str = "st2-test{}" # {} will be replaced by test slot (a format string) + # username and password are not required to validate connectivity, so this doesn't have them. + db_connection_timeout: int = 3000 execution_slot_var: str = "ST2TESTS_PARALLEL_SLOT" + @classmethod + def from_env( + cls, execution_slot_var: str, env: EnvironmentVars + ) -> UsesMongoRequest: + default = cls() + host = env.get("ST2_DATABASE__HOST", default.db_host) + port_raw = env.get("ST2_DATABASE__PORT", str(default.db_port)) + db_name = default.db_name # not overridable via ST2_DATABASE__DB_NAME + db_connection_timeout_raw = env.get( + "ST2_DATABASE__CONNECTION_TIMEOUT", str(default.db_connection_timeout) + ) + + try: + port = int(port_raw) + except (TypeError, ValueError): + port = default.db_port + + try: + db_connection_timeout = int(db_connection_timeout_raw) + except (TypeError, ValueError): + db_connection_timeout = default.db_connection_timeout + + return cls( + db_host=host, + db_port=port, + db_name=db_name, + db_connection_timeout=db_connection_timeout, + execution_slot_var=execution_slot_var, + ) + @dataclass(frozen=True) class MongoIsRunning: @@ -90,7 +122,9 @@ def is_applicable(cls, target: Target) -> bool: level=LogLevel.DEBUG, ) async def mongo_is_running_for_pytest( - request: PytestUsesMongoRequest, pytest: PyTest + request: PytestUsesMongoRequest, + pytest: PyTest, + test_extra_env: TestExtraEnv, ) -> PytestPluginSetup: # TODO: delete these comments once the Makefile becomes irrelevant. # the comments explore how the Makefile prepares to run and runs tests @@ -109,7 +143,9 @@ async def mongo_is_running_for_pytest( # this will raise an error if mongo is not running _ = await Get( MongoIsRunning, - UsesMongoRequest(execution_slot_var=pytest.execution_slot_var or ""), + UsesMongoRequest.from_env( + execution_slot_var=pytest.execution_slot_var or "", env=test_extra_env.env + ), ) return PytestPluginSetup() @@ -151,8 +187,8 @@ async def mongo_is_running( str(request.db_port), request.db_name, str(request.db_connection_timeout), - request.execution_slot_var, ), + extra_env={"PANTS_PYTEST_EXECUTION_SLOT_VAR": request.execution_slot_var}, input_digest=script_digest, execution_slot_variable=request.execution_slot_var, description="Checking to see if Mongo is up and accessible.", @@ -200,6 +236,18 @@ async def mongo_is_running( """ ), service_start_cmd_generic="systemctl start mongod", + env_vars_hint=dedent( + """\ + You can also export the ST2_DATABASE__HOST and ST2_DATABASE__PORT + env vars to automatically use any MongoDB host, local or remote, + while running unit and integration tests. Note that you cannot + override the db-name, which is st2-test suffixed by an integer + to allow tests to safely run in parallel. If needed you can also + override the default username, password, and connection timeout + by exporting one or more of: ST2_DATABASE__USERNAME, + ST2_DATABASE__PASSWORD, and ST2_DATABASE__CONNECTION_TIMEOUT. + """ + ), ), ) diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index ff8a3c1284..03ccb0528e 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -51,7 +51,17 @@ def run_mongo_is_running( "--backend-packages=uses_services", *(extra_args or ()), ], - env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + env_inherit={ + "PATH", + "PYENV_ROOT", + "HOME", + "ST2_DATABASE__HOST", + "ST2_DATABASE__PORT", + "ST2_DATABASE__USERNAME", + "ST2_DATABASE__PASSWORD", + "ST2_DATABASE__CONNECTION_TIMEOUT", + "ST2TESTS_PARALLEL_SLOT", + }, ) result = rule_runner.request( MongoIsRunning, @@ -62,7 +72,9 @@ def run_mongo_is_running( # Warning this requires that mongo be running def test_mongo_is_running(rule_runner: RuleRunner) -> None: - request = UsesMongoRequest() + request = UsesMongoRequest.from_env( + execution_slot_var="ST2TESTS_PARALLEL_SLOT", env=rule_runner.environment + ) mock_platform = platform(os="TestMock") # we are asserting that this does not raise an exception diff --git a/pants-plugins/uses_services/scripts/is_mongo_running.py b/pants-plugins/uses_services/scripts/is_mongo_running.py index a9db658eb7..358d558bf8 100644 --- a/pants-plugins/uses_services/scripts/is_mongo_running.py +++ b/pants-plugins/uses_services/scripts/is_mongo_running.py @@ -54,7 +54,9 @@ def _is_mongo_running( db_name = args.get(3, "st2-test{}") connection_timeout_ms = args.get(4, 3000) - slot_var = args.get(5, "ST2TESTS_PARALLEL_SLOT") + slot_var = os.environ.get( + "PANTS_PYTEST_EXECUTION_SLOT_VAR", "ST2TESTS_PARALLEL_SLOT" + ) db_name = db_name.format(os.environ.get(slot_var) or "") is_running = _is_mongo_running( diff --git a/pants.toml b/pants.toml index 6dc6bba87f..23a2c5f4a2 100644 --- a/pants.toml +++ b/pants.toml @@ -240,6 +240,13 @@ extra_env_vars = [ # Use this so that the test system does not require the stanley user. # For example: export ST2TESTS_SYSTEM_USER=${USER} "ST2TESTS_SYSTEM_USER", + # Use these to override MongoDB connection details + # "ST2_DATABASE__HOST", # Tests override this with "127.0.0.1" + "ST2_DATABASE__PORT", + # "ST2_DATABASE__DB_NAME", # Tests override this with: "st2-test{ST2TESTS_PARALLEL_SLOT}" + "ST2_DATABASE__CONNECTION_TIMEOUT", + "ST2_DATABASE__USERNAME", + "ST2_DATABASE__PASSWORD", # Use these to override the redis host and port "ST2TESTS_REDIS_HOST", "ST2TESTS_REDIS_PORT", From 3cf76caa79377894209e5059b26e8e984842eeb2 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 16 Nov 2024 23:33:23 -0600 Subject: [PATCH 3/3] add changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c3ae075a31..7e5ddb7b73 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -69,7 +69,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 #6260 #6269 #6275 + #6254 #6258 #6259 #6260 #6269 #6275 #6278 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11