FIX: Update embedding script tests (#974)

Co-authored-by: pyansys-ci-bot <[email protected]>
Co-authored-by: Dipin Nair <[email protected]>

doc/changelog.d/974.fixed.md

@@ -0,0 +1 @@
+Update embedding script tests

tests/conftest.py

@@ -54,6 +54,8 @@
 # Check if Mechanical is installed
 # NOTE: checks in this order to get the newest installed version
 
+# Ignore functions starts with `test` from scripts folder
+collect_ignore = ["scripts"]
 
 valid_rver = [str(each) for each in SUPPORTED_MECHANICAL_VERSIONS]
 
@@ -201,6 +203,17 @@ def rootdir():
     yield base.parent
 
 
+# @pytest.fixture()
+# def subprocess_pass_expected(pytestconfig):
+#     """Checks for conditions to see if scripts run in subprocess are expected to pass or not."""
+
+#     version = pytestconfig.getoption("ansys_version")
+#     if os.name != "nt" and int(version) < 251:
+#         yield False
+#     else:
+#         yield True
+
+
 @pytest.fixture()
 def disable_cli():
     ansys.mechanical.core.run.DRY_RUN = True

tests/embedding/test_background.py

@@ -27,46 +27,26 @@
 
 import pytest
 
-
-def _run_background_app_test_process(
-    rootdir: str, run_subprocess, pytestconfig, testname: str, pass_expected: bool = None
-) -> typing.Tuple[bytes, bytes]:
-    """Run the process and return stdout and stderr after it finishes."""
-    version = pytestconfig.getoption("ansys_version")
-    script = os.path.join(rootdir, "tests", "scripts", "background_app_test.py")
-    process, stdout, stderr = run_subprocess(
-        [sys.executable, script, version, testname], None, pass_expected
-    )
-    return stdout, stderr
-
-
-def _assert_success(stdout: str, pass_expected: bool) -> bool:
-    """Check whether the process ran to completion from its stdout
-
-    Duplicate of the `_assert_success` function in test_logger.py
-    """
-
-    if pass_expected:
-        assert "@@success@@" in stdout
-    else:
-        assert "@@success@@" not in stdout
+from .test_logger import _assert_success
 
 
 def _run_background_app_test(
     run_subprocess, rootdir: str, pytestconfig, testname: str, pass_expected: bool = True
-) -> str:
-    """Test stderr logging using a subprocess.
+) -> typing.Tuple[bytes, bytes]:
+    """Run the process and return stdout and stderr after it finishes."""
 
-    Also ensure that the subprocess either passes or fails based on pass_expected
+    version = pytestconfig.getoption("ansys_version")
+    script = os.path.join(rootdir, "tests", "scripts", "background_app_test.py")
 
-    Returns the stderr of the subprocess as a string.
-    """
     subprocess_pass_expected = pass_expected
-    if pass_expected == True and os.name != "nt":
-        subprocess_pass_expected = False
-    stdout, stderr = _run_background_app_test_process(
-        rootdir, run_subprocess, pytestconfig, testname, subprocess_pass_expected
+    if pass_expected and os.name != "nt":
+        if int(version) < 251 or testname == "multiple_instances":
+            subprocess_pass_expected = False
+
+    process, stdout, stderr = run_subprocess(
+        [sys.executable, script, version, testname], None, subprocess_pass_expected
     )
+
     if not subprocess_pass_expected:
         stdout = stdout.decode()
         _assert_success(stdout, pass_expected)
@@ -75,6 +55,7 @@ def _run_background_app_test(
 
 
 @pytest.mark.embedding_scripts
+@pytest.mark.embedding_backgroundapp
 def test_background_app_multiple_instances(rootdir, run_subprocess, pytestconfig):
     """Multiple instances of background app can be used."""
     stderr = _run_background_app_test(
@@ -87,6 +68,7 @@ def test_background_app_multiple_instances(rootdir, run_subprocess, pytestconfig
 
 
 @pytest.mark.embedding_scripts
+@pytest.mark.embedding_backgroundapp
 def test_background_app_use_stopped(rootdir, run_subprocess, pytestconfig):
     """Multiple instances of background app cannot be used after an instance is stopped."""
     stderr = _run_background_app_test(
@@ -96,6 +78,7 @@ def test_background_app_use_stopped(rootdir, run_subprocess, pytestconfig):
 
 
 @pytest.mark.embedding_scripts
+@pytest.mark.embedding_backgroundapp
 def test_background_app_initialize_stopped(rootdir, run_subprocess, pytestconfig):
     """Multiple instances of background app cannot be used after an instance is stopped."""
     stderr = _run_background_app_test(

tests/embedding/test_logger.py

@@ -45,18 +45,31 @@ def _unset_var(env, var) -> None:
     return env
 
 
-def _run_embedding_log_test_process(
-    rootdir: str, run_subprocess, pytestconfig, testname: str, pass_expected: bool = None
+def _run_embedding_log_test(
+    run_subprocess,
+    rootdir: str,
+    pytestconfig,
+    testname: str,
+    pass_expected: bool = True,
 ) -> typing.Tuple[bytes, bytes]:
     """Runs the process and returns it after it finishes"""
     version = pytestconfig.getoption("ansys_version")
     embedded_py = os.path.join(rootdir, "tests", "scripts", "embedding_log_test.py")
+
+    subprocess_pass_expected = pass_expected
+    if pass_expected == True and os.name != "nt" and int(version) < 251:
+        subprocess_pass_expected = False
+
     process, stdout, stderr = run_subprocess(
         [sys.executable, embedded_py, version, testname],
         _get_env_without_logging_variables(),
-        pass_expected,
+        subprocess_pass_expected,
     )
-    return stdout, stderr
+    if not subprocess_pass_expected:
+        stdout = stdout.decode()
+        _assert_success(stdout, pass_expected)
+    stderr = stderr.decode()
+    return stderr
 
 
 def _assert_success(stdout: str, pass_expected: bool) -> int:
@@ -74,31 +87,8 @@ def _assert_success(stdout: str, pass_expected: bool) -> int:
         assert "@@success@@" not in stdout
 
 
-def _run_embedding_log_test(
-    run_subprocess, rootdir: str, pytestconfig, testname: str, pass_expected: bool = True
-) -> str:
-    """Test stderr logging using a subprocess.
-
-    Also ensure that the subprocess either passes or fails based on pass_expected
-    Mechanical logging all goes into the process stderr at the C level, but capturing
-    that from python isn't possible because python's stderr stream isn't aware of content
-    that doesn't come from python (or its C/API)
-
-    Returns the stderr
-    """
-    subprocess_pass_expected = pass_expected
-    if pass_expected == True and os.name != "nt":
-        subprocess_pass_expected = False
-    stdout, stderr = _run_embedding_log_test_process(
-        rootdir, run_subprocess, pytestconfig, testname, subprocess_pass_expected
-    )
-    stdout = stdout.decode()
-    stderr = stderr.decode()
-    _assert_success(stdout, pass_expected)
-    return stderr
-
-
 @pytest.mark.embedding_scripts
+@pytest.mark.embedding_logging
 def test_logging_write_log_before_init(rootdir, run_subprocess, pytestconfig):
     """Test that an error is thrown when trying to log before initializing"""
     stderr = _run_embedding_log_test(
@@ -108,28 +98,37 @@ def test_logging_write_log_before_init(rootdir, run_subprocess, pytestconfig):
 
 
 @pytest.mark.embedding_scripts
+@pytest.mark.embedding_logging
 def test_logging_write_info_after_initialize_with_error_level(
     rootdir, run_subprocess, pytestconfig
 ):
     """Test that no output is written when an info is logged when configured at the error level."""
     stderr = _run_embedding_log_test(
-        run_subprocess, rootdir, pytestconfig, "log_info_after_initialize_with_error_level"
+        run_subprocess,
+        rootdir,
+        pytestconfig,
+        "log_info_after_initialize_with_error_level",
     )
     assert "0xdeadbeef" not in stderr
 
 
 @pytest.mark.parametrize("addin_configuration", ["Mechanical", "WorkBench"])
 @pytest.mark.embedding_scripts
+@pytest.mark.embedding_logging
 @pytest.mark.minimum_version(241)
 def test_addin_configuration(rootdir, run_subprocess, pytestconfig, addin_configuration):
     """Test that mechanical can start with both the Mechanical and WorkBench configuration."""
     stderr = _run_embedding_log_test(
-        run_subprocess, rootdir, pytestconfig, f"log_configuration_{addin_configuration}"
+        run_subprocess,
+        rootdir,
+        pytestconfig,
+        f"log_configuration_{addin_configuration}",
     )
     assert f"{addin_configuration} configuration!" in stderr
 
 
 @pytest.mark.embedding_scripts
+@pytest.mark.embedding_logging
 def test_logging_write_error_after_initialize_with_info_level(
     rootdir, run_subprocess, pytestconfig
 ):
@@ -141,11 +140,17 @@ def test_logging_write_error_after_initialize_with_info_level(
 
 
 @pytest.mark.embedding_scripts
+@pytest.mark.embedding_logging
 def test_logging_level_before_and_after_initialization(rootdir, run_subprocess, pytestconfig):
     """Test logging level API  before and after initialization."""
-    stdout, stderr = _run_embedding_log_test_process(
-        rootdir, run_subprocess, pytestconfig, "log_check_can_log_message"
+    _run_embedding_log_test(run_subprocess, rootdir, pytestconfig, "log_check_can_log_message")
+
+
+@pytest.mark.embedding_scripts
+@pytest.mark.embedding_logging
+def test_logging_all_level(rootdir, run_subprocess, pytestconfig):
+    """Test all logging level after initialization."""
+    stderr = _run_embedding_log_test(
+        run_subprocess, rootdir, pytestconfig, "log_check_all_log_level"
     )
-    stdout = stdout.decode()
-    stderr = stderr.decode()
-    _assert_success(stdout, True)
+    assert all(keyword in stderr for keyword in ["debug", "warning", "info", "error", "fatal"])