ansible-test - Update import test and sanity requirements. (ansible#7…

…6308)

* Add script to freeze sanity requirements.
* Declare sanity test requirements and freeze
* Use pinned requirements for import.plugin test.
* Expand scope of import test for ansible-core.
* Add ignores for galaxy import errors.
* Update test-constraints sanity test.

MANIFEST.in

@@ -29,10 +29,10 @@ recursive-include packaging *
 recursive-include test/ansible_test *.py Makefile
 recursive-include test/integration *
 recursive-include test/lib/ansible_test/config *.yml *.template
-recursive-include test/lib/ansible_test/_data *.cfg *.ini *.ps1 *.txt *.yml coveragerc
+recursive-include test/lib/ansible_test/_data *.cfg *.in *.ini *.ps1 *.txt *.yml coveragerc
 recursive-include test/lib/ansible_test/_util *.cfg *.json *.ps1 *.psd1 *.py *.sh *.txt *.yml
 recursive-include test/lib/ansible_test/_util/controller/sanity/validate-modules validate-modules
-recursive-include test/sanity *.json *.py *.txt
+recursive-include test/sanity *.in *.json *.py *.txt
 recursive-include test/support *.py *.ps1 *.psm1 *.cs
 exclude test/sanity/code-smell/botmeta.*
 exclude test/sanity/code-smell/release-names.*
@@ -46,4 +46,5 @@ include changelogs/changelog.yaml
 recursive-include hacking/build_library *.py
 include hacking/build-ansible.py
 include hacking/test-module.py
+include hacking/update-sanity-requirements.py
 include bin/*

changelogs/fragments/ansible-test-sanity-requirements-update.yaml

@@ -0,0 +1,4 @@
+minor_changes:
+  - ansible-test - Declare public dependencies of ansible-core and use to limit unguarded imports in plugins.
+  - ansible-test - Requirements for the plugin import test are now frozen.
+  - ansible-test - Update sanity test requirements.

docs/docsite/rst/dev_guide/testing/sanity/import.rst

@@ -60,9 +60,15 @@ Ansible allows the following unchecked imports from these specific directories:
 * ansible-core:
 
   * For ``lib/ansible/modules/`` and ``lib/ansible/module_utils/``, unchecked imports are only allowed from the Python standard library;
-  * For ``lib/ansible/plugins/``, unchecked imports are only allowed from the Python standard library, from dependencies of ansible-core, and from ansible-core itself;
+  * For ``lib/ansible/plugins/``, unchecked imports are only allowed from the Python standard library, from public dependencies of ansible-core, and from ansible-core itself;
 
 * collections:
 
   * For ``plugins/modules/`` and ``plugins/module_utils/``, unchecked imports are only allowed from the Python standard library;
-  * For other directories in ``plugins/`` (see `the community collection requirements <https://github.com/ansible-collections/overview/blob/main/collection_requirements.rst#modules-plugins>`_ for a list), unchecked imports are only allowed from the Python standard library, from dependencies of ansible-core, and from ansible-core itself.
+  * For other directories in ``plugins/`` (see `the community collection requirements <https://github.com/ansible-collections/overview/blob/main/collection_requirements.rst#modules-plugins>`_ for a list), unchecked imports are only allowed from the Python standard library, from public dependencies of ansible-core, and from ansible-core itself.
+
+Public dependencies of ansible-core are:
+
+  * Jinja2
+  * PyYAML
+  * MarkupSafe (as a dependency of Jinja2)

hacking/update-sanity-requirements.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# PYTHON_ARGCOMPLETE_OK
+"""Generate frozen sanity test requirements from source requirements files."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import pathlib
+import subprocess
+import tempfile
+import typing as t
+import venv
+
+try:
+    import argcomplete
+except ImportError:
+    argcomplete = None
+
+
+FILE = pathlib.Path(__file__).resolve()
+ROOT = FILE.parent.parent
+SELF = FILE.relative_to(ROOT)
+
+
+@dataclasses.dataclass(frozen=True)
+class SanityTest:
+    name: str
+    requirements_path: pathlib.Path
+    source_path: pathlib.Path
+
+    def freeze_requirements(self) -> None:
+        with tempfile.TemporaryDirectory() as venv_dir:
+            venv.create(venv_dir, with_pip=True)
+
+            python = pathlib.Path(venv_dir, 'bin', 'python')
+            pip = [python, '-m', 'pip', '--disable-pip-version-check']
+            env = dict()
+
+            pip_freeze = subprocess.run(pip + ['freeze'], env=env, check=True, capture_output=True, text=True)
+
+            if pip_freeze.stdout:
+                raise Exception(f'Initial virtual environment is not empty:\n{pip_freeze.stdout}')
+
+            subprocess.run(pip + ['install', 'wheel'], env=env, check=True)  # make bdist_wheel available during pip install
+            subprocess.run(pip + ['install', '-r', self.source_path], env=env, check=True)
+
+            pip_freeze = subprocess.run(pip + ['freeze'], env=env, check=True, capture_output=True, text=True)
+
+        requirements = f'# edit "{self.source_path.name}" and generate with: {SELF} --test {self.name}\n{pip_freeze.stdout}'
+
+        with open(self.requirements_path, 'w') as requirement_file:
+            requirement_file.write(requirements)
+
+    @staticmethod
+    def create(path: pathlib.Path) -> SanityTest:
+        return SanityTest(
+            name=path.stem.replace('sanity.', '').replace('.requirements', ''),
+            requirements_path=path,
+            source_path=path.with_suffix('.in'),
+        )
+
+
+def main() -> None:
+    tests = find_tests()
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--test',
+        metavar='TEST',
+        dest='test_names',
+        action='append',
+        choices=[test.name for test in tests],
+        help='test requirements to update'
+    )
+
+    if argcomplete:
+        argcomplete.autocomplete(parser)
+
+    args = parser.parse_args()
+    test_names: set[str] = set(args.test_names or [])
+
+    tests = [test for test in tests if test.name in test_names] if test_names else tests
+
+    for test in tests:
+        print(f'===[ {test.name} ]===')
+        test.freeze_requirements()
+
+
+def find_tests() -> t.List[SanityTest]:
+    globs = (
+        'test/lib/ansible_test/_data/requirements/sanity.*.txt',
+        'test/sanity/code-smell/*.requirements.txt',
+    )
+
+    tests: t.List[SanityTest] = []
+
+    for glob in globs:
+        tests.extend(get_tests(pathlib.Path(glob)))
+
+    return sorted(tests, key=lambda test: test.name)
+
+
+def get_tests(glob: pathlib.Path) -> t.List[SanityTest]:
+    path = pathlib.Path(ROOT, glob.parent)
+    pattern = glob.name
+
+    return [SanityTest.create(item) for item in path.glob(pattern)]
+
+
+if __name__ == '__main__':
+    main()

test/lib/ansible_test/_data/requirements/sanity.ansible-doc.in

@@ -0,0 +1,3 @@
+jinja2  # ansible-core requirement
+packaging  # ansible-core requirement
+pyyaml  # ansible-core requirement

test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt

@@ -1,7 +1,6 @@
-jinja2 == 3.0.1  # ansible-core requirement
-pyyaml == 5.4.1  # ansible-core requirement
-packaging == 21.0  # ansible-doc requirement
-
-# dependencies
-MarkupSafe == 2.0.1
-pyparsing == 2.4.7
+# edit "sanity.ansible-doc.in" and generate with: hacking/update-sanity-requirements.py --test ansible-doc
+Jinja2==3.0.3
+MarkupSafe==2.0.1
+packaging==21.2
+pyparsing==2.4.7
+PyYAML==6.0

test/lib/ansible_test/_data/requirements/sanity.changelog.in

@@ -0,0 +1,2 @@
+antsibull-changelog
+docutils < 0.18  # match version required by sphinx in the docs-build sanity test

test/lib/ansible_test/_data/requirements/sanity.changelog.txt

@@ -1,9 +1,8 @@
-antsibull-changelog == 0.9.0
-
-# dependencies
-pyyaml == 5.4.1
-docutils == 0.17.1
-packaging == 21.0
-pyparsing == 2.4.7
-rstcheck == 3.3.1
-semantic-version == 2.8.5
+# edit "sanity.changelog.in" and generate with: hacking/update-sanity-requirements.py --test changelog
+antsibull-changelog==0.12.0
+docutils==0.17.1
+packaging==21.2
+pyparsing==2.4.7
+PyYAML==6.0
+rstcheck==3.3.1
+semantic-version==2.8.5

test/lib/ansible_test/_data/requirements/sanity.import.in

@@ -0,0 +1 @@
+pyyaml  # needed for yaml_to_json.py

test/lib/ansible_test/_data/requirements/sanity.import.plugin.in

@@ -0,0 +1,2 @@
+jinja2  # ansible-core requirement
+pyyaml  # ansible-core requirement

test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt

@@ -0,0 +1,4 @@
+# edit "sanity.import.plugin.in" and generate with: hacking/update-sanity-requirements.py --test import.plugin
+Jinja2==3.0.3
+MarkupSafe==2.0.1
+PyYAML==6.0

test/lib/ansible_test/_data/requirements/sanity.import.txt

@@ -1 +1,2 @@
-pyyaml == 5.4.1  # needed for yaml_to_json.py
+# edit "sanity.import.in" and generate with: hacking/update-sanity-requirements.py --test import
+PyYAML==6.0

test/lib/ansible_test/_data/requirements/sanity.integration-aliases.in

@@ -0,0 +1 @@
+pyyaml

test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt

@@ -1 +1,2 @@
-pyyaml == 5.4.1
+# edit "sanity.integration-aliases.in" and generate with: hacking/update-sanity-requirements.py --test integration-aliases
+PyYAML==6.0

test/lib/ansible_test/_data/requirements/sanity.pep8.in

@@ -0,0 +1 @@
+pycodestyle

test/lib/ansible_test/_data/requirements/sanity.pep8.txt

@@ -1 +1,2 @@
-pycodestyle == 2.6.0
+# edit "sanity.pep8.in" and generate with: hacking/update-sanity-requirements.py --test pep8
+pycodestyle==2.8.0

test/lib/ansible_test/_data/requirements/sanity.pylint.in

@@ -0,0 +1,2 @@
+pylint == 2.9.3  # currently vetted version
+pyyaml  # needed for collection_detail.py

test/lib/ansible_test/_data/requirements/sanity.pylint.txt

@@ -1,10 +1,9 @@
-pylint == 2.9.3
-pyyaml == 5.4.1  # needed for collection_detail.py
-
-# dependencies
-astroid == 2.6.6
-isort == 5.9.3
-lazy-object-proxy == 1.6.0
-mccabe == 0.6.1
-toml == 0.10.2
-wrapt == 1.12.1
+# edit "sanity.pylint.in" and generate with: hacking/update-sanity-requirements.py --test pylint
+astroid==2.6.6
+isort==5.10.1
+lazy-object-proxy==1.6.0
+mccabe==0.6.1
+pylint==2.9.3
+PyYAML==6.0
+toml==0.10.2
+wrapt==1.12.1

test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.in

@@ -0,0 +1,2 @@
+pyyaml
+voluptuous

test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt

@@ -1,2 +1,3 @@
-pyyaml == 5.4.1
-voluptuous == 0.12.1
+# edit "sanity.runtime-metadata.in" and generate with: hacking/update-sanity-requirements.py --test runtime-metadata
+PyYAML==6.0
+voluptuous==0.12.2

test/lib/ansible_test/_data/requirements/sanity.validate-modules.in

@@ -0,0 +1,3 @@
+jinja2  # ansible-core requirement
+pyyaml  # needed for collection_detail.py
+voluptuous

test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt

@@ -1,6 +1,5 @@
-jinja2 == 3.0.1 # ansible-core requirement
-pyyaml == 5.4.1  # needed for collection_detail.py
-voluptuous == 0.12.1
-
-# dependencies
-MarkupSafe == 2.0.1
+# edit "sanity.validate-modules.in" and generate with: hacking/update-sanity-requirements.py --test validate-modules
+Jinja2==3.0.3
+MarkupSafe==2.0.1
+PyYAML==6.0
+voluptuous==0.12.2

test/lib/ansible_test/_data/requirements/sanity.yamllint.in

@@ -0,0 +1 @@
+yamllint

test/lib/ansible_test/_data/requirements/sanity.yamllint.txt

@@ -1,5 +1,4 @@
-yamllint == 1.26.0
-
-# dependencies
-pathspec == 0.9.0
-pyyaml == 5.4.1
+# edit "sanity.yamllint.in" and generate with: hacking/update-sanity-requirements.py --test yamllint
+pathspec==0.9.0
+PyYAML==6.0
+yamllint==1.26.3

test/lib/ansible_test/_internal/commands/sanity/__init__.py

@@ -242,7 +242,7 @@ def command_sanity(args):  # type: (SanityConfig) -> None
                     elif isinstance(test, SanitySingleVersion):
                         # single version sanity tests use the controller python
                         test_profile = host_state.controller_profile
-                        virtualenv_python = create_sanity_virtualenv(args, test_profile.python, test.name, context=test.name)
+                        virtualenv_python = create_sanity_virtualenv(args, test_profile.python, test.name)
 
                         if virtualenv_python:
                             virtualenv_yaml = check_sanity_virtualenv_yaml(virtualenv_python)
@@ -1077,24 +1077,20 @@ def create_sanity_virtualenv(
         args,  # type: SanityConfig
         python,  # type: PythonConfig
         name,  # type: str
-        ansible=False,  # type: bool
         coverage=False,  # type: bool
         minimize=False,  # type: bool
-        context=None,  # type: t.Optional[str]
 ):  # type: (...) -> t.Optional[VirtualPythonConfig]
     """Return an existing sanity virtual environment matching the requested parameters or create a new one."""
     commands = collect_requirements(  # create_sanity_virtualenv()
         python=python,
         controller=True,
         virtualenv=False,
         command=None,
-        # used by import tests
-        ansible=ansible,
-        cryptography=ansible,
+        ansible=False,
+        cryptography=False,
         coverage=coverage,
         minimize=minimize,
-        # used by non-import tests
-        sanity=context,
+        sanity=name,
     )
 
     if commands:

test/lib/ansible_test/_internal/commands/sanity/import.py

@@ -87,8 +87,17 @@ class ImportTest(SanityMultipleVersion):
     """Sanity test for proper import exception handling."""
     def filter_targets(self, targets):  # type: (t.List[TestTarget]) -> t.List[TestTarget]
         """Return the given list of test targets, filtered to include only those relevant for the test."""
+        if data_context().content.is_ansible:
+            # all of ansible-core must pass the import test, not just plugins/modules
+            # modules/module_utils will be tested using the module context
+            # everything else will be tested using the plugin context
+            paths = ['lib/ansible']
+        else:
+            # only plugins/modules must pass the import test for collections
+            paths = list(data_context().content.plugin_paths.values())
+
         return [target for target in targets if os.path.splitext(target.path)[1] == '.py' and
-                any(is_subdir(target.path, path) for path in data_context().content.plugin_paths.values())]
+                any(is_subdir(target.path, path) for path in paths)]
 
     @property
     def needs_pypi(self):  # type: () -> bool
@@ -112,9 +121,9 @@ def test(self, args, targets, python):  # type: (SanityConfig, SanityTargets, Py
 
         messages = []
 
-        for import_type, test, controller in (
-                ('module', _get_module_test(True), False),
-                ('plugin', _get_module_test(False), True),
+        for import_type, test in (
+                ('module', _get_module_test(True)),
+                ('plugin', _get_module_test(False)),
         ):
             if import_type == 'plugin' and python.version in REMOTE_ONLY_PYTHON_VERSIONS:
                 continue
@@ -124,7 +133,7 @@ def test(self, args, targets, python):  # type: (SanityConfig, SanityTargets, Py
             if not data and not args.prime_venvs:
                 continue
 
-            virtualenv_python = create_sanity_virtualenv(args, python, f'{self.name}.{import_type}', ansible=controller, coverage=args.coverage, minimize=True)
+            virtualenv_python = create_sanity_virtualenv(args, python, f'{self.name}.{import_type}', coverage=args.coverage, minimize=True)
 
             if not virtualenv_python:
                 display.warning(f'Skipping sanity test "{self.name}" on Python {python.version} due to missing virtual environment support.')
@@ -143,7 +152,7 @@ def test(self, args, targets, python):  # type: (SanityConfig, SanityTargets, Py
             )
 
             if data_context().content.collection:
-                external_python = create_sanity_virtualenv(args, args.controller_python, self.name, context=self.name)
+                external_python = create_sanity_virtualenv(args, args.controller_python, self.name)
 
                 env.update(
                     SANITY_COLLECTION_FULL_NAME=data_context().content.collection.full_name,

test/sanity/code-smell/botmeta.requirements.in

@@ -0,0 +1,2 @@
+pyyaml
+voluptuous

test/sanity/code-smell/botmeta.requirements.txt

@@ -1,2 +1,3 @@
-pyyaml == 5.4.1
-voluptuous == 0.12.1
+# edit "botmeta.requirements.in" and generate with: hacking/update-sanity-requirements.py --test botmeta
+PyYAML==6.0
+voluptuous==0.12.2

test/sanity/code-smell/deprecated-config.requirements.in

@@ -0,0 +1,2 @@
+jinja2  # ansible-core requirement
+pyyaml

test/sanity/code-smell/deprecated-config.requirements.txt

@@ -1,5 +1,4 @@
-jinja2 == 3.0.1 # ansible-core requirement
-pyyaml == 5.4.1
-
-# dependencies
-MarkupSafe == 2.0.1
+# edit "deprecated-config.requirements.in" and generate with: hacking/update-sanity-requirements.py --test deprecated-config
+Jinja2==3.0.3
+MarkupSafe==2.0.1
+PyYAML==6.0

test/sanity/code-smell/docs-build.requirements.in

@@ -0,0 +1,8 @@
+jinja2
+pyyaml
+resolvelib < 0.6.0
+sphinx == 4.2.0
+sphinx-notfound-page
+sphinx-ansible-theme
+straight.plugin
+antsibull