From 9870a29c17eff9d6be2425f9ee22a4eb7c8530f6 Mon Sep 17 00:00:00 2001 From: Andreas Maier Date: Wed, 3 Jul 2024 09:12:49 +0200 Subject: [PATCH] Cleaned up pip requirements/constraints files and safety policy files Details: * Removed Makefile step that ensures pip is at least at version 9.0.1. * Added requirements-base.txt for specifying requirements for the base packages (setuptools, pip, wheel). * Renamed pip requirements/constraints files for consistency: - dev-requirements.txt to requirements-develop.txt - minimum-constraints.txt to minimum-constraints-develop.txt * Removed inclusion of requirements/constraints files for installation into files for development, and instead creaed a combined file where needed. This allows check_reqs to see all minimum constraints (it ignores the include directive). * Renamed safety policy file .safety-policy-all.yml to .safety-policy-develop.yml because it only covers packages needed for development (safety tool ignored the previously present inclusion of minimum-constraints-install.txt into minimum-constraints.txt). Signed-off-by: Andreas Maier --- ...licy-all.yml => .safety-policy-develop.yml | 6 +--- Makefile | 34 +++++++++++-------- ansible-constraints.txt | 6 ++-- docs/source/development.rst | 6 ++-- galaxy.yml | 11 ++++-- ...nts.txt => minimum-constraints-develop.txt | 21 ++++-------- minimum-constraints-install.txt | 9 ++--- requirements-base.txt | 10 ++++++ ...quirements.txt => requirements-develop.txt | 8 ++--- 9 files changed, 58 insertions(+), 53 deletions(-) rename .safety-policy-all.yml => .safety-policy-develop.yml (82%) rename minimum-constraints.txt => minimum-constraints-develop.txt (82%) create mode 100644 requirements-base.txt rename dev-requirements.txt => requirements-develop.txt (97%) diff --git a/.safety-policy-all.yml b/.safety-policy-develop.yml similarity index 82% rename from .safety-policy-all.yml rename to .safety-policy-develop.yml index 45aca6437..71e705579 100644 --- a/.safety-policy-all.yml +++ b/.safety-policy-develop.yml @@ -1,11 +1,7 @@ # Safety policy file for packages needed for development. # For documentation, see https://docs.pyup.io/docs/safety-20-policy-file -# Note: This policy file is used against the versions in minimum-constraints.txt -# That file includes minimum-constraints-install.txt with '-c'. Since that is -# not supported and ignored by the safety tool, this policy file ends up being -# checked just against the minimum versions for development, without install. -# That's ok since the 'install' safety profile checks the install packages. +# Note: This policy file is used against the versions in minimum-constraints-develop.txt. # Configuration for the 'safety check' command security: diff --git a/Makefile b/Makefile index 4ed1b4c02..a433314af 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ ifndef PACKAGE_LEVEL PACKAGE_LEVEL := latest endif ifeq ($(PACKAGE_LEVEL),minimum) - pip_level_opts := -c minimum-constraints.txt + pip_level_opts := -c minimum-constraints-develop.txt -c minimum-constraints-install.txt pip_level_opts_new := else ifeq ($(PACKAGE_LEVEL),ansible) @@ -127,7 +127,7 @@ sanity_tar_file := tmp_sanity.tar #Safety policy file (for packages needed for installation) safety_install_policy_file := .safety-policy-install.yml -safety_all_policy_file := .safety-policy-all.yml +safety_develop_policy_file := .safety-policy-develop.yml # Packages whose dependencies are checked using pip-missing-reqs # ansible_test and pylint are checked only on officially supported Python versions @@ -225,7 +225,8 @@ help: @echo " PACKAGE_LEVEL - Package level to be used for installing dependent Python" @echo " packages in 'install' and 'develop' targets:" @echo " latest - Latest package versions available on Pypi" - @echo " minimum - A minimum version as defined in minimum-constraints.txt" + @echo " ansible - Specific Ansible versions" + @echo " minimum - A minimum version as defined in minimum-constraints*.txt" @echo " Optional, defaults to 'latest'." @echo ' PYTHON_CMD=... - Name of python command. Default: python' @echo ' PIP_CMD=... - Name of pip command. Default: pip' @@ -269,7 +270,7 @@ check: _check_version $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done @echo '$@ done.' .PHONY: safety -safety: $(done_dir)/safety_all_$(pymn)_$(PACKAGE_LEVEL).done $(done_dir)/safety_install_$(pymn)_$(PACKAGE_LEVEL).done +safety: $(done_dir)/safety_develop_$(pymn)_$(PACKAGE_LEVEL).done $(done_dir)/safety_install_$(pymn)_$(PACKAGE_LEVEL).done @echo "Makefile: $@ done." .PHONY: bandit @@ -331,7 +332,7 @@ endif @echo '$@ done.' .PHONY: check_reqs -check_reqs: _check_version $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done minimum-constraints.txt minimum-constraints-install.txt requirements.txt +check_reqs: _check_version $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done minimum-constraints-develop.txt minimum-constraints-install.txt requirements.txt ifeq ($(PACKAGE_LEVEL),ansible) @echo "Makefile: Warning: Skipping the checking of missing dependencies for PACKAGE_LEVEL=ansible" >&2 else @@ -341,7 +342,9 @@ else # pip-missing-reqs $(src_py_dir) --requirements-file=minimum-constraints-install.txt @echo "Makefile: Done checking missing dependencies of this package" @echo "Makefile: Checking missing dependencies of some development packages" - @rc=0; for pkg in $(check_reqs_packages); do dir=$$($(PYTHON_CMD) -c "import $${pkg} as m,os; dm=os.path.dirname(m.__file__); d=dm if not dm.endswith('site-packages') else m.__file__; print(d)"); cmd="pip-missing-reqs $${dir} --requirements-file=minimum-constraints.txt"; echo $${cmd}; $${cmd}; rc=$$(expr $${rc} + $${?}); done; exit $${rc} + bash -c "cat minimum-constraints-develop.txt minimum-constraints-install.txt >tmp_minimum-constraints.txt" + @rc=0; for pkg in $(check_reqs_packages); do dir=$$($(PYTHON_CMD) -c "import $${pkg} as m,os; dm=os.path.dirname(m.__file__); d=dm if not dm.endswith('site-packages') else m.__file__; print(d)"); cmd="pip-missing-reqs $${dir} --requirements-file=tmp_minimum-constraints.txt"; echo $${cmd}; $${cmd}; rc=$$(expr $${rc} + $${?}); done; exit $${rc} + -$(call RM_FUNC,tmp_minimum-constraints.txt) @echo "Makefile: Done checking missing dependencies of some development packages" endif @echo "Makefile: $@ done." @@ -426,28 +429,29 @@ else @true >/dev/null endif -$(done_dir)/install_deps_$(pymn)_$(PACKAGE_LEVEL).done: Makefile requirements.txt +$(done_dir)/install_deps_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done requirements.txt $(PYTHON_CMD) -m pip install $(pip_level_opts) $(pip_level_opts_new) -r requirements.txt echo "done" >$@ -$(done_dir)/install_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/install_deps_$(pymn)_$(PACKAGE_LEVEL).done $(dist_file) requirements.txt +$(done_dir)/install_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done $(done_dir)/install_deps_$(pymn)_$(PACKAGE_LEVEL).done $(dist_file) requirements.txt ansible-galaxy collection install --force $(dist_file) echo "done" >$@ -$(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/install_pip_$(pymn)_$(PACKAGE_LEVEL).done tools/os_setup.sh dev-requirements.txt +$(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done tools/os_setup.sh requirements-develop.txt bash -c 'tools/os_setup.sh' - $(PYTHON_CMD) -m pip install $(pip_level_opts) $(pip_level_opts_new) -r dev-requirements.txt + $(PYTHON_CMD) -m pip install $(pip_level_opts) $(pip_level_opts_new) -r requirements-develop.txt echo "done" >$@ -$(done_dir)/install_pip_$(pymn)_$(PACKAGE_LEVEL).done: Makefile - bash -c 'pv=$$($(PYTHON_CMD) -m pip --version); if [[ $$pv =~ (^pip [1-8]\..*) ]]; then $(PYTHON_CMD) -m pip install pip==9.0.1; fi' - $(PYTHON_CMD) -m pip install $(pip_level_opts) pip setuptools wheel +$(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done: Makefile requirements-base.txt minimum-constraints-develop.txt minimum-constraints-install.txt + -$(call RM_FUNC,$@) + @echo "Installing/upgrading pip, setuptools and wheel with PACKAGE_LEVEL=$(PACKAGE_LEVEL)" + $(PYTHON_CMD) -m pip install $(pip_level_opts) -r requirements-base.txt echo "done" >$@ -$(done_dir)/safety_all_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(safety_all_policy_file) minimum-constraints.txt minimum-constraints-install.txt +$(done_dir)/safety_develop_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(safety_develop_policy_file) minimum-constraints-develop.txt minimum-constraints-install.txt @echo "Makefile: Running Safety for all packages" -$(call RM_FUNC,$@) - bash -c "safety check --policy-file $(safety_all_policy_file) -r minimum-constraints.txt --full-report || test '$(RUN_TYPE)' != 'release' || exit 1" + bash -c "safety check --policy-file $(safety_develop_policy_file) -r minimum-constraints-develop.txt --full-report || test '$(RUN_TYPE)' != 'release' || exit 1" echo "done" >$@ @echo "Makefile: Done running Safety" diff --git a/ansible-constraints.txt b/ansible-constraints.txt index 9c58ec202..0fdb38efc 100644 --- a/ansible-constraints.txt +++ b/ansible-constraints.txt @@ -1,7 +1,7 @@ -# Pip constraints file for testing of the project. +# Pip constraints file for packages needed for development for specific Ansible versions. # # Make sure that the constraints defined in this file are consistent with -# the versions specified in minimum-constraints.txt and requirements.txt. +# the versions specified in requirements*.txt. # Ansible versions to test on @@ -27,7 +27,7 @@ Jinja2==3.1.3 MarkupSafe==2.0.0 -# Direct dependencies for development (must be consistent with dev-requirements.txt) +# Direct dependencies for development (must be consistent with requirements-develop.txt) # ansible-test yamllint==1.25.0; python_version <= '3.9' diff --git a/docs/source/development.rst b/docs/source/development.rst index 93038b368..0e07bea82 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -168,9 +168,9 @@ test matrix in the ``.github/workflows/test.yml`` file. The following table shows for the full set of test environments which Ansible versions are tested on which Python versions. The 'Packages' column indicates whether the latest versions of Python packages are used (i.e. what pip installs -by default, given the ``requirements.txt`` and ``dev-requirements.txt`` files), -the minimum versions as defined in the ``minimum-constraint.txt`` file, or -specific Ansible versions as defined in the ``ansible-constraint.txt`` file: +by default, given the ``requirements*.txt`` files), +the minimum versions as defined in the ``minimum-constraints*.txt`` files, or +specific Ansible versions as defined in the ``ansible-constraints.txt`` file: ====== ======== ======= ============ Python Packages Ansible Ansible core diff --git a/galaxy.yml b/galaxy.yml index 3280c6308..f0a2f4e72 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -81,7 +81,8 @@ build_ignore: - .flake8 # only in older branches - .travis.yml - .whitesource - - .safety-policy-all.yml + - .safety-policy-all.yml # old name + - .safety-policy-develop.yml # old name - .safety-policy-install.yml - CODE_OF_CONDUCT.md - CONTRIBUTING.rst # only in older branches @@ -89,9 +90,13 @@ build_ignore: - LICENSE - Makefile # requirements.txt is intentionally included for dependent install - - dev-requirements.txt + - dev-requirements.txt # old name + - requirements-base.txt + - requirements-develop.txt - ansible-constraints.txt - - minimum-constraints.txt + - minimum-constraints.txt # old name + - minimum-constraints-install.txt # old name + - minimum-constraints-develop.txt # old name - tools - docs_local - docs diff --git a/minimum-constraints.txt b/minimum-constraints-develop.txt similarity index 82% rename from minimum-constraints.txt rename to minimum-constraints-develop.txt index bb6f75ec7..052065b23 100644 --- a/minimum-constraints.txt +++ b/minimum-constraints-develop.txt @@ -1,19 +1,12 @@ -# Pip constraints file for install and development dependencies. +# Pip constraints file for packages needed for development. # # This file specifies constraints that match the minimum versions specified in -# the requirement files for install and development. They are used for example -# when setting up test environments with the minimum versions of any dependent -# packages. - - -# Note: The included file is processed by Pip but ignored by the Safety tool. -# Thus, safety with the 'all' profile actually only checks development packages, -# but that's ok since the 'install' profile checks the install packages. --c minimum-constraints-install.txt +# requirements-develop.txt. They are used for example when setting up test +# environments with the minimum versions of any dependent packages. # Direct dependencies for development and indirect dependencies for development -# that are needed for some reason (must be consistent with dev-requirements.txt) +# that are needed for some reason (must be consistent with requirements-develop.txt) # Ansible validate-modules (no imports, invoked via validate-modules script) mock==2.0.0 @@ -92,9 +85,9 @@ sphinxcontrib-websupport==1.2.4 autodocsumm==0.2.12 Babel==2.9.1 -# sphinx-versions: Retrieved from git repo in dev-requirements.txt +# sphinx-versions: Retrieved from git repo in requirements-develop.txt # ansible-doc-extractor and it's dependent theme for docs generation (no imports, invoked via script): -# The Makefile defines in doc_build on which Python versions docs are built. Keep in sync with Makefile and dev-requirements.txt. +# The Makefile defines in doc_build on which Python versions docs are built. Keep in sync with Makefile and requirements-develop.txt. ansible-doc-extractor==0.1.8 antsibull-docs-parser==1.0.0 @@ -124,7 +117,7 @@ pip-check-reqs==2.4.3; python_version <= '3.11' pip-check-reqs==2.5.3; python_version >= '3.12' -# Indirect dependencies for development that are not in dev-requirements.txt +# Indirect dependencies for development that are not in requirements-develop.txt alabaster==0.7.9 attrs==19.2.0 diff --git a/minimum-constraints-install.txt b/minimum-constraints-install.txt index bf3c3fa01..f40c905c7 100644 --- a/minimum-constraints-install.txt +++ b/minimum-constraints-install.txt @@ -1,11 +1,12 @@ -# Pip constraints file for install dependencies. +# Pip constraints file for packages needed for install. # # This file specifies constraints that match the minimum versions specified in -# the requirement files for install. They are used for example when setting up -# test environments with the minimum versions of any dependent packages. +# requirements.txt and requirements-base.txt. They are used for example when +# setting up test environments with the minimum versions of any dependent +# packages. -# Base dependencies +# Base dependencies (must be consistent with requirements-base.txt) pip==23.3 setuptools==66.1.0 diff --git a/requirements-base.txt b/requirements-base.txt new file mode 100644 index 000000000..f07a95383 --- /dev/null +++ b/requirements-base.txt @@ -0,0 +1,10 @@ +# Pip requirements file for base packages for a Python env. +# Base packages are those needed for pip and automatic package version detection. + + +# Base dependencies (must be consistent with minimum-constraints-install.txt) + +pip>=23.3 +setuptools>=66.1.0 +setuptools-scm[toml]>=6.4.0 +wheel>=0.38.1 diff --git a/dev-requirements.txt b/requirements-develop.txt similarity index 97% rename from dev-requirements.txt rename to requirements-develop.txt index 48a49958c..83ffdd2d0 100644 --- a/dev-requirements.txt +++ b/requirements-develop.txt @@ -1,12 +1,8 @@ # Pip requirements file for development dependencies. -# Include install dependencies --r requirements.txt - - # Direct dependencies for development and indirect dependencies for development -# that are needed for some reason (must be consistent with minimum-constraints.txt) +# that are needed for some reason (must be consistent with minimum-constraints-develop.txt) # Ansible validate-modules (no imports, invoked via validate-modules script) mock>=2.0.0 @@ -104,7 +100,7 @@ Babel>=2.9.1 git+https://github.com/andy-maier/sphinx-versions.git@1.1.3.post2#egg=sphinx-versions # ansible-doc-extractor and it's dependent theme for docs generation (no imports, invoked via script): -# The Makefile defines in doc_build on which Python versions docs are built. Keep in sync with Makefile and minimum-constraints.txt. +# The Makefile defines in doc_build on which Python versions docs are built. Keep in sync with Makefile and minimum-constraints-develop.txt. ansible-doc-extractor>=0.1.8 antsibull-docs-parser>=1.0.0