Cleaned up pip requirements/constraints files and safety policy files

Details: * Removed Makefile step that ensures pip is at least at version 9.0.1. * Added requirements-base.txt for specifying requirements for the base packages (setuptools, pip, wheel). * Renamed pip requirements/constraints files for consistency: - dev-requirements.txt to requirements-develop.txt - minimum-constraints.txt to minimum-constraints-develop.txt * Removed inclusion of requirements/constraints files for installation into files for development, and instead creaed a combined file where needed. This allows check_reqs to see all minimum constraints (it ignores the include directive). * Renamed safety policy file .safety-policy-all.yml to .safety-policy-develop.yml because it only covers packages needed for development (safety tool ignored the previously present inclusion of minimum-constraints-install.txt into minimum-constraints.txt). Signed-off-by: Andreas Maier <[email protected]>
zhmcclient · Jul 3, 2024 · 9870a29 · 9870a29
1 parent 85abb79
commit 9870a29
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 53 deletions.
diff --git a/.safety-policy-all.yml → .safety-policy-develop.yml b/.safety-policy-all.yml → .safety-policy-develop.yml
@@ -1,11 +1,7 @@
 # Safety policy file for packages needed for development.
 # For documentation, see https://docs.pyup.io/docs/safety-20-policy-file
 
-# Note: This policy file is used against the versions in minimum-constraints.txt
-# That file includes minimum-constraints-install.txt with '-c'. Since that is
-# not supported and ignored by the safety tool, this policy file ends up being
-# checked just against the minimum versions for development, without install.
-# That's ok since the 'install' safety profile checks the install packages.
+# Note: This policy file is used against the versions in minimum-constraints-develop.txt.
 
 # Configuration for the 'safety check' command
 security:

diff --git a/Makefile b/Makefile
@@ -30,7 +30,7 @@ ifndef PACKAGE_LEVEL
   PACKAGE_LEVEL := latest
 endif
 ifeq ($(PACKAGE_LEVEL),minimum)
-  pip_level_opts := -c minimum-constraints.txt
+  pip_level_opts := -c minimum-constraints-develop.txt -c minimum-constraints-install.txt
   pip_level_opts_new :=
 else
   ifeq ($(PACKAGE_LEVEL),ansible)
@@ -127,7 +127,7 @@ sanity_tar_file := tmp_sanity.tar
 
 #Safety policy file (for packages needed for installation)
 safety_install_policy_file := .safety-policy-install.yml
-safety_all_policy_file := .safety-policy-all.yml
+safety_develop_policy_file := .safety-policy-develop.yml
 
 # Packages whose dependencies are checked using pip-missing-reqs
 # ansible_test and pylint are checked only on officially supported Python versions
@@ -225,7 +225,8 @@ help:
 	@echo "  PACKAGE_LEVEL - Package level to be used for installing dependent Python"
 	@echo "      packages in 'install' and 'develop' targets:"
 	@echo "        latest - Latest package versions available on Pypi"
-	@echo "        minimum - A minimum version as defined in minimum-constraints.txt"
+	@echo "        ansible - Specific Ansible versions"
+	@echo "        minimum - A minimum version as defined in minimum-constraints*.txt"
 	@echo "      Optional, defaults to 'latest'."
 	@echo '  PYTHON_CMD=... - Name of python command. Default: python'
 	@echo '  PIP_CMD=... - Name of pip command. Default: pip'
@@ -269,7 +270,7 @@ check: _check_version $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done
 	@echo '$@ done.'
 
 .PHONY: safety
-safety: $(done_dir)/safety_all_$(pymn)_$(PACKAGE_LEVEL).done $(done_dir)/safety_install_$(pymn)_$(PACKAGE_LEVEL).done
+safety: $(done_dir)/safety_develop_$(pymn)_$(PACKAGE_LEVEL).done $(done_dir)/safety_install_$(pymn)_$(PACKAGE_LEVEL).done
 	@echo "Makefile: $@ done."
 
 .PHONY: bandit
@@ -331,7 +332,7 @@ endif
 	@echo '$@ done.'
 
 .PHONY: check_reqs
-check_reqs: _check_version $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done minimum-constraints.txt minimum-constraints-install.txt requirements.txt
+check_reqs: _check_version $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done minimum-constraints-develop.txt minimum-constraints-install.txt requirements.txt
 ifeq ($(PACKAGE_LEVEL),ansible)
 	@echo "Makefile: Warning: Skipping the checking of missing dependencies for PACKAGE_LEVEL=ansible" >&2
 else
@@ -341,7 +342,9 @@ else
 	# pip-missing-reqs $(src_py_dir) --requirements-file=minimum-constraints-install.txt
 	@echo "Makefile: Done checking missing dependencies of this package"
 	@echo "Makefile: Checking missing dependencies of some development packages"
-	@rc=0; for pkg in $(check_reqs_packages); do dir=$$($(PYTHON_CMD) -c "import $${pkg} as m,os; dm=os.path.dirname(m.__file__); d=dm if not dm.endswith('site-packages') else m.__file__; print(d)"); cmd="pip-missing-reqs $${dir} --requirements-file=minimum-constraints.txt"; echo $${cmd}; $${cmd}; rc=$$(expr $${rc} + $${?}); done; exit $${rc}
+	bash -c "cat minimum-constraints-develop.txt minimum-constraints-install.txt >tmp_minimum-constraints.txt"
+	@rc=0; for pkg in $(check_reqs_packages); do dir=$$($(PYTHON_CMD) -c "import $${pkg} as m,os; dm=os.path.dirname(m.__file__); d=dm if not dm.endswith('site-packages') else m.__file__; print(d)"); cmd="pip-missing-reqs $${dir} --requirements-file=tmp_minimum-constraints.txt"; echo $${cmd}; $${cmd}; rc=$$(expr $${rc} + $${?}); done; exit $${rc}
+	-$(call RM_FUNC,tmp_minimum-constraints.txt)
 	@echo "Makefile: Done checking missing dependencies of some development packages"
 endif
 	@echo "Makefile: $@ done."
@@ -426,28 +429,29 @@ else
 	@true >/dev/null
 endif
 
-$(done_dir)/install_deps_$(pymn)_$(PACKAGE_LEVEL).done: Makefile requirements.txt
+$(done_dir)/install_deps_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done requirements.txt
 	$(PYTHON_CMD) -m pip install $(pip_level_opts) $(pip_level_opts_new) -r requirements.txt
 	echo "done" >$@
 
-$(done_dir)/install_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/install_deps_$(pymn)_$(PACKAGE_LEVEL).done $(dist_file) requirements.txt
+$(done_dir)/install_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done $(done_dir)/install_deps_$(pymn)_$(PACKAGE_LEVEL).done $(dist_file) requirements.txt
 	ansible-galaxy collection install --force $(dist_file)
 	echo "done" >$@
 
-$(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/install_pip_$(pymn)_$(PACKAGE_LEVEL).done tools/os_setup.sh dev-requirements.txt
+$(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done: Makefile $(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done tools/os_setup.sh requirements-develop.txt
 	bash -c 'tools/os_setup.sh'
-	$(PYTHON_CMD) -m pip install $(pip_level_opts) $(pip_level_opts_new) -r dev-requirements.txt
+	$(PYTHON_CMD) -m pip install $(pip_level_opts) $(pip_level_opts_new) -r requirements-develop.txt
 	echo "done" >$@
 
-$(done_dir)/install_pip_$(pymn)_$(PACKAGE_LEVEL).done: Makefile
-	bash -c 'pv=$$($(PYTHON_CMD) -m pip --version); if [[ $$pv =~ (^pip [1-8]\..*) ]]; then $(PYTHON_CMD) -m pip install pip==9.0.1; fi'
-	$(PYTHON_CMD) -m pip install $(pip_level_opts) pip setuptools wheel
+$(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done: Makefile requirements-base.txt minimum-constraints-develop.txt minimum-constraints-install.txt
+	-$(call RM_FUNC,$@)
+	@echo "Installing/upgrading pip, setuptools and wheel with PACKAGE_LEVEL=$(PACKAGE_LEVEL)"
+	$(PYTHON_CMD) -m pip install $(pip_level_opts) -r requirements-base.txt
 	echo "done" >$@
 
-$(done_dir)/safety_all_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(safety_all_policy_file) minimum-constraints.txt minimum-constraints-install.txt
+$(done_dir)/safety_develop_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(safety_develop_policy_file) minimum-constraints-develop.txt minimum-constraints-install.txt
 	@echo "Makefile: Running Safety for all packages"
 	-$(call RM_FUNC,$@)
-	bash -c "safety check --policy-file $(safety_all_policy_file) -r minimum-constraints.txt --full-report || test '$(RUN_TYPE)' != 'release' || exit 1"
+	bash -c "safety check --policy-file $(safety_develop_policy_file) -r minimum-constraints-develop.txt --full-report || test '$(RUN_TYPE)' != 'release' || exit 1"
 	echo "done" >$@
 	@echo "Makefile: Done running Safety"
 

diff --git a/ansible-constraints.txt b/ansible-constraints.txt
@@ -1,7 +1,7 @@
-# Pip constraints file for testing of the project.
+# Pip constraints file for packages needed for development for specific Ansible versions.
 #
 # Make sure that the constraints defined in this file are consistent with
-# the versions specified in minimum-constraints.txt and requirements.txt.
+# the versions specified in requirements*.txt.
 
 
 # Ansible versions to test on
@@ -27,7 +27,7 @@ Jinja2==3.1.3
 MarkupSafe==2.0.0
 
 
-# Direct dependencies for development (must be consistent with dev-requirements.txt)
+# Direct dependencies for development (must be consistent with requirements-develop.txt)
 
 # ansible-test
 yamllint==1.25.0; python_version <= '3.9'

diff --git a/docs/source/development.rst b/docs/source/development.rst
@@ -168,9 +168,9 @@ test matrix in the ``.github/workflows/test.yml`` file.
 The following table shows for the full set of test environments which Ansible
 versions are tested on which Python versions. The 'Packages' column indicates
 whether the latest versions of Python packages are used (i.e. what pip installs
-by default, given the ``requirements.txt`` and ``dev-requirements.txt`` files),
-the minimum versions as defined in the ``minimum-constraint.txt`` file, or
-specific Ansible versions as defined in the ``ansible-constraint.txt`` file:
+by default, given the ``requirements*.txt`` files),
+the minimum versions as defined in the ``minimum-constraints*.txt`` files, or
+specific Ansible versions as defined in the ``ansible-constraints.txt`` file:
 
 ======  ========  =======  ============
 Python  Packages  Ansible  Ansible core

diff --git a/galaxy.yml b/galaxy.yml
@@ -81,17 +81,22 @@ build_ignore:
   - .flake8  # only in older branches
   - .travis.yml
   - .whitesource
-  - .safety-policy-all.yml
+  - .safety-policy-all.yml  # old name
+  - .safety-policy-develop.yml  # old name
   - .safety-policy-install.yml
   - CODE_OF_CONDUCT.md
   - CONTRIBUTING.rst  # only in older branches
   - DCO1.1.txt  # only in older branches
   - LICENSE
   - Makefile
   # requirements.txt is intentionally included for dependent install
-  - dev-requirements.txt
+  - dev-requirements.txt  # old name
+  - requirements-base.txt
+  - requirements-develop.txt
   - ansible-constraints.txt
-  - minimum-constraints.txt
+  - minimum-constraints.txt  # old name
+  - minimum-constraints-install.txt  # old name
+  - minimum-constraints-develop.txt  # old name
   - tools
   - docs_local
   - docs

diff --git a/minimum-constraints.txt → minimum-constraints-develop.txt b/minimum-constraints.txt → minimum-constraints-develop.txt
@@ -1,19 +1,12 @@
-# Pip constraints file for install and development dependencies.
+# Pip constraints file for packages needed for development.
 #
 # This file specifies constraints that match the minimum versions specified in
-# the requirement files for install and development. They are used for example
-# when setting up test environments with the minimum versions of any dependent
-# packages.
-
-
-# Note: The included file is processed by Pip but ignored by the Safety tool.
-# Thus, safety with the 'all' profile actually only checks development packages,
-# but that's ok since the 'install' profile checks the install packages.
--c minimum-constraints-install.txt
+# requirements-develop.txt. They are used for example when setting up test
+# environments with the minimum versions of any dependent packages.
 
 
 # Direct dependencies for development and indirect dependencies for development
-# that are needed for some reason (must be consistent with dev-requirements.txt)
+# that are needed for some reason (must be consistent with requirements-develop.txt)
 
 # Ansible validate-modules (no imports, invoked via validate-modules script)
 mock==2.0.0
@@ -92,9 +85,9 @@ sphinxcontrib-websupport==1.2.4
 autodocsumm==0.2.12
 Babel==2.9.1
 
-# sphinx-versions: Retrieved from git repo in dev-requirements.txt
+# sphinx-versions: Retrieved from git repo in requirements-develop.txt
 # ansible-doc-extractor and it's dependent theme for docs generation (no imports, invoked via script):
-# The Makefile defines in doc_build on which Python versions docs are built. Keep in sync with Makefile and dev-requirements.txt.
+# The Makefile defines in doc_build on which Python versions docs are built. Keep in sync with Makefile and requirements-develop.txt.
 
 ansible-doc-extractor==0.1.8
 antsibull-docs-parser==1.0.0
@@ -124,7 +117,7 @@ pip-check-reqs==2.4.3; python_version <= '3.11'
 pip-check-reqs==2.5.3; python_version >= '3.12'
 
 
-# Indirect dependencies for development that are not in dev-requirements.txt
+# Indirect dependencies for development that are not in requirements-develop.txt
 
 alabaster==0.7.9
 attrs==19.2.0

diff --git a/minimum-constraints-install.txt b/minimum-constraints-install.txt
@@ -1,11 +1,12 @@
-# Pip constraints file for install dependencies.
+# Pip constraints file for packages needed for install.
 #
 # This file specifies constraints that match the minimum versions specified in
-# the requirement files for install. They are used for example when setting up
-# test environments with the minimum versions of any dependent packages.
+# requirements.txt and requirements-base.txt. They are used for example when
+# setting up test environments with the minimum versions of any dependent
+# packages.
 
 
-# Base dependencies
+# Base dependencies (must be consistent with requirements-base.txt)
 
 pip==23.3
 setuptools==66.1.0

diff --git a/requirements-base.txt b/requirements-base.txt
@@ -0,0 +1,10 @@
+# Pip requirements file for base packages for a Python env.
+# Base packages are those needed for pip and automatic package version detection.
+
+
+# Base dependencies (must be consistent with minimum-constraints-install.txt)
+
+pip>=23.3
+setuptools>=66.1.0
+setuptools-scm[toml]>=6.4.0
+wheel>=0.38.1
diff --git a/dev-requirements.txt → requirements-develop.txt b/dev-requirements.txt → requirements-develop.txt
@@ -1,12 +1,8 @@
 # Pip requirements file for development dependencies.
 
 
-# Include install dependencies
--r requirements.txt
-
-
 # Direct dependencies for development and indirect dependencies for development
-# that are needed for some reason (must be consistent with minimum-constraints.txt)
+# that are needed for some reason (must be consistent with minimum-constraints-develop.txt)
 
 # Ansible validate-modules (no imports, invoked via validate-modules script)
 mock>=2.0.0
@@ -104,7 +100,7 @@ Babel>=2.9.1
 git+https://github.com/andy-maier/[email protected]#egg=sphinx-versions
 
 # ansible-doc-extractor and it's dependent theme for docs generation (no imports, invoked via script):
-# The Makefile defines in doc_build on which Python versions docs are built. Keep in sync with Makefile and minimum-constraints.txt.
+# The Makefile defines in doc_build on which Python versions docs are built. Keep in sync with Makefile and minimum-constraints-develop.txt.
 ansible-doc-extractor>=0.1.8
 antsibull-docs-parser>=1.0.0