From 07e50510aae8564f96216febff3d3a54bc836b50 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Thu, 30 Jan 2025 13:46:02 -0500 Subject: [PATCH] fix: setup.py update using script (#77) * fix: setup.py update using script * fix: fix issues preventing setup.py updates * no longer need backports.zoneinfo now that the project supports python version >= 3.9 * fix quality problems flagged by pylint --------- Co-authored-by: Justin Hynes --- MANIFEST.in | 1 + requirements/base.in | 1 - requirements/base.txt | 4 +- requirements/ci.txt | 10 ++--- requirements/dev.txt | 22 +++++----- requirements/quality.txt | 12 +++--- requirements/test.txt | 4 +- setup.py | 90 +++++++++++++++++++++++++++++++++++----- 8 files changed, 106 insertions(+), 38 deletions(-) mode change 100755 => 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in old mode 100755 new mode 100644 index 46b73e62..4f302f9c --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include LICENSE include README.md include requirements/base.in include requirements/test.in +include requirements/constraints.txt diff --git a/requirements/base.in b/requirements/base.in index 544d92c3..d37c167f 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -6,4 +6,3 @@ django-model-utils # Provides TimeStampedModel abstract edx-opaque-keys>=0.2.1 # Create and introspect course and xblock identities six # Utilities for supporting Python 2 & 3 in the same codebase setuptools -backports.zoneinfo; python_version<'3.9' diff --git a/requirements/base.txt b/requirements/base.txt index 27e9afc2..7587b05f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,7 +6,7 @@ # asgiref==3.8.1 # via django -django==4.2.17 +django==4.2.18 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -19,7 +19,7 @@ edx-opaque-keys==2.11.0 # via -r requirements/base.in pbr==6.1.0 # via stevedore -pymongo==4.10.1 +pymongo==4.11 # via edx-opaque-keys six==1.17.0 # via -r requirements/base.in diff --git a/requirements/ci.txt b/requirements/ci.txt index 56fef6cb..cf0a9bcc 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,7 +4,7 @@ # # make upgrade # -cachetools==5.5.0 +cachetools==5.5.1 # via tox chardet==5.2.0 # via tox @@ -12,7 +12,7 @@ colorama==0.4.6 # via tox distlib==0.3.9 # via virtualenv -filelock==3.16.1 +filelock==3.17.0 # via # tox # virtualenv @@ -26,9 +26,9 @@ platformdirs==4.3.6 # virtualenv pluggy==1.5.0 # via tox -pyproject-api==1.8.0 +pyproject-api==1.9.0 # via tox -tox==4.23.2 +tox==4.24.1 # via -r requirements/ci.in -virtualenv==20.28.1 +virtualenv==20.29.1 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 60bd3cf8..fe47c2ab 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -17,7 +17,7 @@ build==1.2.2.post1 # via # -r requirements/pip-tools.txt # pip-tools -cachetools==5.5.0 +cachetools==5.5.1 # via # -r requirements/ci.txt # tox @@ -37,7 +37,7 @@ click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint -code-annotations==2.1.0 +code-annotations==2.2.0 # via # -r requirements/quality.txt # edx-lint @@ -57,7 +57,7 @@ distlib==0.3.9 # via # -r requirements/ci.txt # virtualenv -django==4.2.17 +django==4.2.18 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -68,11 +68,11 @@ dnspython==2.7.0 # via # -r requirements/quality.txt # pymongo -edx-lint==5.4.1 +edx-lint==5.6.0 # via -r requirements/quality.txt edx-opaque-keys==2.11.0 # via -r requirements/quality.txt -filelock==3.16.1 +filelock==3.17.0 # via # -r requirements/ci.txt # tox @@ -81,7 +81,7 @@ iniconfig==2.0.0 # via # -r requirements/quality.txt # pytest -isort==5.13.2 +isort==6.0.0 # via # -r requirements/quality.txt # pylint @@ -129,7 +129,7 @@ pycodestyle==2.12.1 # via -r requirements/quality.txt pydocstyle==6.3.0 # via -r requirements/quality.txt -pylint==3.3.3 +pylint==3.3.4 # via # -r requirements/quality.txt # edx-lint @@ -149,11 +149,11 @@ pylint-plugin-utils==0.8.2 # -r requirements/quality.txt # pylint-celery # pylint-django -pymongo==4.10.1 +pymongo==4.11 # via # -r requirements/quality.txt # edx-opaque-keys -pyproject-api==1.8.0 +pyproject-api==1.9.0 # via # -r requirements/ci.txt # tox @@ -204,13 +204,13 @@ tomlkit==0.13.2 # via # -r requirements/quality.txt # pylint -tox==4.23.2 +tox==4.24.1 # via -r requirements/ci.txt typing-extensions==4.12.2 # via # -r requirements/quality.txt # edx-opaque-keys -virtualenv==20.28.1 +virtualenv==20.29.1 # via # -r requirements/ci.txt # tox diff --git a/requirements/quality.txt b/requirements/quality.txt index 3e535a6f..51b1c08a 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -20,7 +20,7 @@ click==8.1.8 # edx-lint click-log==0.4.0 # via edx-lint -code-annotations==2.1.0 +code-annotations==2.2.0 # via # -r requirements/test.txt # edx-lint @@ -30,7 +30,7 @@ coverage[toml]==7.6.10 # pytest-cov dill==0.3.9 # via pylint -django==4.2.17 +django==4.2.18 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -41,7 +41,7 @@ dnspython==2.7.0 # via # -r requirements/test.txt # pymongo -edx-lint==5.4.1 +edx-lint==5.6.0 # via -r requirements/quality.in edx-opaque-keys==2.11.0 # via -r requirements/test.txt @@ -49,7 +49,7 @@ iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -isort==5.13.2 +isort==6.0.0 # via # -r requirements/quality.in # pylint @@ -81,7 +81,7 @@ pycodestyle==2.12.1 # via -r requirements/quality.in pydocstyle==6.3.0 # via -r requirements/quality.in -pylint==3.3.3 +pylint==3.3.4 # via # edx-lint # pylint-celery @@ -95,7 +95,7 @@ pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pymongo==4.10.1 +pymongo==4.11 # via # -r requirements/test.txt # edx-opaque-keys diff --git a/requirements/test.txt b/requirements/test.txt index 30a4482b..2c269cf0 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -10,7 +10,7 @@ asgiref==3.8.1 # django click==8.1.8 # via code-annotations -code-annotations==2.1.0 +code-annotations==2.2.0 # via -r requirements/test.in coverage[toml]==7.6.10 # via @@ -42,7 +42,7 @@ pbr==6.1.0 # stevedore pluggy==1.5.0 # via pytest -pymongo==4.10.1 +pymongo==4.11 # via # -r requirements/base.txt # edx-opaque-keys diff --git a/setup.py b/setup.py index d6760989..59d6a373 100644 --- a/setup.py +++ b/setup.py @@ -29,17 +29,82 @@ def load_requirements(*requirements_paths): """ Load all requirements from the specified requirements files. - Returns: - list: Requirements file relative path strings + Requirements will include any constraints from files specified + with -c in the requirements files. + Returns a list of requirement strings. """ - requirements = set() + # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why. + + # e.g. {"django": "Django", "confluent-kafka": "confluent_kafka[avro]"} + by_canonical_name = {} + + def check_name_consistent(package): + """ + Raise exception if package is named different ways. + + This ensures that packages are named consistently so we can match + constraints to packages. It also ensures that if we require a package + with extras we don't constrain it without mentioning the extras (since + that too would interfere with matching constraints.) + """ + canonical = package.lower().replace('_', '-').split('[')[0] + seen_spelling = by_canonical_name.get(canonical) + if seen_spelling is None: + by_canonical_name[canonical] = package + elif seen_spelling != package: + raise Exception( + f'Encountered both "{seen_spelling}" and "{package}" in requirements ' + 'and constraints files; please use just one or the other.' + ) + + requirements = {} + constraint_files = set() + + # groups "pkg<=x.y.z,..." into ("pkg", "<=x.y.z,...") + re_package_name_base_chars = r"a-zA-Z0-9\-_." # chars allowed in base package name + # Two groups: name[maybe,extras], and optionally a constraint + requirement_line_regex = re.compile( + r"([%s]+(?:\[[%s,\s]+\])?)([<>=][^#\s]+)?" # pylint: disable=consider-using-f-string + % (re_package_name_base_chars, re_package_name_base_chars) + ) + + def add_version_constraint_or_raise(current_line, current_requirements, add_if_not_present): + regex_match = requirement_line_regex.match(current_line) + if regex_match: + package = regex_match.group(1) + version_constraints = regex_match.group(2) + check_name_consistent(package) + existing_version_constraints = current_requirements.get(package, None) + # It's fine to add constraints to an unconstrained package, + # but raise an error if there are already constraints in place. + if existing_version_constraints and existing_version_constraints != version_constraints: + raise BaseException(f'Multiple constraint definitions found for {package}:' + f' "{existing_version_constraints}" and "{version_constraints}".' + f'Combine constraints into one location with {package}' + f'{existing_version_constraints},{version_constraints}.') + if add_if_not_present or package in current_requirements: + current_requirements[package] = version_constraints + + # Read requirements from .in files and store the path to any + # constraint files that are pulled in. for path in requirements_paths: - requirements.update( - line.split('#')[0].strip() - for line in open(path, encoding='utf-8').readlines() # pylint: disable=consider-using-with - if is_requirement(line.strip()) - ) - return list(requirements) + with open(path, encoding='utf-8') as reqs: + for line in reqs: + if is_requirement(line): + add_version_constraint_or_raise(line, requirements, True) + if line and line.startswith('-c') and not line.startswith('-c http'): + constraint_files.add(os.path.dirname(path) + '/' + line.split('#')[0].replace('-c', '').strip()) + + # process constraint files: add constraints to existing requirements + for constraint_file in constraint_files: + with open(constraint_file, encoding='utf-8') as reader: + for line in reader: + if is_requirement(line): + add_version_constraint_or_raise(line, requirements, False) + + # process back into list of pkg><=constraints strings + constrained_requirements = [f'{pkg}{version or ""}' for (pkg, version) in sorted(requirements.items())] + return constrained_requirements def is_requirement(line): @@ -47,9 +112,12 @@ def is_requirement(line): Return True if the requirement line is a package requirement. Returns: - bool: True if the line is not blank, a comment, a URL, or an included file + bool: True if the line is not blank, a comment, + a URL, or an included file """ - return line and not line.startswith(('-r', '#', '-e', 'git+', '-c')) + # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why + + return line and line.strip() and not line.startswith(('-r', '#', '-e', 'git+', '-c')) if sys.argv[-1] == 'tag':