fix: Unify python dependency installation and update to vulnerability…

… free versions (#222) * Dependency cleanup and Postgres conn modifications * Removed changes done for transaction error * Including requirements.in into tests/requirements.in Co-authored-by: Arunprasad Rajkumar <[email protected]> * Including main requirements.in into tests/requirements.in Co-authored-by: Arunprasad Rajkumar <[email protected]>
fabric8-analytics · Nov 30, 2020 · 76b836f · 76b836f
1 parent b15d1dd
commit 76b836f
Show file tree

Hide file tree

Showing 8 changed files with 221 additions and 87 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,17 +1,11 @@
 FROM registry.centos.org/centos/centos:7
 
-ENV F8A_WORKER_VERSION=6503230
-ENV F8A_AUTH_VERSION=fff8f49
-
 RUN yum install -y epel-release &&\
     yum install -y gcc git python36-pip python36-requests httpd httpd-devel python36-devel &&\
     yum clean all
 
 RUN python3 -m pip install --upgrade pip>=10.0.0
 
-RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@${F8A_WORKER_VERSION}
-RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-auth.git@${F8A_AUTH_VERSION}
-
 COPY ./requirements.txt /
 RUN pip3 install -r requirements.txt && rm requirements.txt
 

diff --git a/Dockerfile.rhel b/Dockerfile.rhel
@@ -2,19 +2,13 @@
 
 FROM quay.io/openshiftio/rhel-base-python3:latest
 
-ENV F8A_WORKER_VERSION=6503230
-ENV F8A_AUTH_VERSION=fff8f49
-
 COPY ./requirements.txt /
 
 RUN python3 -m pip install --upgrade pip>=10.0.0 &&\
     pip3 install -r requirements.txt && rm requirements.txt
 
 COPY ./src /src
 
-RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@${F8A_WORKER_VERSION}
-RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-auth.git@${F8A_AUTH_VERSION}
-
 ADD scripts/entrypoint.sh /bin/entrypoint.sh
 
 RUN chmod 777 /bin/entrypoint.sh

diff --git a/qa/runtests.sh b/qa/runtests.sh
@@ -66,9 +66,8 @@ prepare_venv
 
 # now we are surely in the Python virtual environment
 
-pip3 install -r requirements.txt
-pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@6503230
-pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-auth.git@fff8f49
+pip3 install -r tests/requirements.txt
+
 
 export DEPLOYMENT_PREFIX="${USER}"
 export WORKER_ADMINISTRATION_REGION=api

diff --git a/requirements.in b/requirements.in
@@ -1,16 +1,8 @@
-pytest
-flask
+six
+Flask
 gunicorn
-flask-cors
-requests
-pytest-cov
-coverage
-gevent
-sqlalchemy
-pytest-mock
-codecov
-radon
-boto3
-sentry-sdk
-requests-futures
-psycopg2
+Flask_Cors
+sentry_sdk
+psycopg2-binary
+f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker
+fabric8a_auth @ git+https://github.com/fabric8-analytics/fabric8-analytics-auth.git@5ff9438#egg=fabric8a_auth
diff --git a/requirements.txt b/requirements.txt
@@ -2,49 +2,91 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=requirements.txt requirements.in
+#    pip-compile
 #
-atomicwrites==1.2.1       # via pytest
-attrs==18.2.0             # via pytest
-boto3==1.9.99
-botocore==1.12.99         # via boto3, s3transfer
-certifi==2018.8.24        # via requests, sentry-sdk
+amqp==5.0.2               # via kombu
+anymarkup-core==0.8.1     # via anymarkup
+anymarkup==0.8.1          # via f8a-worker
+attrs==20.3.0             # via jsonschema
+babel==2.9.0              # via flask-babelex
+beautifulsoup4==4.9.3     # via bs4, f8a-worker
+billiard==3.6.3.0         # via celery
+blinker==1.4              # via flask-mail, flask-principal
+boto3==1.16.25            # via f8a-worker
+botocore==1.19.25         # via boto3, f8a-worker, s3transfer
+bs4==0.0.1                # via f8a-utils
+celery==5.0.2             # via selinon
+certifi==2020.11.8        # via requests, sentry-sdk
+cffi==1.14.4              # via cryptography
 chardet==3.0.4            # via requests
-click==7.0                # via flask
-codecov==2.0.15
-colorama==0.4.1           # via radon
-coverage==4.5.1
-docutils==0.14            # via botocore
-flake8-polyfill==1.0.2    # via radon
-flake8==3.5.0             # via flake8-polyfill
-flask-cors==3.0.6
-flask==1.0.2
-gevent==1.3.6
-greenlet==0.4.15          # via gevent
-gunicorn==19.9.0
-idna==2.7                 # via requests
-itsdangerous==0.24        # via flask
-jinja2==2.10              # via flask
-jmespath==0.9.3           # via boto3, botocore
-mando==0.6.4              # via radon
-markupsafe==1.1.1           # via jinja2
-mccabe==0.6.1             # via flake8
-more-itertools==4.3.0     # via pytest
-pluggy==0.7.1             # via pytest
-psycopg2==2.7.7
-py==1.6.0                 # via pytest
-pycodestyle==2.3.1        # via flake8
-pyflakes==1.6.0           # via flake8
-pytest-cov==2.6.0
-pytest-mock==1.10.0
-pytest==3.8.1
-python-dateutil==2.8.0    # via botocore
-radon==3.0.1
-requests-futures==1.0.0
-requests==2.19.1
-s3transfer==0.2.0         # via boto3
-sentry-sdk==0.7.2
-six==1.11.0               # via flask-cors, mando, more-itertools, pytest, python-dateutil
-sqlalchemy==1.2.12
-urllib3==1.23             # via botocore, requests, sentry-sdk
-werkzeug==0.15.3          # via flask
+click-didyoumean==0.0.3   # via celery
+click-repl==0.1.6         # via celery
+click==7.1.2              # via anymarkup, celery, click-didyoumean, click-repl, flask, selinon
+codegen==1.0              # via selinon
+colorama==0.4.4           # via rainbow-logging-handler
+configobj==5.0.6          # via anymarkup
+cryptography==3.2.1       # via f8a-utils, fabric8a-auth
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils  # via f8a-worker
+git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker  # via -r requirements.in
+git+https://github.com/fabric8-analytics/fabric8-analytics-auth.git@5ff9438#egg=fabric8a_auth  # via -r requirements.in
+flask-babelex==0.9.4      # via flask-security
+flask-cors==3.0.9         # via -r requirements.in
+flask-login==0.5.0        # via flask-security
+flask-mail==0.9.1         # via flask-security
+flask-principal==0.4.0    # via flask-security
+flask-security==3.0.0     # via fabric8a-auth
+flask-wtf==0.14.3         # via flask-security
+flask==1.1.2              # via -r requirements.in, fabric8a-auth, flask-babelex, flask-cors, flask-login, flask-mail, flask-principal, flask-security, flask-wtf
+git2json==0.2.3           # via f8a-worker
+gitdb==4.0.5              # via gitpython
+gitpython==3.1.11         # via f8a-worker
+graphviz==0.15            # via selinon
+gunicorn==20.0.4          # via -r requirements.in
+idna==2.10                # via requests
+importlib-metadata==3.1.0  # via jsonschema, kombu
+itsdangerous==1.1.0       # via flask, flask-security, flask-wtf
+jinja2==2.11.2            # via flask, flask-babelex
+jmespath==0.10.0          # via boto3, botocore
+jsl==0.2.4                # via f8a-worker
+json5==0.9.5              # via anymarkup
+jsonschema==3.2.0         # via f8a-worker, selinon
+kombu==5.0.2              # via celery
+logutils==0.3.5           # via rainbow-logging-handler
+lxml==4.6.2               # via f8a-utils, f8a-worker
+markupsafe==1.1.1         # via jinja2, wtforms
+passlib==1.7.4            # via flask-security
+prompt-toolkit==3.0.8     # via click-repl
+psycopg2-binary==2.8.6    # via -r requirements.in
+pycparser==2.20           # via cffi
+pyjwt==1.7.1              # via fabric8a-auth
+pyrsistent==0.17.3        # via jsonschema
+python-dateutil==2.8.1    # via botocore
+pytz==2020.4              # via babel, celery
+pyyaml==5.3.1             # via anymarkup, f8a-worker, selinon
+rainbow-logging-handler==2.2.2  # via selinon
+raven==6.10.0             # via f8a-worker
+requests-futures==1.0.0   # via f8a-worker
+requests==2.25.0          # via f8a-utils, f8a-worker, fabric8a-auth, requests-futures
+s3transfer==0.3.3         # via boto3
+selinon[celery]==1.0.0    # via f8a-worker
+semantic-version==2.8.5   # via f8a-worker
+semver==2.13.0            # via f8a-utils
+sentry-sdk==0.19.4        # via -r requirements.in
+six==1.15.0               # via -r requirements.in, anymarkup-core, click-repl, configobj, cryptography, flask-cors, jsonschema, python-dateutil, tenacity
+smmap==3.0.4              # via gitdb
+soupsieve==2.0.1          # via beautifulsoup4
+speaklater==1.3           # via flask-babelex
+sqlalchemy==1.3.20        # via f8a-worker
+tenacity==6.2.0           # via f8a-utils, f8a-worker
+toml==0.9.4               # via anymarkup, f8a-worker
+urllib3==1.26.2           # via botocore, requests, sentry-sdk
+vine==5.0.0               # via amqp, celery
+wcwidth==0.2.5            # via prompt-toolkit
+werkzeug==1.0.1           # via f8a-worker, flask
+wtforms==2.3.3            # via flask-wtf
+xmltodict==0.12.0         # via anymarkup
+zipp==3.4.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/setup.py b/setup.py
@@ -6,29 +6,27 @@
 
 
 def get_requirements():
-    """Parse all packages mentioned in the 'requirements.txt' file."""
-    with open('requirements.txt') as fd:
-        lines = fd.read().splitlines()
-        reqs, dep_links = [], []
-        for line in lines:
-            if line.startswith('git+'):
-                dep_links.append(line)
-            else:
-                reqs.append(line)
-        return reqs, dep_links
+    """
+    Parse dependencies from 'requirements.in' file.
+
+    Collecting dependencies from 'requirements.in' as a list,
+    this list will be used by 'install_requires' to specify minimal dependencies
+    needed to run the application.
+    """
+    with open('requirements.in') as fd:
+        return fd.read().splitlines()
 
 
 # pip doesn't install from dependency links by default, so one should install dependencies by
 #  `pip install -r requirements.txt`, not by `pip install .`
 #  See https://github.com/pypa/pip/issues/2023
-reqs, dep_links = get_requirements()
+install_requires = get_requirements()
 
 setup(
     name='gemini-server',
     version='0.1',
     packages=find_packages(exclude=['tests', 'tests.*']),
-    install_requires=reqs,
-    dependency_links=dep_links,
+    install_requires=install_requires,
     include_package_data=True,
     author='Samuzzal Choudhury',
     author_email='[email protected]',

diff --git a/tests/requirements.in b/tests/requirements.in
@@ -0,0 +1,8 @@
+pytest
+pytest-cov
+coverage
+pytest-mock
+codecov
+radon
+pydocstyle
+-r ../requirements.in
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -0,0 +1,107 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile
+#
+amqp==5.0.2               # via kombu
+anymarkup-core==0.8.1     # via anymarkup
+anymarkup==0.8.1          # via f8a-worker
+attrs==20.3.0             # via jsonschema, pytest
+babel==2.9.0              # via flask-babelex
+beautifulsoup4==4.9.3     # via bs4, f8a-worker
+billiard==3.6.3.0         # via celery
+blinker==1.4              # via flask-mail, flask-principal
+boto3==1.16.25            # via f8a-worker
+botocore==1.19.25         # via boto3, f8a-worker, s3transfer
+bs4==0.0.1                # via f8a-utils
+celery==5.0.2             # via selinon
+certifi==2020.11.8        # via requests, sentry-sdk
+cffi==1.14.4              # via cryptography
+chardet==3.0.4            # via requests
+click-didyoumean==0.0.3   # via celery
+click-repl==0.1.6         # via celery
+click==7.1.2              # via anymarkup, celery, click-didyoumean, click-repl, flask, selinon
+codecov==2.1.10           # via -r requirements.in
+codegen==1.0              # via selinon
+colorama==0.4.4           # via radon, rainbow-logging-handler
+configobj==5.0.6          # via anymarkup
+coverage==5.3             # via -r requirements.in, codecov, pytest-cov
+cryptography==3.2.1       # via f8a-utils, fabric8a-auth
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils  # via f8a-worker
+git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker  # via -r ../requirements.in
+git+https://github.com/fabric8-analytics/fabric8-analytics-auth.git@5ff9438#egg=fabric8a_auth  # via -r ../requirements.in
+flask-babelex==0.9.4      # via flask-security
+flask-cors==3.0.9         # via -r ../requirements.in
+flask-login==0.5.0        # via flask-security
+flask-mail==0.9.1         # via flask-security
+flask-principal==0.4.0    # via flask-security
+flask-security==3.0.0     # via fabric8a-auth
+flask-wtf==0.14.3         # via flask-security
+flask==1.1.2              # via -r ../requirements.in, fabric8a-auth, flask-babelex, flask-cors, flask-login, flask-mail, flask-principal, flask-security, flask-wtf
+future==0.18.2            # via radon
+git2json==0.2.3           # via f8a-worker
+gitdb==4.0.5              # via gitpython
+gitpython==3.1.11         # via f8a-worker
+graphviz==0.15            # via selinon
+gunicorn==20.0.4          # via -r ../requirements.in
+idna==2.10                # via requests
+importlib-metadata==3.1.0  # via jsonschema, kombu, pluggy, pytest
+iniconfig==1.1.1          # via pytest
+itsdangerous==1.1.0       # via flask, flask-security, flask-wtf
+jinja2==2.11.2            # via flask, flask-babelex
+jmespath==0.10.0          # via boto3, botocore
+jsl==0.2.4                # via f8a-worker
+json5==0.9.5              # via anymarkup
+jsonschema==3.2.0         # via f8a-worker, selinon
+kombu==5.0.2              # via celery
+logutils==0.3.5           # via rainbow-logging-handler
+lxml==4.6.2               # via f8a-utils, f8a-worker
+mando==0.6.4              # via radon
+markupsafe==1.1.1         # via jinja2, wtforms
+packaging==20.4           # via pytest
+passlib==1.7.4            # via flask-security
+pluggy==0.13.1            # via pytest
+prompt-toolkit==3.0.8     # via click-repl
+psycopg2-binary==2.8.6    # via -r ../requirements.in
+py==1.9.0                 # via pytest
+pycparser==2.20           # via cffi
+pydocstyle==5.1.1         # via -r requirements.in
+pyjwt==1.7.1              # via fabric8a-auth
+pyparsing==2.4.7          # via packaging
+pyrsistent==0.17.3        # via jsonschema
+pytest-cov==2.10.1        # via -r requirements.in
+pytest-mock==3.3.1        # via -r requirements.in
+pytest==6.1.2             # via -r requirements.in, pytest-cov, pytest-mock
+python-dateutil==2.8.1    # via botocore
+pytz==2020.4              # via babel, celery
+pyyaml==5.3.1             # via anymarkup, f8a-worker, selinon
+radon==4.3.2              # via -r requirements.in
+rainbow-logging-handler==2.2.2  # via selinon
+raven==6.10.0             # via f8a-worker
+requests-futures==1.0.0   # via f8a-worker
+requests==2.25.0          # via codecov, f8a-utils, f8a-worker, fabric8a-auth, requests-futures
+s3transfer==0.3.3         # via boto3
+selinon[celery]==1.0.0    # via f8a-worker
+semantic-version==2.8.5   # via f8a-worker
+semver==2.13.0            # via f8a-utils
+sentry-sdk==0.19.4        # via -r ../requirements.in
+six==1.15.0               # via -r ../requirements.in, anymarkup-core, click-repl, configobj, cryptography, flask-cors, jsonschema, mando, packaging, python-dateutil, tenacity
+smmap==3.0.4              # via gitdb
+snowballstemmer==2.0.0    # via pydocstyle
+soupsieve==2.0.1          # via beautifulsoup4
+speaklater==1.3           # via flask-babelex
+sqlalchemy==1.3.20        # via f8a-worker
+tenacity==6.2.0           # via f8a-utils, f8a-worker
+toml==0.9.4               # via anymarkup, f8a-worker, pytest
+urllib3==1.26.2           # via botocore, requests, sentry-sdk
+vine==5.0.0               # via amqp, celery
+wcwidth==0.2.5            # via prompt-toolkit
+werkzeug==1.0.1           # via f8a-worker, flask
+wtforms==2.3.3            # via flask-wtf
+xmltodict==0.12.0         # via anymarkup
+zipp==3.4.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools