Skip to content

Commit

Permalink
ci: add hadolint and flake8 linters
Browse files Browse the repository at this point in the history
  • Loading branch information
audrium committed Nov 17, 2020
1 parent 98c5b8e commit c994763
Show file tree
Hide file tree
Showing 10 changed files with 77 additions and 20 deletions.
19 changes: 19 additions & 0 deletions .flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
[flake8]
max-line-length = 89

exclude =
build
dist
docs
coverage.xml
reana_job_controller.egg-info
.*/
env/
.git
__pycache__

ignore = E203, E231, E266, E501, W503, F403, F401

max-complexity = 18

select = B,C,E,F,W,T4,B9
20 changes: 20 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,23 @@ jobs:
pip install black==19.10b0
./run-tests.sh --check-black
lint-flake8:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2

- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: 3.8

- name: Check compliance with pep8, pyflakes and circular complexity
run: |
pip install --upgrade pip
pip install flake8
./run-tests.sh --check-flake8
lint-pydocstyle:
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -142,5 +159,8 @@ jobs:
- name: Checkout
uses: actions/checkout@v2

- name: Lint Dockerfile
run: docker run -i --rm hadolint/hadolint < Dockerfile

- name: Build docker image
run: ./run-tests.sh --check-docker-build
34 changes: 21 additions & 13 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,39 +6,46 @@

# Install base image and its dependencies
FROM python:3.8-slim
# hadolint ignore=DL3008, DL3009, DL3013
RUN apt-get update && \
apt-get install -y vim-tiny && \
apt-get install -y --no-install-recommends vim-tiny && \
pip install --upgrade pip

# Install Kerberos dependencies
# hadolint ignore=DL3008
RUN export DEBIAN_FRONTEND=noninteractive ;\
apt-get -yq install krb5-user \
apt-get -yq install --no-install-recommends \
krb5-user \
krb5-config \
libkrb5-dev \
libauthen-krb5-perl \
gcc;
ADD etc/krb5.conf /etc/krb5.conf
COPY etc/krb5.conf /etc/krb5.conf

# Default compute backend is Kubernetes
ARG COMPUTE_BACKENDS=kubernetes

# CERN HTCondor part taken from https://gitlab.cern.ch/batch-team/condorsubmit
# hadolint ignore=DL3008, DL4006
RUN if echo "$COMPUTE_BACKENDS" | grep -q "htcondorcern"; then \
set -e;\
export DEBIAN_FRONTEND=noninteractive ;\
apt-get -yq install wget alien gnupg2 ;\
apt-get -yq install --no-install-recommends wget alien gnupg2 ;\
wget -O ngbauth-submit.rpm http://linuxsoft.cern.ch/internal/repos/batch7-stable/x86_64/os/Packages/ngbauth-submit-0.23-1.el7.noarch.rpm; \
wget -O myschedd.rpm http://linuxsoft.cern.ch/internal/repos/batch7-stable/x86_64/os/Packages/myschedd-1.5-1.el7.x86_64.rpm; \
yes | alien -i myschedd.rpm; \
yes | alien -i ngbauth-submit.rpm; \
wget -qO - http://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | apt-key add -; \
echo "deb https://research.cs.wisc.edu/htcondor/debian/8.9/buster buster contrib" >>/etc/apt/sources.list; \
apt-get update; \
apt-get install -y condor --no-install-recommends; \
apt-get -y remove gnupg2 wget alien; \
apt-get install -y --no-install-recommends condor; \
apt-get -y remove gnupg2 wget alien && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*; \
fi

# CERN Slurm backend requires SSH to headnode
# hadolint ignore=DL3008, DL4006
RUN if echo "$COMPUTE_BACKENDS" | grep -q "slurmcern"; then \
export DEBIAN_FRONTEND=noninteractive ;\
apt-get -yq install openssh-client \
Expand All @@ -47,13 +54,13 @@ RUN if echo "$COMPUTE_BACKENDS" | grep -q "slurmcern"; then \

# Add HTCondor related files
RUN mkdir -p /etc/myschedd
ADD etc/myschedd.yaml /etc/myschedd/
ADD etc/10_cernsubmit.config /etc/condor/config.d/
ADD etc/ngbauth-submit /etc/sysconfig/
ADD etc/ngauth_batch_crypt_pub.pem /etc/
ADD etc/cerngridca.crt /usr/local/share/ca-certificates/cerngridca.crt
ADD etc/cernroot.crt /usr/local/share/ca-certificates/cernroot.crt
ADD etc/job_wrapper.sh etc/job_wrapper.sh
COPY etc/myschedd.yaml /etc/myschedd/
COPY etc/10_cernsubmit.config /etc/condor/config.d/
COPY etc/ngbauth-submit /etc/sysconfig/
COPY etc/ngauth_batch_crypt_pub.pem /etc/
COPY etc/cerngridca.crt /usr/local/share/ca-certificates/cerngridca.crt
COPY etc/cernroot.crt /usr/local/share/ca-certificates/cernroot.crt
COPY etc/job_wrapper.sh etc/job_wrapper.sh
RUN chmod +x /etc/job_wrapper.sh
RUN update-ca-certificates

Expand All @@ -70,6 +77,7 @@ ARG DEBUG=0
RUN if [ "${DEBUG}" -gt 0 ]; then pip install -e ".[debug]"; else pip install .; fi;

# Are we building with locally-checked-out shared modules?
# hadolint ignore=SC2102
RUN if test -e modules/reana-commons; then pip install -e modules/reana-commons[kubernetes] --upgrade; fi
RUN if test -e modules/reana-db; then pip install -e modules/reana-db --upgrade; fi

Expand Down
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ include *.rst
include *.sh
include *.txt
include *.in
include .flake8
include pytest.ini
include docs/openapi.json
include docs/_static/reana-job-manager.xml
Expand Down
2 changes: 1 addition & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
REANA-Job-Controller
####################

. image:: https://github.com/reanahub/reana-job-controller/workflows/CI/badge.svg
.. image:: https://github.com/reanahub/reana-job-controller/workflows/CI/badge.svg
:target: https://github.com/reanahub/reana-job-controller/actions

.. image:: https://readthedocs.org/projects/reana-job-controller/badge/?version=latest
Expand Down
7 changes: 5 additions & 2 deletions reana_job_controller/htcondorcern_job_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,9 @@ def _get_schedd():
"""Find and return the HTCondor schedd."""
schedd = getattr(thread_local, "MONITOR_THREAD_SCHEDD", None)
if schedd is None:
setattr(thread_local, "MONITOR_THREAD_SCHEDD", htcondor.Schedd())
setattr(
thread_local, "MONITOR_THREAD_SCHEDD", htcondor.Schedd() # noqa: F821
)
logging.info("Getting schedd: {}".format(thread_local.MONITOR_THREAD_SCHEDD))
return thread_local.MONITOR_THREAD_SCHEDD

Expand All @@ -279,7 +281,8 @@ def stop(backend_job_id):
try:
schedd = HTCondorJobManagerCERN._get_schedd()
schedd.act(
htcondor.JobAction.Remove, "ClusterId=={}".format(backend_job_id)
htcondor.JobAction.Remove, # noqa: F821
"ClusterId=={}".format(backend_job_id),
)
except Exception as e:
logging.error(e, exc_info=True)
Expand Down
2 changes: 1 addition & 1 deletion reana_job_controller/kubernetes_job_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ def execute(self):
def _submit(self):
"""Submit job and return its backend id."""
try:
api_response = current_k8s_batchv1_api_client.create_namespaced_job(
current_k8s_batchv1_api_client.create_namespaced_job(
namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE, body=self.job
)
return self.job["metadata"]["name"]
Expand Down
6 changes: 6 additions & 0 deletions run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ check_black () {
black --check .
}

check_flake8 () {
flake8 .
}

check_openapi_spec () {
FLASK_APP=reana_job_controller/app.py flask openapi create openapi.json
diff -q openapi.json docs/openapi.json
Expand Down Expand Up @@ -97,6 +101,7 @@ check_all () {
check_script
check_pydocstyle
check_black
check_flake8
check_openapi_spec
check_manifest
check_sphinx
Expand Down Expand Up @@ -141,6 +146,7 @@ do
--check-shellscript) check_script;;
--check-pydocstyle) check_pydocstyle;;
--check-black) check_black;;
--check-flake8) check_flake8;;
--check-openapi-spec) check_openapi_spec;;
--check-manifest) check_manifest;;
--check-sphinx) check_sphinx;;
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@

# Get the version string. Cannot be done with import!
with open(os.path.join("reana_job_controller", "version.py"), "rt") as f:
version = re.search('__version__\s*=\s*"(?P<version>.*)"\n', f.read()).group(
version = re.search(r'__version__\s*=\s*"(?P<version>.*)"\n', f.read()).group(
"version"
)

Expand Down
4 changes: 2 additions & 2 deletions tests/test_job_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def test_kubernetes_get_job_status(

def test_kubernetes_clean_job(app, mocked_job_managers):
"""Test clean jobs in the Kubernetes compute backend."""
with mock.patch("reana_job_controller.job_monitor." "threading") as threading:
with mock.patch("reana_job_controller.job_monitor." "threading"):
job_monitor_k8s = JobMonitorKubernetes(app=app)
job_id = str(uuid.uuid4())
job_metadata = {
Expand Down Expand Up @@ -114,7 +114,7 @@ def test_kubernetes_should_process_job(
app, compute_backend, deleted, should_process, kubernetes_job_pod
):
"""Test should process job."""
with mock.patch("reana_job_controller.job_monitor." "threading") as threading:
with mock.patch("reana_job_controller.job_monitor." "threading"):
job_monitor_k8s = JobMonitorKubernetes(app=app)
job_id = str(uuid.uuid4())
backend_job_id = str(uuid.uuid4())
Expand Down

0 comments on commit c994763

Please sign in to comment.