diff --git a/.env.example b/.env.example deleted file mode 100644 index be6a950..0000000 --- a/.env.example +++ /dev/null @@ -1,25 +0,0 @@ -PYTHONUNBUFFERED=1 -DEBUG=true -DATABASE_URL=postgresql://dcoleman:postgres@postgres/dcoleman_dev -POSTGRES_PASSWORD=postgres -LANGUAGE_CODE=en -PORT=8000 -REST_ENABLED=true -SECRET_KEY=f*)$)fay97180m+ti%xi8si##u__h(8%(ipr1z-*lsjbucooz& -LOG_LEVEL=INFO -LOG_LEVEL_DJANGO=INFO -LOG_LEVEL_DJANGO_DB=INFO -LOG_LEVEL_DJANGO_REQ=INFO -EMAIL_HOST_USER=YOUREMAIL@gmail.com -EMAIL_HOST_PASSWORD=PASS -EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend -TASKS_SEND_EMAILS_TO_ASSIGNED=true -TASKS_SEND_EMAILS_TO_PARTNERS=true -TASKS_VIEWER_ENABLED=true -AUTH_PASSWORD_VALIDATORS_ENABLED=false -APP_EMAIL=no-reply@localhost -APP_NAME=Django Coleman -SITE_HEADER=Django Coleman - A Simple Task Manager -ADMIN_USERNAME=admin -ADMIN_PASSWORD=admin1234 -DCOLEMAN_ENDPOINT=http://django-coleman:8000/api/v1 diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 8b8e3db..0000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Docker Image CI - -on: [push, pull_request] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Build the Docker image - run: ./docker-build.sh - - name: Run tests - run: docker run --rm -e PROCESS_TYPE=test --name django-coleman mrsarm/django-coleman diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..3e7439a --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,84 @@ +name: Docker Image CI + +on: [push] + +jobs: + + build-test-release: + name: Build, Test and Release + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: ${{ env.DOCKERHUB_TOKEN }} + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Install pose + run: | + wget https://github.com/mrsarm/pose/releases/download/0.4.0/pose-0.4.0-x86_64-unknown-linux-gnu.tar.gz -O - \ + | tar -xz + + - name: Define $TAG variable + run: echo "TAG=$(./pose slug $GITHUB_REF_NAME)" >> "$GITHUB_ENV" + - name: Print tag and image names + run: | + echo "- TAG --> $TAG" + echo "- IMAGE --> mrsarm/django-coleman:$TAG" + + - name: Build the Docker image + run: ./docker-build.sh $TAG + + - name: Run tests + run: docker run --rm -e PROCESS_TYPE=test --name django-coleman "mrsarm/django-coleman:$TAG" + + - name: Release Docker image + if: ${{ env.DOCKERHUB_TOKEN }} + run: docker push "mrsarm/django-coleman:$TAG" + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Get compose.yaml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ./pose get -H "Authorization: token $GITHUB_TOKEN" \ + "https://raw.githubusercontent.com/mrsarm/dcoleman-e2e/$TAG/compose.yaml" "$TAG:main" + - name: Get .env.example + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ./pose get -H "Authorization: token $GITHUB_TOKEN" \ + "https://raw.githubusercontent.com/mrsarm/dcoleman-e2e/$TAG/.env.example" "$TAG:main" + + - name: Setup environment variables + run: cp .env.example .env + + - name: Build compose file for CI + run: | + ./pose --no-docker config --tag $TAG --tag-filter regex=mrsarm/ --progress -o ci.yaml + + - name: Pull images + run: docker compose -f ci.yaml pull + && docker compose -f ci.yaml pull dcoleman-e2e # services with profiles are not pulled by default + + - name: Run e2e tests + run: docker compose -f ci.yaml run dcoleman-e2e + + - name: Tag "latest" + if: ${{ env.DOCKERHUB_TOKEN && github.ref == 'refs/heads/master' }} + run: docker tag "mrsarm/django-coleman:$TAG" mrsarm/django-coleman:latest + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Release "latest" + if: ${{ env.DOCKERHUB_TOKEN && github.ref == 'refs/heads/master' }} + run: docker push mrsarm/django-coleman:latest + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 8ceea19..e3c8b5d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ venv*/ /htmlcov /.coverage + +ci*.yaml diff --git a/Dockerfile b/Dockerfile index 7be9b26..9df4fdd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.11-slim-bookworm LABEL maintainer="Mariano Ruiz " ENV CXXFLAGS="-mtune=intel -Os -pipe" \ @@ -76,4 +76,7 @@ RUN honcho start collectstatic compilemessages \ USER worker +HEALTHCHECK --interval=20s --timeout=3s \ + CMD curl -f http://localhost:8000/health/?format=json || exit 1 + CMD ["sh", "-c", "exec honcho start --no-prefix $PROCESS_TYPE"] diff --git a/Procfile b/Procfile index f24bbd1..d626fed 100644 --- a/Procfile +++ b/Procfile @@ -6,5 +6,5 @@ compilemessages: ./manage.py compilemessages --ignore 'venv*' --ignore '.venv*' migrate: ./manage.py showmigrations && ./manage.py migrate makemigrations: ./manage.py makemigrations && ./manage.py makemigrations partner mtasks createadmin: ./manage.py shell -c "from django.contrib.auth.models import User; User.objects.create_superuser('$ADMIN_USERNAME', password='$ADMIN_PASSWORD')" && printf "User \"$ADMIN_USERNAME\" created.\n---> DON'T forget to CHANGE the password <---\n" -provision: honcho start createdb && honcho start migrate && honcho start createadmin +provision: honcho start createdb && honcho start migrate && honcho start createadmin || true test: pytest --cov --cov-report=html --cov-report=term-missing --no-cov-on-fail --color=yes diff --git a/README.rst b/README.rst index a964fa8..6a0ae01 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,8 @@ Features partner (customer, provider...), description, responsible of the task, priority... * Each task may have items: sub-tasks to be done. * The built-in Django *Authentication and Authorization* system - to manage users and groups, login, etc. + to manage users and groups, login, etc, and optionally SSO with Google + within the Admin (`django-google-sso `_). * Module `django-adminfilters `_ that allows multiselection searches. * Send emails when a task is created. @@ -28,6 +29,11 @@ Features * Pytest with some tests as example and code coverage reports configured. * Docker and Docker Compose configurations (images published in `Docker Hub `_). +* CI environment, and E2E tests written with Playwright: + `dcoleman-e2e `_. CI is executed with + GitHub Actions, and executed on each push in this project, + the viewer repo, or the E2E repo itself. The task also releases the image + in the Docker Registry. * Ready to use "production" configurations as reference. .. image:: docs/source/_static/img/django-coleman.png @@ -39,7 +45,7 @@ Requirements Docker, or: -* Python 3.8+ (tested with Python 3.8 and 3.11). +* Python 3.10+ (tested with 3.11). * Django 4.2 LTS and other dependencies declared in the ``requirements.txt`` file (use virtual environments or containers!). * A Django compatible database like PostgreSQL (by default uses @@ -117,21 +123,24 @@ Docker A reference ``_ is provided, and the image published in `Docker Hub `_. -Also ``_ and `<.env.example>`_ files are provided, you can run -all from here, Django Coleman, the viewer app and Postgres. +Also ``compose.yaml`` and ``.env.example`` files are provided in the +`dcoleman-e2e `_ project, you +can run all from there, Django Coleman, the +`viewer `_ app +and Postgres, and the E2E tests. -First, copy the ``.env.example`` file as ``.env`` file, and edit whatever -value you want to:: +First, copy the ``.env.example`` file as ``.env`` files from the E2E repo, +and edit whatever value you want to:: - $ cp .env.example .env + $ cp ../dcoleman-e2e/.env.example .env Then before run for the first time the containers, you have to either download the images from Docker Hub or build them from the source code. To build the images from the source code, execute:: - $ docker compose build + $ ./docker-build.sh -Or to get the images from Docker Hub, execute:: +Or to get the images from Docker Hub, execute from the dcoleman-e2e repo:: $ docker compose pull @@ -168,7 +177,7 @@ to the Postgres container so even executing ``docker compose down`` won't delete the data, but if you want to start from scratch:: $ docker compose down - $ docker volume rm pg-coleman_data + $ docker volume rm django-coleman_data Add changes in the code ^^^^^^^^^^^^^^^^^^^^^^^ @@ -178,7 +187,7 @@ When adding changes in the code, the image needs to be updated:: $ docker compose build Then run again. A script ``docker-build.sh`` with more advance -features and without using docker-compose is also provided +features and without using docker compose is also provided to re-build the image. @@ -192,10 +201,11 @@ set *debug* options to false:: $ DEBUG=False LANGUAGE_CODE=es-ar python3 manage.py runserver Also in development environments an ``.env`` file can be used to setup -the environment variables easily, checkout the `<.env.example>`_ as example. +the environment variables easily, checkout the +`.env.example `_ as example. You can copy the example file and edit the variables you want to change:: - $ cp .env.example .env + $ cp ../dcoleman-e2e/.env.example .env $ vi .env Some available settings: @@ -300,6 +310,6 @@ About **Project**: https://github.com/mrsarm/django-coleman -**Authors**: (2017-2023) Mariano Ruiz +**Authors**: (2017-2024) Mariano Ruiz **License**: AGPL-v3 diff --git a/coleman/settings.py b/coleman/settings.py index 7bad842..ff4d2f9 100644 --- a/coleman/settings.py +++ b/coleman/settings.py @@ -48,6 +48,7 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'django_extensions', + 'health_check', ] REST_ENABLED = env.bool('REST_ENABLED', False) @@ -175,6 +176,21 @@ ] } +SESSION_COOKIE_AGE = 8 * 60 * 60 + + +# Google SSO (django-google-sso) +GOOGLE_SSO_ENABLED = env.bool('GOOGLE_SSO_ENABLED', False) +if GOOGLE_SSO_ENABLED: + SSO_SHOW_FORM_ON_ADMIN_PAGE = env.bool('SSO_SHOW_FORM_ON_ADMIN_PAGE', True) + GOOGLE_SSO_CLIENT_ID = env.str("GOOGLE_SSO_CLIENT_ID", None) + GOOGLE_SSO_CLIENT_SECRET = env.str('GOOGLE_SSO_CLIENT_SECRET', None) + GOOGLE_SSO_PROJECT_ID = env.str('GOOGLE_SSO_PROJECT_ID', "django-coleman") + GOOGLE_SSO_AUTO_CREATE_USERS = True + GOOGLE_SSO_STAFF_LIST = ["*"] + GOOGLE_SSO_ALLOWABLE_DOMAINS = env.str('GOOGLE_SSO_ALLOWABLE_DOMAINS', "gmail.com").split(',') + INSTALLED_APPS += ['django_google_sso'] + # # Custom configurations diff --git a/coleman/urls.py b/coleman/urls.py index 129667e..0ead31a 100644 --- a/coleman/urls.py +++ b/coleman/urls.py @@ -13,9 +13,8 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ -from django.urls import re_path, include +from django.urls import path, re_path, include from django.contrib import admin -from django.urls import path from django.conf import settings from django.http import HttpResponseRedirect @@ -29,6 +28,7 @@ urlpatterns = [ re_path('^api/v1/', include(router.urls)), + re_path(r'^health/', include('health_check.urls')), ] if settings.ADMIN: @@ -37,5 +37,15 @@ path('admin/', admin.site.urls), ] + urlpatterns +if settings.GOOGLE_SSO_ENABLED: + urlpatterns = [ + path( + "google_sso/", include( + "django_google_sso.urls", + namespace="django_google_sso" + ) + ), + ] + urlpatterns + admin.site.site_title = admin.site.site_header = settings.SITE_HEADER admin.site.index_title = settings.INDEX_TITLE diff --git a/compose.yml b/compose.yml deleted file mode 100644 index 17717a6..0000000 --- a/compose.yml +++ /dev/null @@ -1,146 +0,0 @@ -volumes: - data: # Volume name listed: django-coleman_data - driver: local - -services: - django-coleman: - image: "mrsarm/django-coleman:${TAG:-latest}" - build: - context: . - args: - - BUILD - ports: - - "8000:8000" - environment: - # Set the values in an .env file or here - - PYTHONUNBUFFERED - - DEBUG - - DATABASE_URL - - POSTGRES_PASSWORD - - LANGUAGE_CODE - - PORT - - REST_ENABLED - - SECRET_KEY - - LOG_LEVEL - - LOG_LEVEL_DJANGO - - LOG_LEVEL_DJANGO_DB - - LOG_LEVEL_DJANGO_REQ - - EMAIL_HOST_USER - - EMAIL_HOST_PASSWORD - - EMAIL_BACKEND - - TASKS_SEND_EMAILS_TO_ASSIGNED - - TASKS_SEND_EMAILS_TO_PARTNERS - - TASKS_VIEWER_ENABLED - - AUTH_PASSWORD_VALIDATORS_ENABLED - - APP_EMAIL - - APP_NAME - - SITE_HEADER - - ADMIN_USERNAME - - ADMIN_PASSWORD - depends_on: - - postgres - - postgres: - image: postgres:15 - environment: - - POSTGRES_PASSWORD=postgres - ports: - - "5432:5432" - volumes: - - data:/var/lib/postgresql/data - command: ["postgres", "-c", "log_statement=all"] # Comment to disable SQL logs. all: all queries, ddl: DDL only, mod: DDL and modifying statements - - django-coleman-mtasks-viewer: - image: "mrsarm/django-coleman-mtasks-viewer" - ports: - - "8888:8888" - environment: - - PORT=8888 - - DCOLEMAN_ENDPOINT - - DCOLEMAN_TASKS_VIEWER_HASH_SALT - - DCOLEMAN_MASTER_TOKEN - depends_on: - - django-coleman - - # - # **tools** profile, none of these services runs on startup by default, - # and all are commands that exit once the task finished - # - - - psql: - image: postgres:15 - command: psql postgres -h postgres -U postgres - depends_on: - - postgres - profiles: - - tools - - - # Provide all the DB resources: DB and DB user creation, DB migrations and admin user creation - django-coleman-provision: - image: "mrsarm/django-coleman:${TAG:-latest}" - environment: - - PROCESS_TYPE=provision - - POSTGRES_PASSWORD - - DATABASE_URL - - ADMIN_USERNAME - - ADMIN_PASSWORD - depends_on: - - postgres - profiles: - - tools - - - # One by one provision tasks - # - # Useful in case you only need to run one, e.g. to - # create a new administrator user "john", you would - # normally run `docker-compose up' first to launch - # all the apps, and then in another terminal: - # `ADMIN_USERNAME=john ADMIN_PASSWORD=secretpass docker-compose up django-coleman-createadmin' - - django-coleman-createdb: - image: "mrsarm/django-coleman:${TAG:-latest}" - environment: - - PROCESS_TYPE=createdb - - POSTGRES_PASSWORD - - DATABASE_URL - depends_on: - - postgres - profiles: - - tools - - django-coleman-migrate: - image: "mrsarm/django-coleman:${TAG:-latest}" - environment: - - PROCESS_TYPE=migrate - - DATABASE_URL - - LOG_LEVEL_DJANGO_DB=INFO - depends_on: - - postgres - profiles: - - tools - - django-coleman-createadmin: - image: "mrsarm/django-coleman:${TAG:-latest}" - environment: - - PROCESS_TYPE=createadmin - - DATABASE_URL - - ADMIN_USERNAME - - ADMIN_PASSWORD - - LOG_LEVEL_DJANGO_DB=INFO - depends_on: - - postgres - profiles: - - tools - - - # Run automated tests - - django-coleman-test: - image: "mrsarm/django-coleman:${TAG:-latest}" - environment: - - PROCESS_TYPE=test - profiles: - - tools diff --git a/docker-build.sh b/docker-build.sh index 04984b7..730617d 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -19,5 +19,7 @@ GIT_HASH_SHORT=$(git rev-parse --short "$GIT_HASH") export BUILD=${GIT_BRANCH}.${GIT_HASH_SHORT} echo "Building mrsarm/django-coleman:${TAG} with image_build $BUILD ..." + +set -x #docker-compose build docker build --build-arg=BUILD="$BUILD" -t mrsarm/django-coleman:${TAG} . diff --git a/mtasks/migrations/0003_auth_groups.py b/mtasks/migrations/0003_auth_groups.py index 3a50644..6f2a323 100644 --- a/mtasks/migrations/0003_auth_groups.py +++ b/mtasks/migrations/0003_auth_groups.py @@ -64,6 +64,8 @@ class Migration(migrations.Migration): """ dependencies = [ + ('contenttypes', '__latest__'), + ('auth', '__latest__'), ('mtasks', '0002_alter_task_options'), ] diff --git a/pytest.ini b/pytest.ini index 176c25e..8a80708 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,7 +1,7 @@ [pytest] DJANGO_SETTINGS_MODULE = coleman.settings_test -addopts = --create-db +addopts = --create-db -rfExXs filterwarnings = ignore:The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives:DeprecationWarning diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in index 9e1128a..1519c2a 100644 --- a/requirements/requirements-dev.in +++ b/requirements/requirements-dev.in @@ -1,7 +1,9 @@ Django~=4.2 -environs~=9.5.0 -dj-database-url~=1.3.0 +environs~=11.0.0 +dj-database-url~=2.2.0 django-admin-list-filter-dropdown~=1.0.3 -django-adminfilters~=2.1.0 -djangorestframework~=3.14.0 -django-extensions~=3.2.1 +django-adminfilters~=2.4.3 +djangorestframework~=3.15.2 +django-extensions~=3.2.3 +django-google-sso~=6.5.0 +django-health-check~=3.18.3 diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 5345760..7e750fb 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -6,31 +6,76 @@ # asgiref==3.6.0 # via django -dj-database-url==1.3.0 +cachetools==5.5.0 + # via google-auth +certifi==2024.7.4 + # via requests +charset-normalizer==3.3.2 + # via requests +dj-database-url==2.2.0 # via -r requirements-dev.in -django==4.2 +django==4.2.15 # via # -r requirements-dev.in # dj-database-url # django-extensions + # django-google-sso + # django-health-check # djangorestframework django-admin-list-filter-dropdown==1.0.3 # via -r requirements-dev.in -django-adminfilters==2.1.0 +django-adminfilters==2.4.3 # via -r requirements-dev.in -django-extensions==3.2.1 +django-extensions==3.2.3 # via -r requirements-dev.in -djangorestframework==3.14.0 +django-google-sso==6.5.0 # via -r requirements-dev.in -environs==9.5.0 +django-health-check==3.18.3 # via -r requirements-dev.in -marshmallow==3.14.1 +djangorestframework==3.15.2 + # via -r requirements-dev.in +environs==11.0.0 + # via -r requirements-dev.in +google-auth==2.34.0 + # via + # django-google-sso + # google-auth-httplib2 + # google-auth-oauthlib +google-auth-httplib2==0.2.0 + # via django-google-sso +google-auth-oauthlib==1.2.1 + # via django-google-sso +httplib2==0.22.0 + # via google-auth-httplib2 +idna==3.8 + # via requests +loguru==0.7.2 + # via django-google-sso +marshmallow==3.22.0 # via environs +oauthlib==3.2.2 + # via requests-oauthlib +packaging==24.1 + # via marshmallow +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pyparsing==3.1.4 + # via httplib2 python-dotenv==0.19.2 # via environs -pytz==2022.1 - # via djangorestframework -sqlparse==0.4.4 +requests==2.32.3 + # via requests-oauthlib +requests-oauthlib==2.0.0 + # via google-auth-oauthlib +rsa==4.9 + # via google-auth +sqlparse==0.5.0 # via django typing-extensions==4.5.0 # via dj-database-url +urllib3==2.2.2 + # via requests diff --git a/requirements/requirements-prod.in b/requirements/requirements-prod.in index 0dac448..1e311fc 100644 --- a/requirements/requirements-prod.in +++ b/requirements/requirements-prod.in @@ -1,4 +1,4 @@ # Web server -uWSGI~=2.0.21 +uWSGI~=2.0.26 # PosgreSQL driver -psycopg[binary]~=3.1.8 +psycopg[binary]~=3.2.1 diff --git a/requirements/requirements-prod.txt b/requirements/requirements-prod.txt index 380ab6a..504b2f8 100644 --- a/requirements/requirements-prod.txt +++ b/requirements/requirements-prod.txt @@ -4,11 +4,11 @@ # # pip-compile --no-emit-index-url --output-file=requirements-prod.txt requirements-prod.in # -psycopg[binary]==3.1.8 +psycopg[binary]==3.2.1 # via -r requirements-prod.in -psycopg-binary==3.1.8 +psycopg-binary==3.2.1 # via psycopg typing-extensions==4.5.0 # via psycopg -uwsgi==2.0.21 +uwsgi==2.0.26 # via -r requirements-prod.in diff --git a/requirements/requirements-test.in b/requirements/requirements-test.in index efbbea5..159a1a4 100644 --- a/requirements/requirements-test.in +++ b/requirements/requirements-test.in @@ -1,3 +1,3 @@ -pytest~=7.3.0 +pytest~=7.4.4 pytest-cov~=4.0.0 -pytest-django==4.5.2 +pytest-django==4.8.0 diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt index c6e3f97..dcd57d5 100644 --- a/requirements/requirements-test.txt +++ b/requirements/requirements-test.txt @@ -8,20 +8,20 @@ coverage[toml]==6.3.1 # via pytest-cov iniconfig==1.1.1 # via pytest -packaging==21.3 +packaging==24.1 # via pytest pluggy==1.0.0 # via pytest -pyparsing==3.0.7 +pyparsing==3.1.4 # via packaging -pytest==7.3.0 +pytest==7.4.4 # via # -r requirements-test.in # pytest-cov # pytest-django pytest-cov==4.0.0 # via -r requirements-test.in -pytest-django==4.5.2 +pytest-django==4.8.0 # via -r requirements-test.in tomli==2.0.1 # via coverage