diff --git a/.editorconfig b/.editorconfig index 527cfc620..4186d9cc9 100644 --- a/.editorconfig +++ b/.editorconfig @@ -23,6 +23,9 @@ indent_size = 2 [*.conf] indent_size = 2 +[*.php] +indent_size = 4 + [DOCKER_REPOSITORY] insert_final_newline = false diff --git a/.gitignore b/.gitignore index 9e101d50e..0e564d7f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ -.idea -/test/Dockerfile -/test/vendor/ -/test/.bundle/ -/test/*.test -/test/docker.test.*.tar +/.idea +/.cache +/tests/serverspec/vendor +/tests/serverspec/.bundle/ +/tests/serverspec/Dockerfile* /BLACKLIST *.log +*.pyc +.doit.db diff --git a/CHANGELOG.md b/CHANGELOG.md index ffc423e89..4f81fa117 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,12 @@ All notable changes to this project will be documented in this file. This project adheres to [WebDevOps.io Dockerfile](https://github.com/webdevops/Dockerfile). -## [1.0.0] - upcoming +## [1.0.0] - 2016-11-28 +- Introduced python based processing script +- Introduced testinfra test suite +- Updated documentation +- Added roundcube to `webdevops/mail-sandbox` +- Update liquibase to 3.5.3 ## [0.57.1] - 2016-10-11 - Added webdevops/liquibase:mysql (same as latest) with mysql support diff --git a/DOCKER_REPOSITORY b/DOCKER_REPOSITORY deleted file mode 100644 index 35a682fcb..000000000 --- a/DOCKER_REPOSITORY +++ /dev/null @@ -1 +0,0 @@ -webdevops \ No newline at end of file diff --git a/DOCKER_TAG_LATEST b/DOCKER_TAG_LATEST deleted file mode 100644 index 5c55a5005..000000000 --- a/DOCKER_TAG_LATEST +++ /dev/null @@ -1 +0,0 @@ -ubuntu-16.04 \ No newline at end of file diff --git a/Makefile b/Makefile index c7fc32f47..5f741cca9 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,11 @@ list: sh -c "echo; $(MAKE) -p no_targets__ | awk -F':' '/^[a-zA-Z0-9][^\$$#\\t=]*:([^=]|$$)/ {split(\$$1,A,/ /);for(i in A)print A[i]}' | grep -v '__\$$' | grep -v 'Makefile'| sort" full: provision build + all: build -build: bootstrap base web php php-dev hhvm service misc applications + +build: + python ./bin/console docker:build --threads=auto bootstrap: webdevops/bootstrap webdevops/ansible base: webdevops/base webdevops/base-app webdevops/storage @@ -26,20 +29,21 @@ applications: webdevops/typo3 webdevops/piwik misc: webdevops/mail-sandbox webdevops/sphinx webdevops/liquibase -test: - cd "test/" && make all +requirements: + pip install -r ./requirements.txt + cd tests/serverspec && bundle install --path=vendor -test-hub-images: - DOCKER_PULL=1 make test +test: + python bin/console test:serverspec --threads=auto/2 -v baselayout: - BASELAYOUT=1 PROVISION=0 bash bin/provision.sh + python bin/console generate:provision --baselayout provision: - python bin/buildDockerfile.py --template=template/ --dockerfile=docker/ - BASELAYOUT=0 PROVISION=1 bash bin/provision.sh + python bin/console generate:dockerfile + python bin/console generate:provision -publish: dist-update rebuild test push +publish: dist-update rebuild test old-test push dist-update: docker pull centos:7 @@ -53,121 +57,104 @@ dist-update: docker pull debian:stretch docker pull alpine:3.4 -rebuild: - # Rebuild all containers but use caching for duplicates - # Bootstrap - FORCE=1 make webdevops/bootstrap - FORCE=0 make webdevops/ansible - # Base - FORCE=1 make webdevops/base - FORCE=1 make webdevops/base-app - FORCE=0 make webdevops/storage - # Other containers - FORCE=1 make web - FORCE=1 make php - FORCE=1 make hhvm - FORCE=1 make service - FORCE=1 make misc - FORCE=1 make applications - push: - BUILD_MODE=push make all + python ./bin/console docker:push --threads=auto setup: pip install --upgrade -I -r ./requirements.txt graph: - python ./bin/diagram.py --dockerfile docker/ --filename documentation/docs/resources/images/docker-image-layout.gv + python ./bin/console generate:graph graph-full: - python ./bin/diagram.py --all --dockerfile docker/ --filename documentation/docs/resources/images/docker-image-full-layout.gv + python ./bin/console generate:graph --all --filename docker-image-full-layout.gv documentation: - docker run -t -i --rm -p 8080:8000 -v "$$(pwd)/documentation/docs/:/opt/docs" webdevops/sphinx sphinx-autobuild --poll -H 0.0.0.0 /opt/docs html + docker run -t -i --rm -p 8080:8000 -v "$$(pwd)/documentation/docs/:/opt/docs" -e "VIRTUAL_HOST=documentation.docker" -e "VIRTUAL_PORT=8000" webdevops/sphinx sphinx-autobuild --poll -H 0.0.0.0 /opt/docs html webdevops/bootstrap: - bash bin/build.sh bootstrap "${DOCKER_REPOSITORY}/bootstrap" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/bootstrap webdevops/ansible: - bash bin/build.sh bootstrap "${DOCKER_REPOSITORY}/ansible" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/bootstrap webdevops/base: - bash bin/build.sh base "${DOCKER_REPOSITORY}/base" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/base webdevops/base-app: - bash bin/build.sh base-app "${DOCKER_REPOSITORY}/base-app" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/base-app webdevops/php: - bash bin/build.sh php "${DOCKER_REPOSITORY}/php" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/php webdevops/php-dev: - bash bin/build.sh php-dev "${DOCKER_REPOSITORY}/php-dev" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/php-dev webdevops/apache: - bash bin/build.sh apache "${DOCKER_REPOSITORY}/apache" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/apache webdevops/apache-dev: - bash bin/build.sh apache-dev "${DOCKER_REPOSITORY}/apache-dev" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/apache-dev webdevops/nginx: - bash bin/build.sh nginx "${DOCKER_REPOSITORY}/nginx" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/nginx webdevops/nginx-dev: - bash bin/build.sh nginx-dev "${DOCKER_REPOSITORY}/nginx-dev" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/nginx-dev webdevops/php-apache: - bash bin/build.sh php-apache "${DOCKER_REPOSITORY}/php-apache" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/php-apache webdevops/php-apache-dev: - bash bin/build.sh php-apache-dev "${DOCKER_REPOSITORY}/php-apache-dev" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/php-apache-dev webdevops/php-nginx: - bash bin/build.sh php-nginx "${DOCKER_REPOSITORY}/php-nginx" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/php-nginx webdevops/php-nginx-dev: - bash bin/build.sh php-nginx-dev "${DOCKER_REPOSITORY}/php-nginx-dev" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/php-nginx-dev webdevops/hhvm: - bash bin/build.sh hhvm "${DOCKER_REPOSITORY}/hhvm" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/hhvm webdevops/hhvm-apache: - bash bin/build.sh hhvm-apache "${DOCKER_REPOSITORY}/hhvm-apache" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/hhvm-apache webdevops/hhvm-nginx: - bash bin/build.sh hhvm-nginx "${DOCKER_REPOSITORY}/hhvm-nginx" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/hhvm-nginx webdevops/ssh: - bash bin/build.sh ssh "${DOCKER_REPOSITORY}/ssh" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/ssh webdevops/storage: - bash bin/build.sh storage "${DOCKER_REPOSITORY}/storage" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/storage webdevops/vsftp: - bash bin/build.sh vsftp "${DOCKER_REPOSITORY}/vsftp" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/vsftp webdevops/postfix: - bash bin/build.sh postfix "${DOCKER_REPOSITORY}/postfix" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/postfix webdevops/mail-sandbox: - bash bin/build.sh mail-sandbox "${DOCKER_REPOSITORY}/mail-sandbox" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/mail-sandbox webdevops/typo3: - bash bin/build.sh typo3 "${DOCKER_REPOSITORY}/typo3" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/typo3 webdevops/piwik: - bash bin/build.sh piwik "${DOCKER_REPOSITORY}/piwik" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/piwik webdevops/samson-deployment: - bash bin/build.sh samson-deployment "${DOCKER_REPOSITORY}/samson-deployment" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/samson-deployment webdevops/sphinx: - bash bin/build.sh sphinx "${DOCKER_REPOSITORY}/sphinx" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/sphinx webdevops/varnish: - bash bin/build.sh varnish "${DOCKER_REPOSITORY}/varnish" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/varnish webdevops/certbot: - bash bin/build.sh certbot "${DOCKER_REPOSITORY}/certbot" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/certbot webdevops/liquibase: - bash bin/build.sh liquibase "${DOCKER_REPOSITORY}/liquibase" "${DOCKER_TAG_LATEST}" + python ./bin/console docker:build --threads=auto --whitelist=webdevops/liquibase diff --git a/README.md b/README.md index e6416b8fc..8ea120777 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Local building of containers can be done with `make` and `Makefile`: Command | Description --------------------------- | ---------------------------------------------------------------------------------- +`sudo make setup` | To Install dependancies of build chain tools `make all` | Build all containers *fast mode* (parallel building, `FAST=1`) `FAST=0 make all` | Build all containers *slow mode* (serial building) `DEBUG=1 make all` | Show log of build process even if process is successfull diff --git a/baselayout/usr/local/bin/apk-install b/baselayout/usr/local/bin/apk-install index cbcedca6e..fe4a6af48 100755 --- a/baselayout/usr/local/bin/apk-install +++ b/baselayout/usr/local/bin/apk-install @@ -7,7 +7,7 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true apk update # Install packages -apk add --upgrade $* +apk add --no-cache --upgrade $* # Clear files (reduce snapshot size) -rm -rf var/cache/apk/* +rm -rf /var/cache/apk/* diff --git a/baselayout/usr/local/bin/apk-install-edge b/baselayout/usr/local/bin/apk-install-edge new file mode 100755 index 000000000..bffed9dbd --- /dev/null +++ b/baselayout/usr/local/bin/apk-install-edge @@ -0,0 +1,18 @@ +#!/bin/sh + +set -o pipefail # trace ERR through pipes +set -o nounset ## set -u : exit the script if you try to use an uninitialised variable +set -o errexit ## set -e : exit the script if any statement returns a non-true return value + +cp -a /etc/apk/repositories /etc/apk/.repositories.original +sed -i -e 's/v[\.0-9]*/edge/g' /etc/apk/repositories + +apk update + +# Install packages +apk add --no-cache --upgrade $* + +# Clear files (reduce snapshot size) +rm -rf /var/cache/apk/* + +mv /etc/apk/.repositories.original /etc/apk/repositories diff --git a/baselayout/usr/local/bin/apk-upgrade b/baselayout/usr/local/bin/apk-upgrade index f577b464a..3512d3122 100755 --- a/baselayout/usr/local/bin/apk-upgrade +++ b/baselayout/usr/local/bin/apk-upgrade @@ -8,7 +8,7 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true apk update # Install packages -apk upgrade --force +apk upgrade --no-cache --force # Clear files (reduce snapshot size) -rm -rf var/cache/apk/* +rm -rf /var/cache/apk/* diff --git a/bin/__init__.py b/bin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bin/build.sh b/bin/build.sh deleted file mode 100755 index 00b926217..000000000 --- a/bin/build.sh +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "$FAST" ]; then - FAST=1 -fi - -if [ -z "$DEBUG" ]; then - DEBUG=0 -fi - -if [ -z "$FORCE" ]; then - FORCE=0 -fi - -if [ -z "$WHITELIST" ]; then - WHITELIST="" -fi - -if [ -z "${BUILD_MODE}" ]; then - BUILD_MODE="build" -fi - -case "$BUILD_MODE" in - - build|push) - BUILD_MODE="$BUILD_MODE" - ;; - - *) - echo "[ERROR] Unknown build mode \"$BUILD_MODE\"" - exit 1 - ;; -esac - - -set -o pipefail # trace ERR through pipes -set -o errtrace # trace ERR through 'time command' and other functions -set -o nounset ## set -u : exit the script if you try to use an uninitialised variable -set -o errexit ## set -e : exit the script if any statement returns a non-true return value - -TARGET="$1" -BASENAME="$2" -LATEST="$3" -BASE_DIR="$(pwd)" - -source "${BASE_DIR}/bin/functions.sh" - -### - # Build dockerfile - # - # will build one docker container, background mode if FAST mode is active - # - # $1 -> dockerfile path (php/) - # $2 -> docker container name (eg. webdevops/php) - # $3 -> docker container tag (eg. ubuntu-14.04) - # - ## -function buildDockerfile() { - local DOCKERFILE_PATH="$1" - local CONTAINER_NAME="$2" - local CONTAINER_TAG="$3" - - echo ">> Starting build of ${CONTAINER_NAME}:${CONTAINER_TAG}" - - if [ "${FAST}" -eq 1 ]; then - LOGFILE="$(mktemp /tmp/docker.build.XXXXXXXXXX)" - "${BASE_DIR}/bin/retry.sh" "${BASE_DIR}/bin/buildContainer.sh" "${DOCKERFILE_PATH}" "${CONTAINER_NAME}" "${CONTAINER_TAG}" &> "$LOGFILE" & - addBackgroundPidToList "${CONTAINER_TAG}" "$LOGFILE" - else - "${BASE_DIR}/bin/retry.sh" "${BASE_DIR}/bin/buildContainer.sh" "${DOCKERFILE_PATH}" "${CONTAINER_NAME}" "${CONTAINER_TAG}" - fi - - cd "$BASE_DIR" -} - -### - # Push dockerfile - # - # will push one docker container, background mode if FAST mode is active - # - # $1 -> dockerfile path (php/) - # $2 -> docker container name (eg. webdevops/php) - # $3 -> docker container tag (eg. ubuntu-14.04) - # - ## -function pushDockerfile() { - local DOCKERFILE_PATH="$1" - local CONTAINER_NAME="$2" - local CONTAINER_TAG="$3" - - echo ">> Starting push of ${CONTAINER_NAME}:${CONTAINER_TAG}" - - if [ "${FAST}" -eq 1 ]; then - LOGFILE="$(mktemp /tmp/docker.push.XXXXXXXXXX)" - "${BASE_DIR}/bin/retry.sh" docker push "${CONTAINER_NAME}:${CONTAINER_TAG}" &> "$LOGFILE" & - addBackgroundPidToList "${CONTAINER_TAG}" "$LOGFILE" - else - "${BASE_DIR}/bin/retry.sh" docker push "${CONTAINER_NAME}:${CONTAINER_TAG}" - fi - - cd "$BASE_DIR" -} - -### - # Wait for build - # - # will wait for parallel build processes (only FAST mode) - # - ## -function waitForBuild() { - if [ "${FAST}" -eq 1 ]; then - waitForBuildStep - echo " -> $BASENAME build finished" - fi -} - -### - # Wait for build - # - # will wait for parallel build processes (only FAST mode) - # - ## -function waitForBuildStep() { - if [ "${FAST}" -eq 1 ]; then - echo " -> waiting for background processes..." - waitForBackgroundProcesses - wait - fi -} - -### - # Run build task - # - ## -function buildTarget() { - case "$BUILD_MODE" in - build) - buildDockerfile "${DOCKERFILE_PATH}" "${BASENAME}" "${TAGNAME}" - sleep 0.05 - ;; - - push) - pushDockerfile "${DOCKERFILE_PATH}" "${BASENAME}" "${TAGNAME}" - sleep 0.05 - ;; - esac -} - -### - # Run build task for latest container - # - ## -function buildTargetLatest() { - TAGNAME="latest" - - buildTarget -} - -### - # Check if docker image is available - ## -function checkBuild() { - if [[ -n "$(docker images -q "${BASENAME}:${TAGNAME}" 2> /dev/null)" ]]; then - echo " -> Image ${BASENAME}:${TAGNAME} found" - else - echo " [ERROR] Docker image '${BASENAME}:${TAGNAME}' not found, build failure!" - exit 1; - fi -} - -### - # Check if docker image is available (latest image) - ## -function checkBuildLatest() { - TAGNAME="latest" - checkBuild -} - - -############################################################################### -# MAIN -############################################################################### - -printLine "=" - -if [ "${FAST}" -eq 1 ]; then - echo -n "== Docker image $BASENAME (PARALLEL MODE)" -else - echo -n "== Docker image $BASENAME" -fi - -if [ "${DEBUG}" -eq 1 ]; then - echo -n " >>DEBUG MODE<<" -fi - - -if [ "${FORCE}" -eq 1 ]; then - echo -n " >>FORCE MODE<<" -fi - -echo "" - -printLine "=" -echo "" - -sleep 0.5 - -############################# -# Main -############################# - -## Init - -initPidList -timerStart - -## Build image - -echo "Building ${BASENAME}" -## Build each docker tag -foreachDockerfileInPath "docker/${TARGET}" "buildTarget" - -# wait for build process -waitForBuildStep - -## Build docker tag latest -if [ -z "$WHITELIST" ]; then - foreachDockerfileInPath "docker/${TARGET}" "buildTargetLatest" "${LATEST}" -fi -# wait for final build -waitForBuild - -logOutputFromBackgroundProcesses - -## Check builds (only "build" mode) -case "$BUILD_MODE" in - build) - echo ">> Checking built images" - foreachDockerfileInPath "docker/${TARGET}" "checkBuild" - if [ -z "$WHITELIST" ]; then - foreachDockerfileInPath "docker/${TARGET}" "checkBuildLatest" "${LATEST}" - fi - ;; -esac - -echo "" -echo ">>> Build time: $(timerStep)" -echo "" diff --git a/bin/buildContainer.sh b/bin/buildContainer.sh deleted file mode 100755 index 74bbf339f..000000000 --- a/bin/buildContainer.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -LOGFILE="" - -if [ -z "$DEBUG" ]; then - DEBUG=0 -fi - -if [ -z "$FORCE" ]; then - FORCE=0 -fi - -if [ -z "$DOCKER_OPTS" ]; then - DOCKER_OPTS="" -fi - -set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - -DOCKERFILE_PATH="$1" -CONTAINER_NAME="$2" -CONTAINER_TAG="$3" - -############################################################################### -# MAIN -############################################################################### - -if [ "$FORCE" -eq 1 ]; then - DOCKER_OPTS="$DOCKER_OPTS --no-cache" -fi - -# Prevent pull of parent docker images -DOCKER_OPTS="$DOCKER_OPTS --pull=false" - -cd "$DOCKERFILE_PATH" - -if [ "$DEBUG" -eq 0 ]; then - # Background mode, write all logs into tmpfile - LOGFILE="$(mktemp /tmp/docker.build.XXXXXXXXXX)" - docker build $DOCKER_OPTS -t "${CONTAINER_NAME}:${CONTAINER_TAG}" . &> "$LOGFILE" - DOCKER_BUILD_RET="$?" -else - # Foreground mode, write all logs to STDOUT - docker build $DOCKER_OPTS -t "${CONTAINER_NAME}:${CONTAINER_TAG}" . - DOCKER_BUILD_RET="$?" -fi - - -if [ "$DOCKER_BUILD_RET" -ne 0 ]; then - # docker build failed - # output the logfile - if [ -n "$LOGFILE" ]; then - # LOGFILE is set so the build was done in background mode - # -> output logfile content - - cat "$LOGFILE" - fi - - # output error message - echo "" - echo "" - echo "-----------------------------------------------------------" - echo " --- BUILD FAILURE -> ${CONTAINER_NAME}:${CONTAINER_TAG}" - echo "-----------------------------------------------------------" -fi - -if [ -n "$LOGFILE" ]; then - # remove tmpfile (logfile) - rm -f -- "$LOGFILE" -fi - -# exit with docker build return code -exit "$DOCKER_BUILD_RET" diff --git a/bin/buildDockerfile.py b/bin/buildDockerfile.py deleted file mode 100644 index 87381ad62..000000000 --- a/bin/buildDockerfile.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env/python - -from jinja2 import Environment, FileSystemLoader -from datetime import datetime -import os -import argparse - -PATH = os.path.dirname(os.path.abspath(__file__)) - -templateHeader = '{% extends "Dockerfile/layout.jinja2" %}\n{% block content %}' -templateFooter = '{% endblock %}' - -def get_current_date(): - import datetime - return datetime.date.today().strftime("%d.%m.%Y") - -def processDockerfile(inputFile, template): - outputFile = os.path.splitext(inputFile) - outputFile = os.path.join(os.path.dirname(outputFile[0]),os.path.basename(outputFile[0])) - - dockerImage = os.path.basename(os.path.dirname(os.path.dirname(outputFile))) - dockerTag = os.path.basename(os.path.dirname(outputFile)) - - context = { - 'Dockerfile': { - 'image': dockerImage, - 'tag': dockerTag - } - } - - print "* Processing Dockerfile for " + dockerImage + ":" + dockerTag - - with open(inputFile, 'r') as fileInput: - templateContent = fileInput.read() - - templateContent = templateHeader + templateContent + templateFooter - - renderedContent = template.from_string(templateContent).render(context) - renderedContent = renderedContent.lstrip() - - with open(outputFile, 'w') as fileOutput: - fileOutput.write(renderedContent) - - - -def main(args): - templatePath = os.path.abspath(args.template) - dockerfilePath = os.path.abspath(args.dockerfile) - - template = Environment( - autoescape=False, - loader=FileSystemLoader([templatePath]), - trim_blocks=False - ) - - for root, dirs, files in os.walk(dockerfilePath): - for file in files: - if file.endswith("Dockerfile.jinja2"): - processDockerfile(os.path.join(root, file), template) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-t','--template' ,help='',type=str) - parser.add_argument('-d','--dockerfile' ,help='',type=str) - args = parser.parse_args() - main(args) diff --git a/bin/command/__init__.py b/bin/command/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bin/command/docker_build_command.py b/bin/command/docker_build_command.py new file mode 100644 index 000000000..8472564ef --- /dev/null +++ b/bin/command/docker_build_command.py @@ -0,0 +1,48 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from cleo import Output +from webdevops import Dockerfile +from webdevops.command import DoitCommand +from webdevops.taskloader import DockerBuildTaskLoader + +class DockerBuildCommand(DoitCommand): + """ + Build images + + docker:build + {--dry-run : show only which images will be build} + {--no-cache : build without caching} + {--t|threads=0 : threads} + {--r|retry=0 : retry} + {--whitelist=?* : image/tag whitelist } + {--blacklist=?* : image/tag blacklist } + """ + + def run_task(self, configuration): + configuration.set('dockerBuild.noCache', self.option('no-cache')) + + return self.run_doit( + task_loader=DockerBuildTaskLoader(configuration), + configuration=configuration + ) + + + diff --git a/bin/command/docker_pull_command.py b/bin/command/docker_pull_command.py new file mode 100644 index 000000000..9608b8477 --- /dev/null +++ b/bin/command/docker_pull_command.py @@ -0,0 +1,44 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from cleo import Output +from webdevops.command import DoitCommand +from webdevops.taskloader import DockerPullTaskLoader + +class DockerPullCommand(DoitCommand): + """ + Pull all built images from registry/hub + + docker:pull + {--dry-run : show only which images will be build} + {--t|threads=0 : threads} + {--r|retry=0 : retry} + {--whitelist=?* : image/tag whitelist } + {--blacklist=?* : image/tag blacklist } + """ + + def run_task(self, configuration): + return self.run_doit( + task_loader=DockerPullTaskLoader(configuration), + configuration=configuration + ) + + + diff --git a/bin/command/docker_push_command.py b/bin/command/docker_push_command.py new file mode 100644 index 000000000..6f0f55daf --- /dev/null +++ b/bin/command/docker_push_command.py @@ -0,0 +1,45 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from cleo import Output +from webdevops.command import DoitCommand +from webdevops.taskloader import DockerPushTaskLoader + +class DockerPushCommand(DoitCommand): + """ + Push images to registry/hub + + docker:push + {--dry-run : show only which images will be build} + {--t|threads=0 : threads} + {--r|retry=0 : retry} + {--whitelist=?* : image/tag whitelist } + {--blacklist=?* : image/tag blacklist } + """ + + def run_task(self, configuration): + return self.run_doit( + task_loader=DockerPushTaskLoader(configuration), + configuration=configuration + ) + + + + diff --git a/bin/command/generate_dockerfile_command.py b/bin/command/generate_dockerfile_command.py new file mode 100644 index 000000000..b5a27b22f --- /dev/null +++ b/bin/command/generate_dockerfile_command.py @@ -0,0 +1,100 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +from cleo import Output +from jinja2 import Environment, FileSystemLoader +from webdevops import DockerfileUtility +from webdevops.command import BaseCommand + +class GenerateDockerfileCommand(BaseCommand): + """ + Build Dockerfile containers + + generate:dockerfile + {--whitelist=?* : image/tag whitelist } + {--blacklist=?* : image/tag blacklist } + """ + + template = '' + + template_header = '{% extends "Dockerfile/layout.jinja2" %}\n{% block content %}' + template_footer = '{% endblock %}' + + def run_task(self, configuration): + template_path = configuration.get('templatePath') + dockerfile_path = configuration.get('dockerPath') + whitelist = self.get_whitelist() + blacklist = self.get_blacklist() + + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line('-> docker path : %s' % dockerfile_path) + self.line('-> template path : %s' % template_path) + + if whitelist: + self.line('-> whitelist :') + for crit in whitelist: + self.line("\t * %s" % crit) + + if blacklist: + self.line('-> blacklist :') + for crit in blacklist: + self.line("\t * %s" % crit) + + self.template = Environment( + autoescape=False, + loader=FileSystemLoader([template_path]), + trim_blocks=False + ) + + for file in DockerfileUtility.find_file_in_path(dockerfile_path=dockerfile_path, filename="Dockerfile.jinja2", whitelist=whitelist, blacklist=blacklist): + self.process_dockerfile(file) + + def process_dockerfile(self, input_file): + """ + :param input_file: Input File + :type input_file: str + """ + output_file = os.path.splitext(input_file) + output_file = os.path.join(os.path.dirname(output_file[0]), os.path.basename(output_file[0])) + + docker_image = os.path.basename(os.path.dirname(os.path.dirname(output_file))) + docker_tag = os.path.basename(os.path.dirname(output_file)) + + context = { + 'Dockerfile': { + 'image': docker_image, + 'tag': docker_tag + } + } + + if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): + self.line("* Processing Dockerfile for %s:%s" % (docker_image,docker_tag)) + + with open(input_file, 'r') as fileInput: + template_content = fileInput.read() + + template_content = self.template_header + template_content + self.template_footer + + rendered_content = self.template.from_string(template_content).render(context) + rendered_content = rendered_content.lstrip() + + with open(output_file, 'w') as file_output: + file_output.write(rendered_content) diff --git a/bin/command/generate_graph_command.py b/bin/command/generate_graph_command.py new file mode 100644 index 000000000..1a82ee378 --- /dev/null +++ b/bin/command/generate_graph_command.py @@ -0,0 +1,248 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import re +import yaml +from datetime import date +from graphviz import Digraph +from cleo import Output +from webdevops import DockerfileUtility +from webdevops.command import BaseCommand +from cleo.validators import Enum + +class GenerateGraphCommand(BaseCommand): + """ + Generate a diagram of container + + generate:graph + {--a|all : Show all informations} + {--o|output= : path output} + {--F|format=png (choice) : output format } + {--f|filename=docker-image-layout.gv : file output} + """ + + validation = { + '--format': Enum(['png', 'jpg', 'pdf', 'svg']) + } + + from_regex = re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE) + + containers = {} + + tags = {} + + subgraph = {} + + edges = {} + + conf = '' + + def run_task(self, configuration): + if self.option('output'): + configuration.set('imagePath', self.option('output')) + + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line('ALL : %s' % self.option('all')) + self.line('output : %s' % configuration.get('imagePath')) + self.line('format : %s' % self.option('format')) + self.line('dockerPath : %s' % configuration.get('dockerPath')) + self.line('filename : %s' % self.option('filename')) + self.__load_configuration() + + dockerfileList = DockerfileUtility.find_dockerfiles_in_path( + base_path=configuration.get('dockerPath'), + path_regex=configuration.get('docker.pathRegex'), + image_prefix=configuration.get('docker.imagePrefix'), + ) + + for dockerfile in dockerfileList: + self.__process_dockerfile(dockerfile) + + dia = self.build_graph() + dia.render() + + def __process_dockerfile(self, dockerfile): + """ + + :param dockerfile_full_path: + :type dockerfile_full_path: str + + :return: self + :rtype: self + """ + + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line('-> Processing: %s' % dockerfile['image']['fullname']) + + docker_image = dockerfile['image']['name'] + parent_image_name = DockerfileUtility.image_basename(dockerfile['image']['from']) + parent_image_tag = DockerfileUtility.extract_image_name_tag(dockerfile['image']['from']) + + if not parent_image_name in self.containers: + self.containers[parent_image_name] = 'scratch' + self.__append_tag(parent_image_name, 'latest') + + self.containers[docker_image] = parent_image_name + self.__append_tag(docker_image, parent_image_tag) + + return self + + def __append_tag(self, docker_image, tag): + """ + + :param docker_image: + :type docker_image: str + + :param tag: + :type tag: str + + :return: self + """ + if not self.tags.has_key(docker_image): + self.tags[docker_image] = {} + self.tags[docker_image][tag] = tag + return self + + def __get_graph(self, default_graph, name): + """ + + :param default_graph: Main diagram + :type default_graph: Digraph + + :param name: Name of the sub-diagram + :type name: str + + :return: the selected diagram + :rtype: Digraph + """ + for group, group_attr in self.conf['diagram']['groups'].items(): + for dockerRegex in group_attr['docker']: + if re.match(dockerRegex, name): + return group, self.subgraph[group] + return '__root__', default_graph + + def __load_configuration(self): + conf_file_path = os.path.join(self.configuration.get('confPath'), 'diagram.yml') + + stream = open(conf_file_path, 'r') + self.conf = yaml.safe_load(stream) + + def __apply_styles(self, graph, styles): + """ + + :param graph: Diagraph to apply styles + :type graph Digraph + + :param styles: Styles properties of Diagraph + :type styles: dict + + :return: The GRaph with Style + :rtype: Digraph + """ + graph.graph_attr.update( + ('graph' in styles and styles['graph']) or {} + ) + graph.node_attr.update( + ('nodes' in styles and styles['nodes']) or {} + ) + graph.edge_attr.update( + ('edges' in styles and styles['edges']) or {} + ) + return graph + + def build_graph(self): + """ + + :return: the graph + :rtype: Digraph + """ + dia = Digraph('webdevops', + filename=self.option('filename'), + format=self.option('format'), + directory=self.configuration.get('imagePath') + ) + dia = self.__apply_styles(dia, self.conf['diagram']['styles']) + + label = r'label = "\n\n%s"' % self.configuration.get('graph.label') + + dia.body.append(label % date.today().strftime("%Y-%m-%d")) + dia.body.append('newrank=true;') + + rank_image_list = {} + rank_group_list = {} + + # Create subgraph + for group, group_attr in self.conf['diagram']['groups'].items(): + self.subgraph[group] = Digraph('cluster_%s' % group) + self.subgraph[group].body.append(r'label = "%s"' % group_attr['name']) + self.subgraph[group] = self.__apply_styles(self.subgraph[group], group_attr['styles']) + + if 'rank' in group_attr: + rank_group_list[group] = group_attr['rank'] + + for image, base in self.containers.items(): + group_image, graph_image = self.__get_graph(dia, image) + group_base, graph_base = self.__get_graph(dia, base) + if not "scratch" in base: + if graph_image == graph_base: + graph_image.edge(base, image) + else: + graph_image.node(image) + self.edges[image] = base + if self.option('all'): + self.__attach_tag(graph_image, image) + else: + graph_image.node(image) + + if group_image in rank_group_list: + image_rank = rank_group_list[group_image] + + if not image_rank in rank_image_list: + rank_image_list[image_rank] = [] + + rank_image_list[image_rank].append(image) + + # add repositories (subgraph/cluster) + for name, subgraph in self.subgraph.items(): + dia.subgraph(subgraph) + + # add images (node) + for image, base in self.edges.items(): + dia.edge(base, image) + + # add invisible constraints to add ranked groups + for rank, imagelist in rank_image_list.items(): + rank_next = rank + 1 + + if rank_next in rank_image_list: + imagelist_next = rank_image_list[rank_next] + + for image in imagelist: + for image_next in imagelist_next: + dia.body.append('{ "%s" -> "%s" [style=invis] }' % (image, image_next)) + + return dia + + def __attach_tag(self, graph, image): + for tag in self.tags[image]: + node_name = "%s-%s" % (image, tag) + graph.node(node_name, label=tag, fillcolor='#eeeeee', shape='folder') + graph.edge(image, node_name) diff --git a/bin/command/generate_provision_command.py b/bin/command/generate_provision_command.py new file mode 100644 index 000000000..5ed4d831d --- /dev/null +++ b/bin/command/generate_provision_command.py @@ -0,0 +1,103 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import yaml +import yamlordereddictloader +import time +import Queue +import shutil +from cleo import Output +from webdevops import Provisioner +from webdevops.command import BaseCommand + +class GenerateProvisionCommand(BaseCommand): + """ + Provisionning docker images + + generate:provision + {--image=?* : filter on images name } + {--baselayout : Build the baselayout } + {--t|threads=0 : threads} + """ + + conf = '' + + __threads = [] + + __queue = '' + + def run_task(self, configuration): + self.__queue = Queue.Queue() + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line('provision : %s' % configuration.get('provisionPath')) + self.line('dockerfile : %s' % configuration.get('dockerPath')) + self.line('baselayout : %s' % self.option('baselayout')) + if 0 < len(self.option('image')): + self.line('images :') + for crit in self.option('image'): + self.line("\t * %s" % crit) + self.__load_configuration() + self.__build_base_layout() + self.__create_thread() + for image_name in self.conf['provision']: + if 0 == len(self.option('image')) or image_name in self.option('image'): + self.__queue.put({'image_name': image_name, 'image_config': self.conf['provision'][image_name]}) + self.__queue.join() + if os.path.exists('baselayout.tar'): + os.remove('baselayout.tar') + + def __create_thread(self): + for i in range(self.get_threads()): + thread_name = "Pixie_%d" % i + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line("* -> Create thread %s" % thread_name) + provisioner = Provisioner( + self.configuration.get('dockerPath'), + self.configuration.get('provisionPath'), + self.__queue, + self.output + ) + provisioner.setDaemon(True) + provisioner.setName(thread_name) + provisioner.start() + # self.__threads.append(provisioner) + + def __load_configuration(self): + """ + Load the configuration for provisioning image + """ + configuration_file = os.path.join(self.configuration.get('confPath'), 'provision.yml') + stream = open(configuration_file, "r") + self.conf = yaml.load(stream, Loader=yamlordereddictloader.Loader) + + def __build_base_layout(self): + """ + Build tar file from _localscripts for bootstrap containers + """ + if self.option('baselayout'): + if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): + self.line('* Building localscipts') + base_path = os.path.join(self.configuration.get('baselayoutPath'), 'baselayout') + shutil.make_archive('baselayout', 'bztar', base_path) + os.rename('baselayout.tar.bz2', 'baselayout.tar') + + + diff --git a/bin/command/test_serverspec_command.py b/bin/command/test_serverspec_command.py new file mode 100644 index 000000000..2cb878f20 --- /dev/null +++ b/bin/command/test_serverspec_command.py @@ -0,0 +1,60 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os, glob +from cleo import Output +from webdevops import Dockerfile, DockerfileUtility +from webdevops.command import DoitCommand +from webdevops.taskloader import DockerTestServerspecTaskLoader + +class TestServerspecCommand(DoitCommand): + """ + Test docker images with Serverspec + + test:serverspec + {--dry-run : show only which images will be build} + {--t|threads=0 : threads} + {--r|retry=0 : retry} + {--whitelist=?* : image/tag whitelist } + {--blacklist=?* : image/tag blacklist } + """ + + serverspec_path = False + + def run_task(self, configuration): + self.serverspec_path = configuration.get('serverspecPath') + + self.cleanup_dockerfiles() + + return self.run_doit( + task_loader=DockerTestServerspecTaskLoader(configuration), + configuration=configuration + ) + + def cleanup_dockerfiles(self): + """ + Cleanup old dockerfiles in test directory + """ + for file in glob.glob(os.path.join(self.serverspec_path, 'Dockerfile.*.tmp')): + os.remove(file) + + def teardown(self, exitcode): + self.cleanup_dockerfiles() + diff --git a/bin/command/test_testinfra_command.py b/bin/command/test_testinfra_command.py new file mode 100644 index 000000000..2250cd675 --- /dev/null +++ b/bin/command/test_testinfra_command.py @@ -0,0 +1,41 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from cleo import Output +from webdevops.command import DoitCommand +from webdevops.taskloader import DockerTestTestinfraTaskLoader + +class TestTestinfraCommand(DoitCommand): + """ + Test docker images with Testinfra + + test:testinfra + {--dry-run : show only which images will be build} + {--t|threads=0 : threads} + {--whitelist=?* : image/tag whitelist } + {--blacklist=?* : image/tag blacklist } + """ + + def run_task(self, configuration): + return self.run_doit( + task_loader=DockerTestTestinfraTaskLoader(configuration), + configuration=configuration + ) + diff --git a/bin/console b/bin/console new file mode 100755 index 000000000..2496589e1 --- /dev/null +++ b/bin/console @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os, sys + +# prevent bytecode +sys.dont_write_bytecode = True + +# unbuffered stdout / stderr +sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) +sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) + +import re, yaml +from cleo import Application + +from webdevops import Configuration +from webdevops.docker.DockerBaseClient import DockerBaseClient +from webdevops.docker.DockerPyClient import DockerPyClient +from webdevops.docker.DockerCliClient import DockerCliClient +from command.docker_build_command import DockerBuildCommand +from command.docker_push_command import DockerPushCommand +from command.docker_pull_command import DockerPullCommand +from command.test_testinfra_command import TestTestinfraCommand +from command.test_serverspec_command import TestServerspecCommand +from command.generate_dockerfile_command import GenerateDockerfileCommand +from command.generate_graph_command import GenerateGraphCommand +from command.generate_provision_command import GenerateProvisionCommand + +if __name__ == '__main__': + # Generate common paths + script_path = os.path.dirname(os.path.realpath(__file__)) + root_path = os.path.dirname(script_path) + conf_path = os.path.join(root_path, 'conf') + + + def generatePath(path): + """ + Generate full path based on root path + """ + return os.path.abspath(os.path.join(root_path, (path))) + + # Read console.yml for configuration + with open(os.path.join(conf_path, 'console.yml'), 'r') as stream: + try: + configuration = yaml.load(stream) + configuration['confPath'] = conf_path + except yaml.YAMLError as e: + configuration = None + print ' !!! Exception while loading configuration from %s:' % conf_path + print '' + print e + print '' + sys.exit(1) + + # Check if configuration is valid + if configuration is None: + print ' !!! Configuration not found' + sys.exit(1) + + # generate full paths + path_entries = [ + 'dockerPath', + 'templatePath', + 'provisionPath', + 'imagePath', + 'baselayoutPath', + 'testinfraPath', + 'serverspecPath', + 'blacklistFile', + ] + for key in path_entries: + if key in configuration: + configuration[key] = generatePath(configuration[key]) + + # Translate regexp + if 'docker' in configuration: + if 'pathRegex' in configuration['docker']: + configuration['docker']['pathRegex'] = re.compile(configuration['docker']['pathRegex']) + + if 'pathRegex' in configuration['docker']: + configuration['docker']['autoPullBlacklist'] = re.compile(configuration['docker']['autoPullBlacklist']) + + configuration = Configuration.merge(configuration) + configuration = Configuration.dotdictify(configuration) + + # Init application + application = Application() + application.add(DockerBuildCommand(configuration=configuration)) + application.add(DockerPushCommand(configuration=configuration)) + application.add(DockerPullCommand(configuration=configuration)) + application.add(TestTestinfraCommand(configuration=configuration)) + application.add(TestServerspecCommand(configuration=configuration)) + application.add(GenerateDockerfileCommand(configuration=configuration)) + application.add(GenerateGraphCommand(configuration=configuration)) + application.add(GenerateProvisionCommand(configuration=configuration)) + + application.run() diff --git a/bin/diagram.py b/bin/diagram.py deleted file mode 100644 index fe27b29e1..000000000 --- a/bin/diagram.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env/python - -from datetime import datetime -import os -import argparse -import re -from graphviz import Digraph -import yaml - -PATH = os.path.dirname(os.path.abspath(__file__)) -FROM_REGEX = re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE) -CONTAINERS = {} -TAGS = {} -SUBGRAPH = {} -EDGES = {} - -def get_current_date(): - import datetime - return datetime.date.today().strftime("%d.%m.%Y") - -def processDockerfile(inputFile): - outputFile = os.path.splitext(inputFile) - outputFile = os.path.join(os.path.dirname(outputFile[0]),os.path.basename(outputFile[0])) - - dockerImage = os.path.basename(os.path.dirname(os.path.dirname(outputFile))) - dockerTag = os.path.basename(os.path.dirname(outputFile)) - - with open(inputFile, 'r') as fileInput: - DockerfileContent = fileInput.read() - data = ([m.groupdict() for m in FROM_REGEX.finditer(DockerfileContent)])[0] - key="webdevops/%s"%dockerImage - CONTAINERS[key] = data.get('image') - appendTag(key, data.get('tag')) - -def appendTag(dockerImage, tag): - if False == TAGS.has_key(dockerImage): - TAGS[dockerImage] = {} - TAGS[dockerImage][tag] = tag - -def apply_styles(graph, styles): - graph.graph_attr.update( - ('graph' in styles and styles['graph']) or {} - ) - graph.node_attr.update( - ('nodes' in styles and styles['nodes']) or {} - ) - graph.edge_attr.update( - ('edges' in styles and styles['edges']) or {} - ) - return graph - -def get_graph(conf,default_graph,name): - - for group, group_attr in conf['diagram']['groups'].items(): - if name in group_attr['docker']: - return SUBGRAPH[group] - return default_graph - -def build_graph(args): - stream = open(os.path.dirname(__file__)+"/diagram.yml", "r") - conf_diagram = yaml.safe_load(stream) - dia = Digraph('webdevops', filename=args.filename, format=args.format, directory=args.path) - dia = apply_styles(dia,conf_diagram['diagram']['styles']) - dia.body.append(r'label = "\n\nWebdevops Images\n at :%s"' % get_current_date() ) - - # Create subgraph - for group, group_attr in conf_diagram['diagram']['groups'].items(): - SUBGRAPH[group] = Digraph("cluster_"+group); - SUBGRAPH[group].body.append(r'label = "%s"' % group_attr['name'] ) - SUBGRAPH[group] = apply_styles(SUBGRAPH[group],group_attr['styles'] ) - for image, base in CONTAINERS.items(): - graph_image = get_graph(conf_diagram, dia, image) - graph_base = get_graph(conf_diagram, dia, base) - if "webdevops" in base: - if graph_image == graph_base: - graph_image.edge(base, image) - else: - graph_image.node(image) - EDGES[image] = base - if args.all : - attach_tag(graph_image, image) - else: - graph_image.node(image) - - for name, subgraph in SUBGRAPH.items(): - dia.subgraph(subgraph) - - for image, base in EDGES.items(): - dia.edge(base, image) - return dia - -def attach_tag(graph, image): - for tag in TAGS[image]: - node_name = "%s-%s"%(image,tag) - node = graph.node(node_name, label=tag, fillcolor='#eeeeee', shape='folder' ) - edge = graph.edge(image, node_name ) - -def main(args): - dockerfilePath = os.path.abspath(args.dockerfile) - - # Parse Docker file - for root, dirs, files in os.walk(dockerfilePath): - - for file in files: - if file.endswith("Dockerfile"): - processDockerfile(os.path.join(root, file)) - - dia = build_graph(args) - dia.render() - print " render to: %s"%args.filename - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-d','--dockerfile' ,help='path to the folder containing dockerfile analyze',type=str) - parser.add_argument('-f','--filename' ,help='file output (default: webdevops.gv)',default='webdevops.gv',type=str) - parser.add_argument('-F','--format' ,help='output format (default: png)',default='png',choices=('png','jpg','pdf','svg')) - parser.add_argument('-p','--path' ,help='path output',default=os.path.dirname(__file__)+"/../",type=str) - parser.add_argument('--all' ,help='show all info',dest='all' ,action='store_true') - parser.set_defaults(all=False) - - args = parser.parse_args() - - main(args) diff --git a/bin/functions.sh b/bin/functions.sh deleted file mode 100755 index 0f707936f..000000000 --- a/bin/functions.sh +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env bash - -SED='sed' - -[[ `uname` == 'Darwin' ]] && { - which gsed > /dev/null && { - SED='gsed' - } || { - echo 'ERROR: GNU utils required for Mac. You may use homebrew to install them: brew install coreutils gnu-sed' - exit 1 - } -} - -################################################# -# General handling -################################################# - -log () { - echo "$1" -} - -logError() { - echo " [ERROR] $1" >&2 -} - -exitError() { - echo "" - echo "$(date) Exit -> $1" >&2 - exit $1 -} - -RETRY=5 - -retry() { - local n=1 - local max="$RETRY" - local delay=15m - while true; do - "$@" && break || { - if [[ $n -lt $max ]]; then - ((n++)) - log "Command failed. Attempt $n/$max:" - sleep $delay; - else - exitError "The command has failed after $n attempts." - fi - } - done -} - - -function printLine() { - printf "${1}%.0s" $(seq 1 50) - echo -} - - -################################################# -# Macro functions -################################################# - -### - # Get list of macros in file - # - # $1 -> Target file - ## -function getMacroList() { - FILE_TARGET="$1" - - grepRegexp='^# ' - sedRegexp='^# ]*\)>' - - MARKER_LIST="$(grep -h -E -e "$grepRegexp" "$FILE_TARGET" || exit 0 )" - - if [[ -n "$MARKER_LIST" ]]; then - echo "$MARKER_LIST" | $SED -e "s/$sedRegexp/\\1/" - fi -} - - -### - # Replace marker area in one file with another file - # - # $1 -> Target file - # $2 -> Marker content file - # $3 -> Marker name - ## -function replaceMacro() { - FILE_TARGET="$1" - FILE_CONTENT="$2" - MACRO="$3" - - lead="^# " - tail="^# <\/Macro>" - - $SED -i -e "/$lead/,/$tail/{ /$lead/{p; r $FILE_CONTENT - }; /$tail/p; d }" "$FILE_TARGET" -} - -################################################# -# Background process handling -################################################# - -initPidList() { - unset PID_LIST - declare -a PID_LIST - PID_LIST=() - - unset PID_LOG_TITLE - declare -a PID_LOG_TITLE - PID_LOG_TITLE=() - - unset PID_LOG_FILE - declare -a PID_LOG_FILE - PID_LOG_FILE=() -} - -addBackgroundPidToList() { - local BG_PID="$!" - - case "$#" in - 1) - PID_LIST[$BG_PID]="$1" - ;; - - 2) - PID_LIST[$BG_PID]="$1" - PID_LOG_TITLE[$BG_PID]="$1" - PID_LOG_FILE[$BG_PID]="$2" - ;; - - *) - PID_LIST[$BG_PID]="$BG_PID" - ;; - esac -} - - -ALWAYS_SHOW_LOGS=0 -waitForBackgroundProcesses() { - local WAIT_BG_RETURN=0 - - ## check if pidlist exists - if [[ -z "${PID_LIST[@]:+${PID_LIST[@]}}" ]]; then - log " -> No background processes found" - return - fi - - while [ 1 ]; do - - for pid in "${!PID_LIST[@]}"; do - title="${PID_LIST[$pid]}" - - # check if pid is finished - if ! kill -0 "$pid" 2> /dev/null; then - # get return code - if wait "$pid" 2> /dev/null; then - # success - log " - \"${title}\" finished successfully" - unset PID_LIST[$pid] - else - # failed - log " - Process \"${title}\" FAILED" - WAIT_BG_RETURN=1 - unset PID_LIST[$pid] - fi - - fi - done - - if [ "${#PID_LIST[@]}" -eq 0 ]; then - # check if any subprocess failed - if [ "$WAIT_BG_RETURN" -ne 0 ]; then - logError "One or more child processes exited with failure!" - - logOutputFromBackgroundProcesses - - exitError 1 - fi - - break - fi - - sleep 3 - - done - - if [[ "$ALWAYS_SHOW_LOGS" -eq 1 ]]; then - logOutputFromBackgroundProcesses - fi - - initPidList -} - -logOutputFromBackgroundProcesses() { - ## check if pidlist exists - if [[ -z "${PID_LOG_FILE[@]:+${PID_LOG_FILE[@]}}" ]]; then - return - fi - - echo "" - echo "Showing logs:" - - for pid in "${!PID_LOG_FILE[@]}"; do - title="${PID_LOG_TITLE[$pid]}" - log="${PID_LOG_FILE[$pid]}" - - if [[ -s "$log" ]]; then - echo "-- begin of \"$title\" log --:" - - cat "$log" - rm -f -- "$log" - - echo "-- end of \"$title\" log --" - - echo "" - echo "" - fi - - unset PID_LOG_TITLE[$pid] - unset PID_LOG_FILE[$pid] - - done -} - -################################################# -# Timing handling -################################################# - -timerStart() { - TIMER_START="$(date +%s)" -} - -timerStep() { - local TIMER_NOW="$(date +%s)" - local TIMER_DIFF="$(expr ${TIMER_NOW} - ${TIMER_START})" - - if [ "$TIMER_DIFF" -lt 60 ]; then - echo "${TIMER_DIFF} seconds" - elif [ "$TIMER_DIFF" -lt 3600 ]; then - echo "$(expr ${TIMER_DIFF} / 60) minutes" - else - echo "$(expr ${TIMER_DIFF} / 60 / 60) hours" - fi -} - -################################################# -# Docker -################################################# - -### - # Push image - # - # $1 -> docker container name (eg. webdevops/php) - # $2 -> docker container tag (eg. ubuntu-14.04) - ## -function dockerPushImage() { - CONTAINER_NAME="$1" - CONTAINER_TAG="$2" - - docker push "${CONTAINER_NAME}:${CONTAINER_TAG}" -} - -function checkBlacklist() { - if [[ -s "${BASE_DIR}/BLACKLIST" ]]; then - echo "$*" | grep -vF "$(cat "${BASE_DIR}/BLACKLIST")" - else - echo "$*" - fi -} - -### - # Push image - # - # $1 -> base path - # $2 -> docker container tag (eg. ubuntu-14.04) - # $3 -> filter - ## -function foreachDockerfileInPath() { - DOCKER_BASE_PATH="$1" - CALLBACK="$2" - FILTER=".*" - - if [ "$#" -ge 3 ]; then - FILTER="$3" - fi - - if [ -n "${WHITELIST}" ]; then - for WTAG in $WHITELIST; do - if [ "${FILTER}" = ".*" ]; then - FILTER="${WTAG}" - else - FILTER="${FILTER}\|${WTAG}" - fi - done - fi - - # build each subfolder as tag - for DOCKERFILE_PATH in $(find ${DOCKER_BASE_PATH} -mindepth 1 -maxdepth 1 -type d -regex ".*\(${FILTER}\).*"); do - # check if there is a Dockerfile - - if [ -f "${DOCKERFILE_PATH}/Dockerfile" ]; then - DOCKERFILE="${DOCKERFILE_PATH}/Dockerfile" - TAGNAME=$(basename "${DOCKERFILE_PATH}") - - if [[ -n "$(checkBlacklist "${BASENAME}:${TAGNAME}")" ]]; then - ${CALLBACK} - fi - fi - done -} diff --git a/bin/provision.sh b/bin/provision.sh deleted file mode 100755 index e74bff962..000000000 --- a/bin/provision.sh +++ /dev/null @@ -1,442 +0,0 @@ -#!/usr/bin/env bash - -if [[ -n "$1" ]]; then - BUILD_TARGET="$1" -else - BUILD_TARGET="all" -fi - -if [[ "$BUILD_MODE" == "push" ]]; then - # skip provision on push - exit 0 -fi - -if [[ "$BASELAYOUT" -eq 1 ]]; then - BASELAYOUT=1 -else - BASELAYOUT=0 -fi - -if [[ "$PROVISION" -eq 1 ]]; then - PROVISION=1 -else - PROVISION=0 -fi - -set -o pipefail # trace ERR through pipes -set -o errtrace # trace ERR through 'time command' and other functions -set -o nounset ## set -u : exit the script if you try to use an uninitialised variable -set -o errexit ## set -e : exit the script if any statement returns a non-true return value - -# Fix readlink issue on macos - -READLINK='readlink' -TAR='tar' - -[[ `uname` == 'Darwin' ]] && { - which greadlink > /dev/null && { - READLINK='greadlink' - } || { - echo 'ERROR: GNU utils required for Mac. You may use homebrew to install them: brew install coreutils gnu-sed' - exit 1 - } - - which gtar > /dev/null && { - TAR='gtar' - } || { - echo 'ERROR: GNU tar required for Mac. You may use homebrew to install them: brew install gnu-tar' - exit 1 - } -} - -SCRIPT_DIR=$(dirname "$($READLINK -f "$0")") -BASE_DIR=$(dirname "$SCRIPT_DIR") - -source "$SCRIPT_DIR/functions.sh" - -BASELAYOUT_DIR="${BASE_DIR}/baselayout" -PROVISION_DIR="${BASE_DIR}/provisioning" -MACRO_DIR="${BASE_DIR}/macro" -DOCKER_DIR="${BASE_DIR}/docker" - - -### - # Relative dir - # - # $1 -> absolute path - # stdout -> relative path (to current base dir) - # - ## -function relativeDir() { - echo ${1#${BASE_DIR}/} -} - -### - # Relative dir - # - # $1 -> build target (eg. "bootstrap", "base", "php" ...) - # stdout -> "1" if target is matched - # - ## -function checkBuildTarget() { - if [ "$BUILD_TARGET" == "all" -o "$BUILD_TARGET" == "$1" ]; then - echo 1 - fi -} - -### - # Generate list of directories - # - # $1 -> Directory - # - ## -function listDirectories() { - find "$1" -maxdepth 1 -type d -} - -### - # Generate list of directories with iname filter - # - # $1 -> Directory - # s2 -> Filter (find iname) - # - ## -function listDirectoriesWithFilter() { - find "$1" -maxdepth 1 -type d -iname "$2" -} - -####################################### -# Localscripts -####################################### - -### - # Build localscripts - # - # Build tar file from _localscripts for bootstrap containers - # - ## -function buildBaselayout() { - if [[ "$BASELAYOUT" -eq 1 ]]; then - echo " * Building localscripts" - - cd "${BASELAYOUT_DIR}" - rm -f baselayout.tar - $TAR -jc --owner=0 --group=0 -f baselayout.tar * - fi -} - -### - # Deploy localscripts - # - # Copy tar to various containers - # - ## -function deployBaselayout() { - if [[ "$BASELAYOUT" -eq 1 ]]; then - DOCKER_CONTAINER="$1" - DOCKER_FILTER="$2" - - listDirectoriesWithFilter "${DOCKER_DIR}/${DOCKER_CONTAINER}" "${DOCKER_FILTER}" | while read DOCKER_DIR; do - if [ -f "${DOCKER_DIR}/Dockerfile" ]; then - echo " - $(relativeDir $DOCKER_DIR)" - cp baselayout.tar "${DOCKER_DIR}/baselayout.tar" - fi - done - fi -} - -####################################### -# Configuration -####################################### - -### - # Clear configuration - # - # Clear conf/ directory of each docker container - # - # $1 -> container name (eg. php) - # $2 -> sub directory filter (eg. "*" for all or "ubuntu-*" for only ubuntu containers) - # - ## -function clearConfiguration() { - if [[ "$PROVISION" -eq 1 ]]; then - DOCKER_CONTAINER="$1" - DOCKER_FILTER="$2" - - echo " -> Clearing configuration" - listDirectoriesWithFilter "${DOCKER_DIR}/${DOCKER_CONTAINER}" "${DOCKER_FILTER}" | while read DOCKER_DIR; do - if [ -f "${DOCKER_DIR}/Dockerfile" ]; then - echo " - $(relativeDir $DOCKER_DIR)" - rm -rf "${DOCKER_DIR}/conf/" - fi - done - fi -} - -### - # Deploy configuration - # - # Deploy conf/ directory into each docker container - # - # $1 -> configuration directory from _provisioning (eg. php/general) - # $2 -> container name (eg. php) - # $3 -> sub directory filter (eg. "*" for all or "ubuntu-*" for only ubuntu containers) - # - ## -function deployConfiguration() { - if [[ "$PROVISION" -eq 1 ]]; then - PROVISION_SUB_DIR="$1" - DOCKER_CONTAINER="$2" - DOCKER_FILTER="$3" - - if [ "$DOCKER_FILTER" == "*" ]; then - echo " -> Deploying configuration" - else - echo " -> Deploying configuration with filter '$DOCKER_FILTER'" - fi - - listDirectoriesWithFilter "${DOCKER_DIR}/${DOCKER_CONTAINER}" "${DOCKER_FILTER}" | while read DOCKER_DIR; do - if [ -f "${DOCKER_DIR}/Dockerfile" ]; then - echo " - $(relativeDir $DOCKER_DIR)" - cp -f -r "${PROVISION_DIR}/${PROVISION_SUB_DIR}/." "${DOCKER_DIR}/conf/" - fi - done - fi -} - -### - # Header message - # - # $1 -> container name (eg. php) - ## -function header() { - echo "Building configuration for webdevops/$1" -} - -############################################################################### -# MAIN -############################################################################### - -## Build bootstrap -[[ $(checkBuildTarget bootstrap) ]] && { - header "bootstrap" - buildBaselayout - deployBaselayout bootstrap '*' - - # Samson - deployBaselayout samson-deployment '*' - - rm -f baselayout.tar -} - -## Build base -[[ $(checkBuildTarget base) ]] && { - header "base" - clearConfiguration base '*' - deployConfiguration base/general base '*' - deployConfiguration base/centos base 'centos-*' - deployConfiguration base/alpine base 'alpine-*' -} - -## Build base-app -[[ $(checkBuildTarget base-app) ]] && { - header "base-app" - clearConfiguration base-app '*' - deployConfiguration base-app/general base-app '*' -} - -## Build apache -[[ $(checkBuildTarget apache) ]] && { - header "apache" - clearConfiguration apache '*' - deployConfiguration apache/general apache '*' - deployConfiguration apache/centos apache 'centos-*' - deployConfiguration apache/alpine apache 'alpine-*' -} - -## Build apache-dev -[[ $(checkBuildTarget apache-dev) ]] && { - header "apache-dev" - clearConfiguration apache-dev '*' - deployConfiguration apache-dev/general apache-dev '*' -} - -## Build nginx -[[ $(checkBuildTarget nginx) ]] && { - header "nginx" - clearConfiguration nginx '*' - deployConfiguration nginx/general nginx '*' - deployConfiguration nginx/centos nginx 'centos-*' - deployConfiguration nginx/alpine nginx 'alpine-*' -} - -## Build nginx-dev -[[ $(checkBuildTarget nginx) ]] && { - header "nginx-dev" - clearConfiguration nginx-dev '*' - deployConfiguration nginx-dev/general nginx-dev '*' -} - -## Build hhvm -[[ $(checkBuildTarget hhvm) ]] && { - header "hhvm" - clearConfiguration hhvm '*' - deployConfiguration hhvm/general hhvm '*' -} - -## Build hhvm-apache -[[ $(checkBuildTarget hhvm-apache) ]] && { - header "hhvm-apache" - clearConfiguration hhvm-apache '*' - deployConfiguration apache/general hhvm-apache '*' - deployConfiguration hhvm-apache/general hhvm-apache '*' -} - -## Build hhvm-nginx -[[ $(checkBuildTarget hhvm-nginx) ]] && { - header "hhvm-nginx" - clearConfiguration hhvm-nginx '*' - deployConfiguration nginx/general hhvm-nginx '*' - deployConfiguration nginx/centos hhvm-nginx 'centos-*' - deployConfiguration hhvm-nginx/general hhvm-nginx '*' -} - -## Build php -[[ $(checkBuildTarget php) ]] && { - header "php" - clearConfiguration php '*' - deployConfiguration php/general php '*' - deployConfiguration php/ubuntu-12.04 php 'ubuntu-12.04' - deployConfiguration php/alpine php 'alpine-*' - - # deploy php7 configuration to *-php7 containers - deployConfiguration php/php7 php '*-php7' - deployConfiguration php/php7 php 'debian-9' - deployConfiguration php/php7 php 'ubuntu-16.04' -} - -## Build php-apache -[[ $(checkBuildTarget php-apache) ]] && { - header "php-apache" - clearConfiguration php-apache '*' - deployConfiguration apache/general php-apache '*' - deployConfiguration apache/centos php-apache 'centos-*' - deployConfiguration apache/alpine php-apache 'alpine-*' - deployConfiguration php-apache/general php-apache '*' -} - -## Build php-nginx -[[ $(checkBuildTarget php-nginx) ]] && { - header "php-nginx" - clearConfiguration php-nginx '*' - deployConfiguration nginx/general php-nginx '*' - deployConfiguration nginx/centos php-nginx 'centos-*' - deployConfiguration nginx/alpine php-nginx 'alpine-*' - deployConfiguration php-nginx/general php-nginx '*' -} - - -## Build php-dev -[[ $(checkBuildTarget php-dev) ]] && { - header "php-dev" - clearConfiguration php-dev '*' - deployConfiguration php-dev/general php-dev '*' -} - -## Build php-apache-dev -[[ $(checkBuildTarget php-apache-dev) ]] && { - header "php-apache-dev" - clearConfiguration php-apache-dev '*' - deployConfiguration apache/general php-apache-dev '*' - deployConfiguration apache/centos php-apache-dev 'centos-*' - deployConfiguration apache/alpine php-apache-dev 'alpine-*' - deployConfiguration php-apache/general php-apache-dev '*' - deployConfiguration php-dev/general php-apache-dev '*' - deployConfiguration apache-dev/general php-apache-dev '*' -} - -## Build php-nginx-dev -[[ $(checkBuildTarget php-nginx-dev) ]] && { - header "php-nginx-dev" - clearConfiguration php-nginx-dev '*' - deployConfiguration nginx/general php-nginx-dev '*' - deployConfiguration nginx/centos php-nginx-dev 'centos-*' - deployConfiguration nginx/alpine php-nginx-dev 'alpine-*' - deployConfiguration php-nginx/general php-nginx-dev '*' - deployConfiguration php-dev/general php-nginx-dev '*' - deployConfiguration nginx-dev/general php-nginx-dev '*' -} - -## Build postfix -[[ $(checkBuildTarget postfix) ]] && { - header "postfix" - clearConfiguration postfix '*' - deployConfiguration postfix/general postfix '*' -} - -## Build mail-sandbox -[[ $(checkBuildTarget mail-sandbox) ]] && { - header "mail-sandbox" - clearConfiguration mail-sandbox '*' - deployConfiguration mail-sandbox/general mail-sandbox '*' -} - -## Build vsftp -[[ $(checkBuildTarget vsftp) ]] && { - header "vsftp" - clearConfiguration vsftp '*' - deployConfiguration vsftp/general vsftp '*' -} - -## Build typo3 -[[ $(checkBuildTarget typo3) ]] && { - header "typo3" - clearConfiguration typo3 '*' - deployConfiguration typo3/general typo3 '*' -} - -## Build piwik -[[ $(checkBuildTarget piwik) ]] && { - header "piwik" - clearConfiguration piwik '*' - deployConfiguration piwik/general piwik '*' -} - -## Build varnish -[[ $(checkBuildTarget varnish) ]] && { - header "varnish" - clearConfiguration varnish '*' - deployConfiguration varnish/general varnish '*' -} - -## Build samson-deployment -[[ $(checkBuildTarget samson-deployment) ]] && { - header "samson-deployment" - - # Bootstrap - buildBaselayout - deployBaselayout samson-deployment '*' - rm -f baselayout.tar - - # Base - deployConfiguration base/general samson-deployment 'latest' - deployConfiguration base-app/general samson-deployment 'latest' - - # Samson deployment - deployConfiguration samson-deployment/general samson-deployment 'latest' -} - -## Build cerbot -[[ $(checkBuildTarget certbot) ]] && { - header "certbot" -} - -## Build liquibase -[[ $(checkBuildTarget liquibase) ]] && { - header "liquibase" - clearConfiguration liquibase '*' - deployConfiguration liquibase/general liquibase '*' -} - - -exit 0 diff --git a/bin/retry.sh b/bin/retry.sh deleted file mode 100755 index 13da5ea98..000000000 --- a/bin/retry.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "$RETRY" ]; then - RETRY=1 -fi - -if [ -z "$RETRY_DELAY" ]; then - RETRY_DELAY="1m" -fi - -RETURN_CODE=0 - -MKTEMP='mktemp' - -[[ `uname` == 'Darwin' ]] && { - which greadlink > /dev/null && { - MKTEMP='gmktemp' - } || { - echo 'ERROR: GNU utils required for Mac. You may use homebrew to install them: brew install coreutils' - exit 1 - } -} - - -if [[ "$RETRY" -le 1 ]]; then - exec "$@" -fi - - -LOGFILE="$($MKTEMP --tmpdir retry.XXXXXXXXXX)" - -retry() { - local n=0 - - until [[ "$n" -ge "$RETRY" ]]; do - # Reset logfile for this try - echo > "$LOGFILE" - - RETURN_CODE="0" - "$@" && break || { - ((n++)) - echo "" - echo " [WARNING] Command failed. Retry now ... $n/$RETRY:" - echo "" - echo "" - RETURN_CODE=1 - sleep "$RETRY_DELAY"; - } - done - - if [[ "$RETURN_CODE" -ne 0 ]]; then - echo " [ERROR] The command has failed after $n attempts." - fi -} - -retry "$@" &> "$LOGFILE" - -if [[ "$RETURN_CODE" -ne 0 ]]; then - cat "$LOGFILE" -fi -rm -f "$LOGFILE" - -exit "$RETURN_CODE" diff --git a/bin/webdevops/Command.py b/bin/webdevops/Command.py new file mode 100644 index 000000000..3b179aefd --- /dev/null +++ b/bin/webdevops/Command.py @@ -0,0 +1,68 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os, subprocess, tempfile + +def execute(cmd, cwd=False, env=None): + """ + Execute cmd and output stdout/stderr + """ + + print 'Execute: %s' % ' '.join(cmd) + + # remove _ from env (prevent errors) + if env is not None and '_' in env: + del env['_'] + + # set current working directory + path_current = os.getcwd() + if cwd: + os.chdir(cwd) + + # stdout file + # (stdout and stderr will be redirected to it, pieping both isn't possible) + file_stdout = tempfile.NamedTemporaryFile() + + # create subprocess + proc = subprocess.Popen( + cmd, + stdout=file_stdout, + stderr=file_stdout, + bufsize=-1, + env=env + ) + + # wait for process end + while proc.poll() is None: + pass + + # output stdout + with open(file_stdout.name, 'r') as f: + for line in f: + print line.rstrip('\n') + + # restore current work directory + os.chdir(path_current) + + if proc.returncode == 0: + return True + else: + print '>> failed command with return code %s' % proc.returncode + return False diff --git a/bin/webdevops/Configuration.py b/bin/webdevops/Configuration.py new file mode 100644 index 000000000..7306727b4 --- /dev/null +++ b/bin/webdevops/Configuration.py @@ -0,0 +1,184 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import re +from webdevops.doit.DoitReporter import DoitReporter +from webdevops.docker.DockerCliClient import DockerCliClient + +default = { + 'dockerPath': False, + 'templatePath': False, + 'provisionPath': False, + 'imagePath': False, + 'confPath': False, + 'baselayoutPath': False, + 'testPath': False, + + 'dockerClient': DockerCliClient(), + + 'blacklistFile': False, + + 'doitConfig': { + 'GLOBAL': { + 'reporter': DoitReporter, + } + }, + + 'docker': { + 'imagePrefix': '', + 'autoLatestTag': False, + 'fromRegExp': re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE), + 'pathRegex': False, + 'autoPull': False, + 'autoPullWhitelist': False, + 'autoPullBlacklist': False, + }, + + 'dockerTest': { + 'toolImages': {}, + 'serverspec': { + }, + + 'env': {}, + }, + + 'dockerBuild': { + 'noCache': False, + }, + + 'dockerPush': { + }, + + 'filter': { + 'whitelist': False, + 'blacklist': False, + }, + + 'verbosity': 0, + 'threads': 1, + 'retry': 5, + 'dryRun': False, +} + +def merge(configuration): + def dictmerge(original, update): + """ + Recursively update a dict. + Subdict's won't be overwritten but also updated. + """ + for key, value in original.iteritems(): + if key not in update: + update[key] = value + elif isinstance(value, dict): + dictmerge(value, update[key]) + return update + + return dictmerge(default, configuration) + + +class dotdictify(dict): + def __init__(self, value=None): + if value is None: + pass + elif isinstance(value, dict): + for key in value: + self.__setitem_internal__(key, value[key]) + else: + raise TypeError, 'expected dict' + + def __setitem_internal__(self, key, value): + """ + Set dict as raw value (preserv key with dots) + """ + if isinstance(value, dict) and not isinstance(value, dotdictify): + value = dotdictify(value) + dict.__setitem__(self, key, value) + + def __setitem__(self, key, value): + if key is not None and '.' in key: + myKey, restOfKey = key.split('.', 1) + target = self.setdefault(myKey, dotdictify()) + if not isinstance(target, dotdictify): + raise KeyError, 'cannot set "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target)) + target[restOfKey] = value + else: + if isinstance(value, dict) and not isinstance(value, dotdictify): + value = dotdictify(value) + dict.__setitem__(self, key, value) + + def __getitem__(self, key, raw=False): + if key is None or '.' not in key or raw: + return dict.get(self, key, None) + myKey, restOfKey = key.split('.', 1) + target = dict.get(self, myKey, None) + if not isinstance(target, dotdictify): + raise KeyError, 'cannot get "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target)) + return target[restOfKey] + + def __contains__(self, key): + """ + Check if element is contained in tree + """ + if key is None or '.' not in key: + return dict.__contains__(self, key) + myKey, restOfKey = key.split('.', 1) + if not dict.__contains__(self, myKey): + return False + target = dict.__getitem__(self, myKey) + if not isinstance(target, dotdictify): + return False + return restOfKey in target + + def setdefault(self, key, default): + """ + Set default value by using dotted notation + """ + if key not in self: + self[key] = default + return self[key] + + def to_dict(self): + """ + Convert to dict + :return: dict + """ + ret = {} + for key in self: + if key is not None: + ret[key] = self.__getitem__(key, True) + return ret + + def get(self, k, d=None): + """ + Get element by using dotted notation + """ + if dotdictify.__contains__(self, k): + return dotdictify.__getitem__(self, k) + return d + + def set(self, key, value): + """ + Set value by using dotted notation + """ + self[key] = value + return self[key] + + __setattr__ = __setitem__ + __getattr__ = __getitem__ diff --git a/bin/webdevops/Dockerfile.py b/bin/webdevops/Dockerfile.py new file mode 100644 index 000000000..8955a0597 --- /dev/null +++ b/bin/webdevops/Dockerfile.py @@ -0,0 +1,116 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import re + + +def finder(dockerfile_path, filename="Dockerfile", filter=[]): + """ + Search all file un dockerfile_path with filename ends with "filename" + And match filter + + :param dockerfile_path: path where to search file + :type dockerfile_path: str + + :param filename: pattern which the file must be validate + :type filename: str + + :param filter: list of term must be match in path + :type filter: list + + :return: list of path + :rtype: list + """ + dockerfile_stack = [] + filter_regex = re.compile(ur'.*(%s).*' % "|".join(filter), re.IGNORECASE) + # pprint(filter_regex.pattern) + for root, dirs, files in os.walk(dockerfile_path): + for file in files: + if file.endswith(filename): + if filter_regex.match(root): + dockerfile_stack.append(os.path.join(root, file)) + return dockerfile_stack + + +def find_by_image(dockerfile_path, filename="Dockerfile", image=[]): + """ + Similar that finder but adds a constraint of search on the name of the image docker + + :param dockerfile_path: path where to search file + :type dockerfile_path: str + + :param filename: pattern which the file must be validate + :type filename: str + + :param image: list of term must be match in path + :type image: list + + :return: list of path + :rtype: list + """ + updated_filter = [] + for i, v in enumerate(image): + updated_filter.append("/docker/(%s)/" % v) + return finder(dockerfile_path, filename, updated_filter) + + +def find_by_image_and_tag(dockerfile_path, image, tag): + """ + + :param dockerfile_path: + :type dockerfile_path: str + + :param image: + :type image: str + + :param tag: + :type tag: str + + :return: + :rtype: list + """ + if "*" == tag: + filter = ["/docker/%s/" % image] + else: + filter = ["/docker/%s/%s" % (image, tag.replace('*', '.+'))] + return finder(dockerfile_path, "Dockerfile", filter) + + +def find_by_tags(dockerfile_path, filename="Dockerfile", tags=[]): + """ + Similar that finder but adds a constraint of search on the tag of the image docker + + :param dockerfile_path: path where to search file + :type dockerfile_path: str + + :param filename: pattern which the file must be validate + :type filename: str + + :param tags: list of tags must be match in path + :type tags: list + + :return: list of path + :rtype: list + """ + updated_filter = [] + for i, v in enumerate(tags): + updated_filter.append("/docker/[^/]+/%s" % v) + return finder(dockerfile_path, filename, updated_filter) diff --git a/bin/webdevops/Dockerfile.pyc b/bin/webdevops/Dockerfile.pyc deleted file mode 100644 index 5f013ea4a..000000000 Binary files a/bin/webdevops/Dockerfile.pyc and /dev/null differ diff --git a/bin/webdevops/DockerfileUtility.py b/bin/webdevops/DockerfileUtility.py new file mode 100644 index 000000000..23c3fa5cb --- /dev/null +++ b/bin/webdevops/DockerfileUtility.py @@ -0,0 +1,213 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import re + +DOCKERFILE_STATEMENT_FROM_RE = re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE) + +def find_file_in_path(dockerfile_path, filename="Dockerfile", whitelist=False, blacklist=False): + """ + Search all file un dockerfile_path with filename ends with "filename" + And match filter + + :param dockerfile_path: path where to search file + :type dockerfile_path: str + + :param filename: pattern which the file must be validate + :type filename: str + + :param whitelist: list of term must be match in path + :type whitelist: list + + :param blacklist: list of term must not be match in path + :type blacklist: list + + :return: list of path + :rtype: list + """ + file_list = [] + + # build list of files + for root, dirs, files in os.walk(dockerfile_path): + for file in files: + if file.endswith(filename): + file_list.append(os.path.join(root, file)) + + # filter by whitelist + if whitelist: + for term in whitelist: + file_list = filter(lambda x: term in x, file_list) + + if blacklist: + for term in blacklist: + file_list = filter(lambda x: term not in x, file_list) + + return file_list + +def find_dockerfiles_in_path(base_path, path_regex, image_prefix, whitelist=False, blacklist=False): + """ + Find all Dockerfiles in path (and even in symlinks and build dependencies) + """ + + def parse_docker_info_from_path(path): + image_name_info = ([m.groupdict() for m in path_regex.finditer(os.path.abspath(path))])[0] + + image_repository = (image_name_info['repository'] if 'repository' in image_name_info else '') + image_name = (image_name_info['image'] if 'image' in image_name_info else '') + image_tag = (image_name_info['tag'] if 'tag' in image_name_info else '') + + # check if path is linked + if os.path.islink(os.path.dirname(path)): + linked_image_name_info = ([m.groupdict() for m in path_regex.finditer(os.path.realpath(path))])[0] + + linked_image_repository = (linked_image_name_info['repository'] if 'repository' in linked_image_name_info else '') + linked_image_name = (linked_image_name_info['image'] if 'image' in linked_image_name_info else '') + linked_image_tag = (linked_image_name_info['tag'] if 'tag' in linked_image_name_info else '') + + image_from = image_prefix + linked_image_repository + '/' + linked_image_name + ':' + linked_image_tag + else: + image_from = parse_dockerfile_from_statement(path) + + imageInfo = { + 'fullname': image_prefix + image_repository + '/' + image_name + ':' + image_tag, + 'name': image_prefix + image_repository + '/' + image_name, + 'tag': image_tag, + 'repository': image_prefix + image_repository, + 'imageName': image_name, + 'from': image_from + } + return imageInfo + + ret = [] + for path in find_dockerfile_in_path_recursive(base_path): + base_path = os.path.dirname(path) + if os.path.isfile(path) and os.path.basename(path) == 'Dockerfile': + dockerfile = { + 'path': path, + 'basePath': base_path, + 'abspath': os.path.abspath(path), + 'image': parse_docker_info_from_path(path), + } + ret.append(dockerfile) + + if whitelist or blacklist: + ret = filter_dockerfile( + dockerfile_list=ret, + whitelist=whitelist, + blacklist = blacklist + ) + + return ret + + +def filter_dockerfile(dockerfile_list, whitelist=False, blacklist=False): + """ + Filter Dockerfiles by white- and blacklist + """ + if whitelist: + for term in whitelist: + dockerfile_list = filter(lambda x: term in x['image']['fullname'], dockerfile_list) + + if blacklist: + for term in blacklist: + dockerfile_list = filter(lambda x: term not in x['image']['fullname'], dockerfile_list) + + return dockerfile_list + + +def find_dockerfile_in_path_recursive(basePath): + """ + Find all Dockerfiles paths recursive in path + """ + + ret = [] + for root, subFolders, files in os.walk(basePath, followlinks=True): + for file in files: + if os.path.basename(file) == 'Dockerfile': + ret.append(os.path.join(root, file)) + return ret + + +def parse_dockerfile_from_statement(path): + """ + Extract docker image name from FROM statement + """ + with open(path, 'r') as fileInput: + DockerfileContent = fileInput.read() + data = \ + ([m.groupdict() for m in DOCKERFILE_STATEMENT_FROM_RE.finditer(DockerfileContent)])[0] + ret = data['image'] + + if data['tag']: + ret += ':%s' % data['tag'] + return ret + + +def generate_image_name_with_tag_latest(image_name): + """ + Prepare dockerfile list with dependency and also add "auto latest tag" images + """ + if re.search(':[^:]+$', image_name): + ret = re.sub('(:[^:]+)$', ':latest', image_name) + else: + ret = '%s:latest' % image_name + return ret + +def image_basename(image_name): + """ + Get image name without tag + """ + if re.search(':[^:]+$', image_name): + image_name = re.sub('(:[^:]+)$', '', image_name) + return image_name + +def extract_image_name_tag(image_name): + """ + Get tag from image name + """ + if re.search('^(.*):', image_name): + ret = re.sub('^(.*):', '', image_name) + else: + ret = 'latest' + return ret + +def check_if_base_image_needs_pull(dockerfile, configuration): + ret = False + base_image = dockerfile['image']['from'] + + if configuration.get('docker.autoPull'): + if configuration.get('docker.autoPullWhitelist') and configuration.get('docker.autoPullWhitelist').search(base_image): + """ + Matched whitelist + """ + ret = True + else: + """ + No whitelist, we need to pull every image + """ + ret = True + + if configuration.get('docker.autoPullBlacklist') and configuration.get('docker.autoPullBlacklist').match(base_image): + """ + Matched blacklist + """ + ret = False + return ret diff --git a/bin/webdevops/Provisioner.py b/bin/webdevops/Provisioner.py new file mode 100644 index 000000000..3ce7825ff --- /dev/null +++ b/bin/webdevops/Provisioner.py @@ -0,0 +1,171 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +from cleo import Output +from webdevops import Dockerfile +import os +from distutils.dir_util import copy_tree, remove_tree +from threading import Thread +import Queue +import shutil + + +class Provisioner(Thread): + """ + Asked to provisioning an image + """ + + image_name = '' + + image_config = {} + + output = '' + + dockerfile = '' + + provision = '' + + queue = '' + + # def __init__(self, dockerfile, provision, image_name, image_config, output): + def __init__(self, dockerfile, provision, queue, output): + """ + Construct + + :param dockerfile: path to the folder containing dockerfile analyze + :type dockerfile: str + + :param queue: Stack of image config + :type queue: Queue.Queue + + :param output: stdout + :type output: Output + + """ + Thread.__init__(self) + self.dockerfile = dockerfile + self.provision = provision + self.queue = queue + self.output = output + + def __get_item(self): + """ Get a item in the Queue """ + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line('Looking for the next item') + item = self.queue.get(True, 5.0) + self.image_name = item['image_name'] + self.image_config = item['image_config'] + + def __done_item(self): + """ Set current item of the Queue to done """ + self.queue.task_done() + self.image_name = '' + self.image_config = {} + + def run(self): + while True: + try: + self.__get_item() + if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): + self.line( + 'Building configuration for webdevops/%s' % self.image_name + ) + self.__clear_configuration() + if 'baselayout' in self.image_config: + self.__deploy_base_layout() + if 'configuration' in self.image_config: + self.__deploy_configuration() + self.__done_item() + except Queue.Empty: + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line("End") + break + + def __clear_configuration(self): + """ + Remove the old configuration + """ + dockerfiles = Dockerfile.find_by_image_and_tag(self.dockerfile, self.image_name, '*') + dockerfiles = [os.path.dirname(image_path) for image_path in dockerfiles] + for dest in dockerfiles: + dest = os.path.abspath(os.path.join(dest, 'conf/')) + if os.path.exists(dest): + if Output.VERBOSITY_VERY_VERBOSE <= self.output.get_verbosity(): + self.line('delete configuration : %s' % dest) + remove_tree(dest, 0) + + def __deploy_configuration(self): + """ + Deploy the configuration to the container + """ + for src, tag in self.image_config['configuration'].iteritems(): + if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): + self.line("%s => %s:%s" % (src, self.image_name, tag)) + if isinstance(tag, list): + dockerfiles = [] + for t in tag: + dockerfiles.extend(Dockerfile.find_by_image_and_tag(self.dockerfile, self.image_name, t)) + else: + dockerfiles = Dockerfile.find_by_image_and_tag(self.dockerfile, self.image_name, tag) + dockerfiles = [os.path.dirname(image_path) for image_path in dockerfiles] + self.__copy_configuration(dockerfiles, src) + + def __deploy_base_layout(self): + """ + Deploy localscripts + + copy tar to various containers + """ + if os.path.exists('baselayout.tar') and self.image_config['baselayout']: + dockerfiles = Dockerfile.find_by_image(self.dockerfile, "Dockerfile", [self.image_name]) + dockerfiles = [os.path.dirname(image_path) for image_path in dockerfiles] + for target_path in dockerfiles: + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line('copy baselayout to %s ' % target_path) + shutil.copy2('baselayout.tar', target_path) + + def __copy_configuration(self, dockerfiles, src): + """ + Copy the different configs to the container + + :param dockerfiles: List of path's images to provisioning + :type dockerfiles: list + + :param src: sub-path of the provisioning directory + :type src: str + """ + src = os.path.abspath(os.path.join(self.provision, src)) + "/." + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line('src : %s' % src) + for dest in dockerfiles: + dest = os.path.abspath(os.path.join(dest, 'conf/')) + if not os.path.exists(dest): + if Output.VERBOSITY_DEBUG <= self.output.get_verbosity(): + self.line('create : %s' % dest) + os.mkdir(dest) + if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): + self.line('dest : %s' % dest) + copy_tree(src, dest, 1, 1, 0, 0, 0) + + def line(self, msg): + self.output.writeln( + '({0:^8}) [{1:.<20}] {2}'.format(self.name, self.image_name, msg) + ) diff --git a/bin/webdevops/Provisioner.pyc b/bin/webdevops/Provisioner.pyc deleted file mode 100644 index 7c4e0353b..000000000 Binary files a/bin/webdevops/Provisioner.pyc and /dev/null differ diff --git a/bin/webdevops/__init__.py b/bin/webdevops/__init__.py new file mode 100644 index 000000000..0688bc71a --- /dev/null +++ b/bin/webdevops/__init__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from Provisioner import Provisioner + +__all__ = [ + 'Provisioner', +] diff --git a/bin/webdevops/__init__.pyc b/bin/webdevops/__init__.pyc deleted file mode 100644 index 13956d8bf..000000000 Binary files a/bin/webdevops/__init__.pyc and /dev/null differ diff --git a/bin/webdevops/command/BaseCommand.py b/bin/webdevops/command/BaseCommand.py new file mode 100644 index 000000000..809618d61 --- /dev/null +++ b/bin/webdevops/command/BaseCommand.py @@ -0,0 +1,256 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os, sys, re, traceback +import time, datetime +import multiprocessing +from cleo import Command +from webdevops import Configuration + +class BaseCommand(Command): + configuration = False + + time_startup = False + time_finish = False + + def __init__(self, configuration): + """ + Constructor + """ + Command.__init__(self) + self.configuration = configuration + + def handle(self): + """ + Main command method which will be called by Cleo + """ + self.build_configuration() + + self.startup() + + try: + exitcode = self.run_task(configuration=self.configuration) + except KeyboardInterrupt as e: + print ' !!! Execution aborted by user' + exitcode = 1 + except SystemExit as e: + print ' !!! Execution aborted by SystemExit' + print '' + traceback.print_exc(file=sys.stdout) + exitcode = 1 + + if exitcode == True or exitcode == 0 or exitcode == '' or exitcode is None: + exitcode = 0 + elif exitcode == False: + exitcode = 255 + + self.shutdown(exitcode=exitcode) + sys.exit(exitcode) + + def run_task(self, configuration): + """ + Run task + """ + return + + def startup(self): + """ + Show startup message + """ + self.time_startup = time.time() + + options = [] + + if 'threads' in self.configuration: + options.append('%s threads' % self.configuration.get('threads')) + + if 'retry' in self.configuration: + options.append('%s retries' % self.configuration.get('retry', 1)) + + if 'dryRun' in self.configuration and self.configuration.get('dryRun'): + options.append('dry-run') + + print 'Executing %s (%s)' % (self.name, ', '.join(options)) + print '' + + if self.output.is_verbose(): + try: + whitelist = self.get_whitelist() + if whitelist: + print 'WHITELIST active:' + for item in whitelist: + print ' - %s' % item + print '' + except: + pass + + try: + blacklist = self.get_blacklist() + if blacklist: + print 'BLACKLIST active:' + for item in blacklist: + print ' - %s' % item + print '' + except: + pass + def teardown(self,exitcode): + pass + + def shutdown(self, exitcode=0): + """ + Show shutdown message + """ + + self.time_finish = time.time() + + duration = self.time_finish - self.time_startup + duration = str(datetime.timedelta(seconds=int(duration))) + + self.teardown(exitcode) + + print '' + if exitcode == 0: + print '> finished execution in %s successfully' % (duration) + else: + print '> finished execution in %s with errors (exitcode %s)' % (duration, exitcode) + + def build_configuration(self): + """ + Get configuration + """ + configuration = self.configuration + + # threads + try: + configuration.set('threads', self.get_threads()) + except (Exception): + configuration.set('threads', 1) + + # whitelist + try: + configuration.set('whitelist', self.get_whitelist()) + except (Exception): + pass + + # blacklist + try: + configuration.set('blacklist', self.get_blacklist()) + except (Exception): + pass + + # dryrun + try: + configuration.set('dryRun', self.get_dry_run()) + except (Exception): + pass + + # retry + try: + configuration.set('retry', self.get_retry()) + except (Exception): + configuration.set('retry', 1) + + # verbosity + if self.output.is_verbose(): + configuration.set('verbosity', 2) + + self.configuration = configuration + + def get_configuration(self): + """ + Get configuration + """ + return self.configuration + + def get_whitelist(self): + """ + Get whitelist + """ + return list(self.option('whitelist')) + + def get_blacklist(self): + """ + Get blacklist + """ + ret = list(self.option('blacklist')) + + # static BLACKLIST file + if os.path.isfile(self.configuration.get('blacklistFile')): + lines = [line.rstrip('\n').lstrip('\n') for line in open(self.configuration.get('blacklistFile'))] + lines = filter(bool, lines) + + if lines: + ret.extend(lines) + + return ret + + def get_threads(self): + """ + Get processing thread count + """ + threads = os.getenv('THREADS', self.option('threads')) + + if threads == '0' or threads == '' or threads is None: + # use configuration value + threads = self.configuration.get('threads') + + match = re.match('auto(([-*+/])([0-9]+))?', str(threads)) + if match is not None: + ret = multiprocessing.cpu_count() + + if match.group(2) and match.group(3): + math_sign = match.group(2) + math_value = int(match.group(3)) + + if math_sign == "*": + ret = int(ret * math_value) + elif math_sign == "/": + ret = int(ret / math_value) + elif math_sign == "+": + ret = int(ret + math_value) + elif math_sign == "-": + ret = int(ret - math_value) + else: + ret = max(1, int(self.option('threads'))) + + return int(ret) + + def get_dry_run(self): + """ + Get if dry run is enabled + """ + return bool(self.option('dry-run')) + + def get_retry(self): + """ + Get number of retries + """ + default = 1 + retry = max(0, int(self.option('retry'))) + + if retry > 0: + # user value + return retry + elif 'retry' in self.configuration: + # configuration value + return self.configuration.get('retry') + else: + # defaults + return default diff --git a/bin/webdevops/command/DoitCommand.py b/bin/webdevops/command/DoitCommand.py new file mode 100644 index 000000000..a805bfb95 --- /dev/null +++ b/bin/webdevops/command/DoitCommand.py @@ -0,0 +1,39 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import sys +from .BaseCommand import BaseCommand +from doit.doit_cmd import DoitMain + +class DoitCommand(BaseCommand): + def run_doit(self, task_loader, configuration): + arguments = [] + extra_configuration = {} + + if 'threads' in configuration and configuration.get('threads') > 1: + arguments.extend(['-n', str(configuration.get('threads'))]) + + if 'doitConfig' in configuration: + extra_configuration = configuration.get('doitConfig') + + return DoitMain( + task_loader=task_loader, + extra_config=extra_configuration + ).run(arguments) diff --git a/bin/webdevops/command/__init__.py b/bin/webdevops/command/__init__.py new file mode 100644 index 000000000..9be4e15cb --- /dev/null +++ b/bin/webdevops/command/__init__.py @@ -0,0 +1,27 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from BaseCommand import BaseCommand +from DoitCommand import DoitCommand + +__all__ = [ + 'BaseCommand', + 'DoitCommand', +] diff --git a/bin/webdevops/docker/DockerBaseClient.py b/bin/webdevops/docker/DockerBaseClient.py new file mode 100644 index 000000000..b8b22cd9c --- /dev/null +++ b/bin/webdevops/docker/DockerBaseClient.py @@ -0,0 +1,47 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +class DockerBaseClient: + + def __init__(self): + """ + Constructor + """ + + def pull_image(self, name, tag): + """ + Build dockerfile + """ + print 'Pull image %s:%s' % (name, tag) + return True + + def build_dockerfile(self, path, name, nocache=False): + """ + Build dockerfile + """ + print 'Build Dockerfile %s with name %s' % (path, name) + return True + + def push_image(self, name): + """ + Push one Docker image to registry + """ + print 'Build image %s' % (name) + return True diff --git a/bin/webdevops/docker/DockerCliClient.py b/bin/webdevops/docker/DockerCliClient.py new file mode 100644 index 000000000..994245bae --- /dev/null +++ b/bin/webdevops/docker/DockerCliClient.py @@ -0,0 +1,53 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import subprocess +import os +import tempfile +from .DockerBaseClient import DockerBaseClient +from webdevops import Command + + +class DockerCliClient(DockerBaseClient): + + def pull_image(self, name, tag): + """ + Build dockerfile + """ + cmd = ['docker', 'pull', '%s:%s' % (name, tag)] + return Command.execute(cmd) + + def build_dockerfile(self, path, name, nocache=False): + """ + Build dockerfile + """ + cmd = ['docker', 'build', '--tag', name, os.path.dirname(path)] + + if nocache: + cmd.append('--no-cache') + + return Command.execute(cmd) + + def push_image(self, name): + """ + Push one Docker image to registry + """ + cmd = ['docker', 'push', name] + return Command.execute(cmd) diff --git a/bin/webdevops/docker/DockerPyClient.py b/bin/webdevops/docker/DockerPyClient.py new file mode 100644 index 000000000..70b52cf54 --- /dev/null +++ b/bin/webdevops/docker/DockerPyClient.py @@ -0,0 +1,115 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +import os +from .DockerBaseClient import DockerBaseClient + +class DockerPyClient(DockerBaseClient): + + docker_client = False + last_output_line = False + + def __init__(self, configuration): + """ + Constrcutor + """ + DockerBaseClient.__init__(self) + + import docker + + # Init docker client + self.docker_client = docker.from_env(assert_hostname=False) + + def pull_image(self, name, tag): + """ + Build dockerfile + """ + response = self.docker_client.pull( + repository=name, + tag=tag, + stream=True, + decode=True + ) + + return self.process_client_response(response) + + def build_dockerfile(self, path, name, nocache=False): + """ + Build dockerfile + """ + response = docker_client.build( + path=os.path.dirname(path), + tag=name, + pull=False, + nocache=nocache, + quiet=False, + decode=True + ) + + return self.process_client_response(response) + + def push_image(self, name): + """ + Push one Docker image to registry + """ + response = docker_client.push( + name, + stream=True, + decode=True + ) + return self.process_client_response(response) + + def process_client_response(self, response): + ret = True + last_message = False + + def output_message(message, prevent_repeat=False): + # Prevent repeating messages + if prevent_repeat: + if self.last_output_line and self.last_output_line == message: + return + self.last_output_line = message + else: + self.last_output_line = False + + sys.stdout.write(message.strip(' \t\n\r') + '\n') + + if not response: + return False + + for line in response: + # Keys + # - error + # - stream + # - status, progressDetail, id + # - progressDetail | aux [ tag, digest, size ] + if 'error' in line: + output_message(line['error']) + ret = False + if 'stream' in line: + output_message(line['stream'], prevent_repeat=True) + if 'status' in line: + message = line['status'] + if 'id' in line: + message += ' ' + line['id'] + output_message(message) + print '' + return ret diff --git a/bin/webdevops/docker/__init__.py b/bin/webdevops/docker/__init__.py new file mode 100644 index 000000000..9124d6a99 --- /dev/null +++ b/bin/webdevops/docker/__init__.py @@ -0,0 +1,29 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from DockerBaseClient import DockerBaseClient +from DockerPyClient import DockerPyClient +from DockerCliClient import DockerCliClient + +__all__ = [ + 'DockerBaseClient', + 'DockerPyClient', + 'DockerCliClient', +] diff --git a/bin/webdevops/doit/DoitReporter.py b/bin/webdevops/doit/DoitReporter.py new file mode 100644 index 000000000..cbb14d85b --- /dev/null +++ b/bin/webdevops/doit/DoitReporter.py @@ -0,0 +1,299 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import sys +import os +import time +import datetime +import StringIO +import termcolor +from termcolor import colored + +class TaskResult(object): + """ + result object used by DoitReporter + """ + # FIXME what about returned value from python-actions ? + def __init__(self, task): + self.task = task + self.result = None # fail, success, up-to-date, ignore + self.out = None # stdout from task + self.err = None # stderr from task + self.error = None # error from doit (exception traceback) + self.started = None # datetime when task execution started + self.elapsed = None # time (in secs) taken to execute task + self._started_on = None # timestamp + self._finished_on = None # timestamp + + def start(self): + """ + called when task starts its execution + """ + self._started_on = time.time() + + def set_result(self, result, error=None): + """ + called when task finishes its execution + """ + self._finished_on = time.time() + self.result = result + + self.out = ''.join([a.out for a in self.task.actions if a.out]) + self.err = ''.join([a.err for a in self.task.actions if a.err]) + self.error = error + + self.calculate_elapsed() + + def calculate_elapsed(self): + """ + calculate elapsed time + """ + if self._started_on is not None and self.elapsed is None: + started = datetime.datetime.utcfromtimestamp(self._started_on) + + if self._finished_on is None: + self._finished_on = time.time() + + self.started = str(started) + self.elapsed = self._finished_on - self._started_on + + def to_dict(self): + """ + convert result data to dictionary + """ + + self.calculate_elapsed() + + ret = { + 'name': self.task.name, + 'result': self.result, + 'out': self.out, + 'err': self.err, + 'error': self.error, + 'started': self.started, + 'elapsed': self.elapsed + } + return ret + + +class DoitReporter(object): + """ + output results after finish + """ + + desc = 'output after finish' + + show_out = False + show_err = False + + def __init__(self, outstream, options=None): #pylint: disable=W0613 + # result is sent to stdout when doit finishes running + self.t_results = {} + self.failures = [] + # when using this reporter output can not contain any other output + # than the data. so anything that is sent to stdout/err needs to + # be captured. + self._old_out = sys.stdout + sys.stdout = StringIO.StringIO() + self._old_err = sys.stderr + sys.stderr = StringIO.StringIO() + self.outstream = outstream + # runtime and cleanup errors + self.errors = [] + + self.show_out = options.get('show_out', True) + self.show_err = options.get('show_err', True) + + def get_status(self, task): + """ + called when task is selected (check if up-to-date) + """ + self.t_results[task.name] = TaskResult(task) + + def execute_task(self, task): + """ + called when excution starts + """ + self.t_results[task.name].start() + + def add_failure(self, task, exception): + """ + called when excution finishes with a failure + """ + self.t_results[task.name].set_result('fail', exception.get_msg()) + + if task.actions and (task.name[0] != '_'): + duration = self.duration(self.t_results[task.name].elapsed) + self.write(colored('. %s FAILED (%s)\n' % (task.title(), duration), 'red')) + self.failures.append({'task': task, 'exception': exception}) + + def add_success(self, task): + """ + called when excution finishes successfuly + """ + self.t_results[task.name].set_result('success') + + if task.actions and (task.name[0] != '_'): + duration = self.duration(self.t_results[task.name].elapsed) + self.write(colored('. %s finished (%s)\n' % (task.title(), duration), 'green')) + + def skip_uptodate(self, task): + """ + skipped up-to-date task + """ + self.t_results[task.name].set_result('up-to-date') + + def skip_ignore(self, task): + """ + skipped ignored task + """ + self.t_results[task.name].set_result('ignore') + + def cleanup_error(self, exception): + """ + error during cleanup + """ + self.errors.append(exception.get_msg()) + + def runtime_error(self, msg): + """ + error from doit (not from a task execution) + """ + self.errors.append(msg) + + def teardown_task(self, task): + """ + called when starts the execution of teardown action + """ + pass + + def complete_run(self): + """ + called when finshed running all tasks + """ + # restore stdout + log_out = sys.stdout.getvalue() + sys.stdout = self._old_out + log_err = sys.stderr.getvalue() + sys.stderr = self._old_err + + # add errors together with stderr output + if self.errors: + log_err += "\n".join(self.errors) + + task_result_list = [tr.to_dict() for tr in self.t_results.values()] + + self.writeln('') + self.writeln('-> finished %s tasks' % (len(task_result_list))) + self.writeln('') + + # sort task list by task name + task_result_list = sorted(task_result_list, key=lambda task: task['name']) + + # show tasks if verbosity == 2 + if not self.show_out: + for task in task_result_list: + # Skip finish chain task (no content, just finish tasks) + if 'FinishChain|' in task['name']: + continue + + self.task_stdout( + title=task['name'], + duration=task['elapsed'], + stdout=task['out'], + stderr=task['err'], + error=task['error'] + ) + + # show failed tasks (at the end) + for task in task_result_list: + # Skip finish chain task (no content, just finish tasks) + if 'FinishChain|' in task['name']: + continue + + if task['result'] == 'fail': + self.task_stdout( + title=task['name'], + duration=task['elapsed'], + stdout=task['out'], + stderr=task['err'], + error=task['error'] + ) + + if self.errors: + self.writeln("#" * 40 + "\n") + self.writeln("Execution aborted.\n") + self.writeln("\n".join(self.errors)) + self.writeln("\n") + + def task_stdout(self, title, duration=False, stdout=False, stderr=False, error=False, exception=False): + """ + Show task output + """ + + text_duration = '' + if duration: + text_duration = ' (%s)' % self.duration(duration) + + title_full = 'Task %s%s:' % (title, text_duration) + + self.writeln(title_full) + self.writeln('~' * len(title_full)) + + if stdout: + self.writeln() + self.writeln('%s' % stdout) + + if stderr: + self.writeln() + self.writeln(colored('-- STDERR OUTPUT --', 'red')) + self.write('%s' % stderr) + + if error: + self.writeln() + self.writeln(colored('-- ERROR OUTPUT --', 'red')) + self.write('%s' % error) + + if exception: + self.writeln() + self.writeln(colored('-- EXCEPTION --', 'red')) + self.write('%s' % exception.get_msg()) + + self.writeln() + self.writeln(':: end of output "%s"' % title) + self.writeln() + + def duration(self, duration): + """ + Calculate duration (seconds) to human readable time + """ + return 'duration: %s' % str(datetime.timedelta(seconds=int(duration))) + + def writeln(self, text=''): + """ + Output + """ + self.outstream.write('%s\n' % text) + + def write(self, text): + """ + Output + """ + self.outstream.write(text) diff --git a/bin/webdevops/doit/__init__.py b/bin/webdevops/doit/__init__.py new file mode 100644 index 000000000..29fb79a7c --- /dev/null +++ b/bin/webdevops/doit/__init__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from DoitReporter import DoitReporter + +__all__ = [ + 'DoitReporter', +] diff --git a/bin/webdevops/taskloader/BaseDockerTaskLoader.py b/bin/webdevops/taskloader/BaseDockerTaskLoader.py new file mode 100644 index 000000000..7fae9ba89 --- /dev/null +++ b/bin/webdevops/taskloader/BaseDockerTaskLoader.py @@ -0,0 +1,101 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import sys +import re +import copy +from .BaseTaskLoader import BaseTaskLoader +from webdevops import DockerfileUtility +from doit.task import dict_to_task + + +class BaseDockerTaskLoader(BaseTaskLoader): + + docker_client = False + + def __init__(self, configuration): + """ + Constrcutor + """ + BaseTaskLoader.__init__(self, configuration) + + # Init docker client + self.docker_client = configuration.get('dockerClient') + + def load_tasks(self, cmd, opt_values, pos_args): + """ + DOIT task list generator + """ + config = {'verbosity': self.configuration.get('verbosity')} + + dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( + base_path=self.configuration.get('dockerPath'), + path_regex=self.configuration.get('docker.pathRegex'), + image_prefix=self.configuration.get('docker.imagePrefix'), + whitelist=self.configuration.get('whitelist'), + blacklist=self.configuration.get('blacklist'), + ) + dockerfile_list = self.process_dockerfile_list(dockerfile_list) + + #import json,sys;print json.dumps(dockerfile_list, sort_keys=True, indent = 4, separators = (',', ': '));sys.exit(0); + + tasklist = self.generate_task_list(dockerfile_list) + + if not tasklist or len(tasklist) == 0: + raise Exception('No tasks found') + + tasklist = self.process_tasklist(tasklist) + + return tasklist, config + + def process_dockerfile_list(self, dockerfile_list): + """ + Prepare dockerfile list with dependency and also add "auto latest tag" images + """ + + image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']] + + autoLatestTagImageList = [] + + for dockerfile in dockerfile_list: + # Calculate dependency + dockerfile['dependency'] = False + if dockerfile['image']['from'] and dockerfile['image']['from'] in image_list: + dockerfile['dependency'] = dockerfile['image']['from'] + + # Process auto latest tag + if self.configuration.get('docker.autoLatestTag') and dockerfile['image']['tag'] == self.configuration.get('docker.autoLatestTag'): + imageNameLatest = DockerfileUtility.generate_image_name_with_tag_latest(dockerfile['image']['fullname']) + if imageNameLatest not in image_list: + autoLatestTagImage = copy.deepcopy(dockerfile) + autoLatestTagImage['image']['fullname'] = imageNameLatest + autoLatestTagImage['image']['tag'] = 'latest' + autoLatestTagImage['dependency'] = dockerfile['image']['fullname'] + autoLatestTagImageList.append(autoLatestTagImage) + + # Add auto latest tag images to dockerfile list + dockerfile_list.extend(autoLatestTagImageList) + + return dockerfile_list + + def generate_task_list(self, dockerfile_list): + return [] + diff --git a/bin/webdevops/taskloader/BaseTaskLoader.py b/bin/webdevops/taskloader/BaseTaskLoader.py new file mode 100644 index 000000000..a51d18fd7 --- /dev/null +++ b/bin/webdevops/taskloader/BaseTaskLoader.py @@ -0,0 +1,103 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import sys +import re +import time +import StringIO +from webdevops import DockerfileUtility +from doit.cmd_base import TaskLoader +from doit.task import dict_to_task + +class BaseTaskLoader(TaskLoader): + configuration = False + + def __init__(self, configuration): + """ + Constrcutor + """ + # Build configuration as namespace object + self.configuration = configuration + + + def process_tasklist(self, tasklist): + """ + Process task list and create task objects + """ + ret = [] + for task in tasklist: + ret.append(dict_to_task(task)) + return ret + + + @staticmethod + def human_task_name(name): + """ + Translate internal task name to human readable name + """ + return re.search('^.*\|(.*)', name).group(1) + + + @staticmethod + def human_task_name_list(list): + """ + Translate list of internal task names to human readable names + """ + ret = [] + for name in list: + ret.append(BaseTaskLoader.human_task_name(name)) + return ', '.join(ret) + + + @staticmethod + def action_chain_finish(title): + """ + Action of finish chain + """ + return + + + @staticmethod + def task_title_finish(task): + """ + Finish task title function + """ + return "Finished chain %s" % (BaseTaskLoader.human_task_name(task.name)) + + @staticmethod + def task_runner(func, args, task): + """ + Wrapper for task runner + + Will return the stdout if task fails as exception + """ + backup = sys.stdout + sys.stdout = StringIO.StringIO() + result = func(task=task, *args) + out = sys.stdout.getvalue() + sys.stdout.close() + sys.stdout = backup + + if not result: + raise Exception(out) + else: + print out + + return result diff --git a/bin/webdevops/taskloader/DockerBuildTaskLoader.py b/bin/webdevops/taskloader/DockerBuildTaskLoader.py new file mode 100644 index 000000000..43501140f --- /dev/null +++ b/bin/webdevops/taskloader/DockerBuildTaskLoader.py @@ -0,0 +1,122 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import sys +import re +import copy +import time +from .BaseTaskLoader import BaseTaskLoader +from .BaseDockerTaskLoader import BaseDockerTaskLoader +from webdevops import DockerfileUtility + +class DockerBuildTaskLoader(BaseDockerTaskLoader): + + def generate_task_list(self, dockerfileList): + """ + Generate task list for docker build + """ + tasklist = [] + for dockerfile in dockerfileList: + task = { + 'name': 'DockerBuild|%s' % dockerfile['image']['fullname'], + 'title': DockerBuildTaskLoader.task_title, + 'actions': [(BaseTaskLoader.task_runner, [DockerBuildTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], + 'task_dep': [] + } + + if dockerfile['dependency']: + task['task_dep'].append('DockerBuild|%s' % dockerfile['dependency']); + + tasklist.append(task) + + # task = { + # 'name': 'FinishChain|DockerBuild', + # 'title': DockerBuildTaskLoader.task_title_finish, + # 'actions': [(DockerBuildTaskLoader.action_chain_finish, ['docker build'])], + # 'task_dep': [task.name for task in taskList] + # } + # tasklist.append(task) + + return tasklist + + @staticmethod + def task_run(docker_client, dockerfile, configuration, task): + """ + Build one Dockerfile + """ + + pull_parent_image = DockerfileUtility.check_if_base_image_needs_pull(dockerfile, configuration) + + if configuration.get('dryRun'): + print ' from: %s (pull: %s)' % (dockerfile['image']['from'], ('yes' if pull_parent_image else 'no')) + print ' path: %s' % dockerfile['path'] + print ' dep: %s' % (DockerBuildTaskLoader.human_task_name_list(task.task_dep) if task.task_dep else 'none') + return True + + # Pull base image (FROM: xxx) first + if pull_parent_image: + print ' -> Pull base image %s ' % dockerfile['image']['from'] + + pull_image_name = DockerfileUtility.image_basename(dockerfile['image']['from']) + pull_image_tag = DockerfileUtility.extract_image_name_tag(dockerfile['image']['from']) + + pull_status = False + for retry_count in range(0, configuration.get('retry')): + pull_status = docker_client.pull_image( + name=pull_image_name, + tag=pull_image_tag, + ) + + if pull_status: + break + elif retry_count < (configuration.get('retry') - 1): + print ' failed, retrying... (try %s)' % (retry_count+1) + else: + print ' failed, giving up' + + if not pull_status: + return False + + ## Build image + print ' -> Building image %s ' % dockerfile['image']['fullname'] + build_status = False + for retry_count in range(0, configuration.get('retry')): + build_status = docker_client.build_dockerfile( + path=dockerfile['path'], + name=dockerfile['image']['fullname'], + nocache=configuration.get('dockerBuild.noCache'), + ) + + if build_status: + break + elif retry_count < (configuration.get('retry')-1): + print ' failed, retrying... (try %s)' % (retry_count+1) + else: + print ' failed, giving up' + + return build_status + + @staticmethod + def task_title(task): + """ + Build task title function + """ + return "Docker build %s" % (BaseTaskLoader.human_task_name(task.name)) diff --git a/bin/webdevops/taskloader/DockerPullTaskLoader.py b/bin/webdevops/taskloader/DockerPullTaskLoader.py new file mode 100644 index 000000000..be8a9c3a6 --- /dev/null +++ b/bin/webdevops/taskloader/DockerPullTaskLoader.py @@ -0,0 +1,87 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import sys +import re +import copy +from .BaseTaskLoader import BaseTaskLoader +from .BaseDockerTaskLoader import BaseDockerTaskLoader +from webdevops import DockerfileUtility + +class DockerPullTaskLoader(BaseDockerTaskLoader): + + def generate_task_list(self, dockerfile_list): + """ + Generate task list for docker pull + """ + tasklist = [] + + for dockerfile in dockerfile_list: + task = { + 'name': 'DockerPull|%s' % dockerfile['image']['fullname'], + 'title': DockerPullTaskLoader.task_title, + 'actions': [(BaseTaskLoader.task_runner, [DockerPullTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], + 'task_dep': [] + } + tasklist.append(task) + + # task = { + # 'name': 'FinishChain|DockerPush', + # 'title': DockerPullTaskLoader.task_title_finish, + # 'actions': [(DockerPullTaskLoader.action_chain_finish, ['docker push'])], + # 'task_dep': [task.name for task in taskList] + # } + # tasklist.append(task) + + return tasklist + + @staticmethod + def task_run(docker_client, dockerfile, configuration, task): + """ + Pull one Docker image from registry + """ + if configuration.get('dryRun'): + print ' pull: %s' % (dockerfile['image']['fullname']) + return True + + pull_status = False + for retry_count in range(0, configuration.get('retry')): + pull_status = docker_client.pull_image( + name=dockerfile['image']['name'], + tag=dockerfile['image']['tag'] + ) + + if pull_status: + break + elif retry_count < (configuration.get('retry') - 1): + print ' failed, retrying... (try %s)' % (retry_count+1) + else: + print ' failed, giving up' + + return pull_status + + @staticmethod + def task_title(task): + """ + Pull task title function + """ + return "Docker pull %s" % (BaseTaskLoader.human_task_name(task.name)) +0 diff --git a/bin/webdevops/taskloader/DockerPushTaskLoader.py b/bin/webdevops/taskloader/DockerPushTaskLoader.py new file mode 100644 index 000000000..118c3927e --- /dev/null +++ b/bin/webdevops/taskloader/DockerPushTaskLoader.py @@ -0,0 +1,86 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import sys +import re +import copy +from .BaseTaskLoader import BaseTaskLoader +from .BaseDockerTaskLoader import BaseDockerTaskLoader +from webdevops import DockerfileUtility + +class DockerPushTaskLoader(BaseDockerTaskLoader): + + def generate_task_list(self, dockerfile_list): + """ + Generate task list for docker push + """ + tasklist = [] + + for dockerfile in dockerfile_list: + task = { + 'name': 'DockerPush|%s' % dockerfile['image']['fullname'], + 'title': DockerPushTaskLoader.task_title, + 'actions': [(BaseTaskLoader.task_runner, [DockerPushTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], + 'task_dep': [] + } + tasklist.append(task) + + # task = { + # 'name': 'FinishChain|DockerPush', + # 'title': DockerPushTaskLoader.task_title_finish, + # 'actions': [(DockerPushTaskLoader.action_chain_finish, ['docker push'])], + # 'task_dep': [task.name for task in taskList] + # } + # tasklist.append(task) + + return tasklist + + @staticmethod + def task_run(docker_client, dockerfile, configuration, task): + """ + Push one Docker image to registry + """ + if configuration.get('dryRun'): + print ' push: %s' % (dockerfile['image']['fullname']) + return True + + push_status = False + for retry_count in range(0, configuration.get('retry')): + push_status = docker_client.push_image( + name=dockerfile['image']['fullname'], + ) + + if push_status: + break + elif retry_count < (configuration.get('retry') - 1): + print ' failed, retrying... (try %s)' % (retry_count+1) + else: + print ' failed, giving up' + + return push_status + + @staticmethod + def task_title(task): + """ + Push task title function + """ + return "Docker push %s" % (BaseTaskLoader.human_task_name(task.name)) +0 diff --git a/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py b/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py new file mode 100644 index 000000000..0d362c6e1 --- /dev/null +++ b/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py @@ -0,0 +1,204 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os, re, tempfile, json +from webdevops import Command +from .BaseDockerTaskLoader import BaseDockerTaskLoader +from .BaseTaskLoader import BaseTaskLoader +from doit.task import dict_to_task +import pytest + +class DockerTestServerspecTaskLoader(BaseDockerTaskLoader): + + def generate_task_list(self, dockerfile_list): + """ + Generate task list for docker push + """ + tasklist = [] + + for dockerfile in dockerfile_list: + task = { + 'name': 'DockerTestServerspec|%s' % dockerfile['image']['fullname'], + 'title': DockerTestServerspecTaskLoader.task_title, + 'actions': [(BaseTaskLoader.task_runner, [DockerTestServerspecTaskLoader.task_run, [dockerfile, self.configuration]])], + 'task_dep': [] + } + tasklist.append(task) + + # task = { + # 'name': 'FinishChain|DockerTestServerspec', + # 'title': DockerTestTestinfraTaskLoader.task_title_finish, + # 'actions': [(DockerTestTestinfraTaskLoader.action_chain_finish, ['docker test'])], + # 'task_dep': [task.name for task in taskList] + # } + # tasklist.append(task) + + return tasklist + + @staticmethod + def task_run(dockerfile, configuration, task): + """ + Run test + """ + # Check if current image is a toolimage (no daemon) + is_toolimage = False + for term in configuration.get('dockerTest.toolImages', {}): + if term in dockerfile['image']['fullname']: + is_toolimage = True + + # rspec spec file settings + spec_path = configuration.get('dockerTest.serverspec.specPath') % dockerfile['image']['imageName'] + spec_abs_path = os.path.join(configuration.get('serverspecPath'), spec_path) + + # create dockerfile + tmp_suffix = '.%s_%s_%s.tmp' % (dockerfile['image']['repository'], dockerfile['image']['imageName'], dockerfile['image']['tag']) + test_dockerfile = tempfile.NamedTemporaryFile(prefix='Dockerfile.', suffix=tmp_suffix, dir=configuration.get('serverspecPath'), bufsize=0, delete=False) + + # serverspec options + serverspec_opts = [] + serverspec_opts.extend(['--pattern', spec_path]) + + # serverspec env + serverspec_env = DockerTestServerspecTaskLoader.generate_serverspec_environment( + path=os.path.basename(test_dockerfile.name), + dockerfile=dockerfile, + configuration=configuration, + is_toolimage=is_toolimage + ) + + # dockerfile content + dockerfile_content = DockerTestServerspecTaskLoader.generate_dockerfile( + dockerfile=dockerfile, + configuration=configuration, + is_toolimage=is_toolimage + ) + + # DryRun + if configuration.get('dryRun'): + if not os.path.isfile(spec_abs_path): + print ' no tests found' + + print ' image: %s' % (dockerfile['image']['fullname']) + print ' path: %s' % (spec_path) + print ' args: %s' % (' '.join(serverspec_opts)) + print '' + print 'environment:' + print '------------' + print json.dumps(serverspec_env, indent=4, sort_keys=True) + print '' + print 'Dockerfile:' + print '-----------' + print dockerfile_content + + os.remove(test_dockerfile.name) + return True + + # check if we have any tests + if not os.path.isfile(spec_abs_path): + print ' no tests defined (%s)' % (spec_path) + return True + + # build rspec/serverspec command + cmd = ['bundle', 'exec', 'rspec'] + cmd.extend(serverspec_opts) + + # Set environment variables + env = os.environ.copy() + env.update(serverspec_env) + + # create Dockerfile + with open(test_dockerfile.name, mode='w', buffering=0) as f: + f.write(dockerfile_content) + f.flush() + f.close() + + test_status = False + for retry_count in range(0, configuration.get('retry')): + try: + test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath'), env=env) + except Exception as e: + print e + pass + + if test_status: + break + elif retry_count < (configuration.get('retry') - 1): + print ' failed, retrying... (try %s)' % (retry_count + 1) + else: + print ' failed, giving up' + + os.remove(test_dockerfile.name) + return test_status + + @staticmethod + def generate_serverspec_environment(path, dockerfile, configuration, is_toolimage=False): + """ + Generate serverspec environment dict + """ + ret = {} + + # add default vars + default_env_list = configuration.get('dockerTest.environment.default', False) + if default_env_list: + ret = default_env_list.to_dict() + + # add docker image specific vars + image_env_list = configuration.get('dockerTest.environment.image') + if image_env_list: + image_env_list = image_env_list.to_dict() + for term in image_env_list: + if term in dockerfile['image']['fullname']: + for key in image_env_list[term]: + ret[key] = str(image_env_list[term][key]) + + # add spec specific vars + ret['DOCKER_IMAGE'] = dockerfile['image']['fullname'] + ret['DOCKER_TAG'] = dockerfile['image']['tag'] + ret['DOCKERFILE'] = path + ret['DOCKER_IS_TOOLIMAGE'] = str(int(is_toolimage)) + + return ret + + + @staticmethod + def generate_dockerfile(dockerfile, configuration, is_toolimage=False): + """ + Generate Dockerfile content + """ + ret = [] + + ret.append('FROM %s' % dockerfile['image']['fullname']) + ret.append('COPY conf/ /') + + if is_toolimage: + ret.append('RUN chmod +x /loop-entrypoint.sh') + ret.append('ENTRYPOINT /loop-entrypoint.sh') + + for term in configuration.get('dockerTest.dockerfile', {}): + if term in dockerfile['image']['fullname']: + ret.extend( configuration.get('dockerTest.dockerfile').get(term)) + return '\n'.join(ret) + + @staticmethod + def task_title(task): + """ + Build task title function + """ + return "Run serverspec for %s" % (BaseTaskLoader.human_task_name(task.name)) diff --git a/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py b/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py new file mode 100644 index 000000000..f4cc4f7cd --- /dev/null +++ b/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py @@ -0,0 +1,86 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import re +from .BaseDockerTaskLoader import BaseDockerTaskLoader +from .BaseTaskLoader import BaseTaskLoader +from webdevops.testinfra import TestinfraDockerPlugin +from doit.task import dict_to_task +import pytest + +class DockerTestTestinfraTaskLoader(BaseDockerTaskLoader): + + def generate_task_list(self, dockerfile_list): + """ + Generate task list for docker push + """ + tasklist = [] + + for dockerfile in dockerfile_list: + task = { + 'name': 'DockerTestTestinfra|%s' % dockerfile['image']['fullname'], + 'title': DockerTestTestinfraTaskLoader.task_title, + 'actions': [(BaseTaskLoader.task_runner, [DockerTestTestinfraTaskLoader.task_run, [dockerfile, self.configuration]])], + 'task_dep': [] + } + tasklist.append(task) + + # task = { + # 'name': 'FinishChain|DockerTest', + # 'title': DockerTestTestinfraTaskLoader.task_title_finish, + # 'actions': [(DockerTestTestinfraTaskLoader.action_chain_finish, ['docker test'])], + # 'task_dep': [task.name for task in taskList] + # } + # tasklist.append(task) + + return tasklist + + @staticmethod + def task_run(dockerfile, configuration, task): + """ + Run test + """ + + test_opts = [] + + test_opts.extend(['-x', configuration.get('testinfraPath')]) + + if configuration.get('verbosity') > 1: + test_opts.extend(['-v']) + + if configuration.get('dryRun'): + print ' image: %s' % (dockerfile['image']['fullname']) + print ' args: %s' % (' '.join(test_opts)) + return True + + exitcode = pytest.main(test_opts, plugins=[TestinfraDockerPlugin(configuration=configuration, docker_image=dockerfile['image']['fullname'])]) + + if exitcode == 0: + return True + else: + return False + + @staticmethod + def task_title(task): + """ + Build task title function + """ + return "Run pytest for %s" % (BaseTaskLoader.human_task_name(task.name)) diff --git a/bin/webdevops/taskloader/__init__.py b/bin/webdevops/taskloader/__init__.py new file mode 100644 index 000000000..e333eaa0a --- /dev/null +++ b/bin/webdevops/taskloader/__init__.py @@ -0,0 +1,36 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from BaseTaskLoader import BaseTaskLoader +from BaseDockerTaskLoader import BaseDockerTaskLoader +from DockerBuildTaskLoader import DockerBuildTaskLoader +from DockerPushTaskLoader import DockerPushTaskLoader +from DockerPullTaskLoader import DockerPullTaskLoader +from DockerTestTestinfraTaskLoader import DockerTestTestinfraTaskLoader +from DockerTestServerspecTaskLoader import DockerTestServerspecTaskLoader + +__all__ = [ + 'BaseTaskLoader', + 'DockerBuildTaskLoader', + 'DockerPushTaskLoader', + 'DockerTestTestinfraTaskLoader', + 'DockerTestServerspecTaskLoader', + 'DockerPullTaskLoader', +] diff --git a/bin/webdevops/testinfra/TestinfraDockerPlugin.py b/bin/webdevops/testinfra/TestinfraDockerPlugin.py new file mode 100644 index 000000000..ffe9ed322 --- /dev/null +++ b/bin/webdevops/testinfra/TestinfraDockerPlugin.py @@ -0,0 +1,116 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import pytest +import re +from webdevops import DockerfileUtility + +class TestinfraDockerPlugin: + docker_image_list = [] + configuration = False + + def __init__(self, configuration, docker_image=False): + """ + Constructor + """ + self.docker_image_list = [] + self.configuration = configuration + self.init_docker_image_list(docker_image) + + def init_docker_image_list(self, docker_image=False): + """ + Init and build list of available docker images + """ + if docker_image: + self.docker_image_list.append(docker_image) + else: + dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( + base_path=self.configuration.get('dockerPath'), + path_regex=self.configuration.get('docker.pathRegex'), + image_prefix=self.configuration.get('docker.imagePrefix'), + whitelist=self.configuration.get('whitelist'), + blacklist=self.configuration.get('blacklist'), + ) + + for image in dockerfile_list: + self.docker_image_list.append(image['image']['fullname']) + + def get_image_list_by_regexp(self, filter_regexp): + """ + Get image list by filtering via filter regexp + """ + ret = [] + + filter_regexp = re.compile(filter_regexp, re.IGNORECASE) + + for image_name in self.docker_image_list: + if filter_regexp.search(image_name): + ret.append(image_name) + + return ret + + def filter_list_by_term(self, list, term): + """ + Filter list by using blacklist term + """ + tmp = [] + for item in list: + if not term in item: + tmp.append(item) + return tmp + + def pytest_generate_tests(self, metafunc): + """ + Generate tests + """ + if "TestinfraBackend" in metafunc.fixturenames: + images = [] + + # Lookup "docker_images" marker + marker = getattr(metafunc.function, "docker_images", None) + if marker is not None: + for marker_image_name in marker.args: + images.extend(self.get_image_list_by_regexp(marker_image_name)) + + # Lookup "docker_images.blacklist" marker + marker = getattr(metafunc.function, "docker_images_blacklist", None) + if marker is not None: + for blacklist_term in marker.args: + images = self.filter_list_by_term( + list=images, + term=blacklist_term + ) + + # Check for infinite loop + marker = getattr(metafunc.function, "docker_loop", None) + if marker is not None: + images = ['{}#loop'.format(item) for item in images] + + # If the test has a destructive marker, we scope TestinfraBackend + # at function level (i.e. executing for each test). If not we scope + # at session level (i.e. all tests will share the same container) + if getattr(metafunc.function, "destructive", None) is not None: + scope = "function" + else: + scope = "session" + + metafunc.parametrize( + "TestinfraBackend", images, indirect=True, scope=scope + ) diff --git a/bin/webdevops/testinfra/__init__.py b/bin/webdevops/testinfra/__init__.py new file mode 100644 index 000000000..7a919e2a8 --- /dev/null +++ b/bin/webdevops/testinfra/__init__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env/python +# -*- coding: utf-8 -*- +# +# (c) 2016 WebDevOps.io +# +# This file is part of Dockerfile Repository. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from TestinfraDockerPlugin import TestinfraDockerPlugin + +__all__ = [ + 'TestinfraDockerPlugin', +] diff --git a/conf/console.yml b/conf/console.yml new file mode 100644 index 000000000..2edabae57 --- /dev/null +++ b/conf/console.yml @@ -0,0 +1,113 @@ +--- + +dockerPath: 'docker' +templatePath: 'template' +provisionPath: 'provisioning' +imagePath: 'documentation/docs/resources/images/' +baselayoutPath: 'baselayout' +testinfraPath: 'tests/testinfra' +serverspecPath: 'tests/serverspec' +blacklistFile: 'BLACKLIST' + +docker: + imagePrefix: 'webdevops' + autoLatestTag: 'ubuntu-16.04' + pathRegex: '/(?P[^/]+)/(?P[^/]+)/Dockerfile$' + autoPull: True + autoPullBlacklist: '^webdevops/.*' + + +graph: + label: 'Webdevops Images\n%s' + + +dockerTest: + toolImages: + - '/bootstrap' + - '/ansible' + - '/liquibase' + - '/certbot' + + serverspec: + specPath: 'spec/docker/%s_spec.rb' + + dockerfile: + '/varnish': + - 'ENV VARNISH_BACKEND_HOST webdevops.io' + + environment: + default: + OS_FAMILY: 'ubuntu' + OS_VERSION: '16.04' + image: + ':alpine': + OS_FAMILY: 'alpine' + OS_VERSION: '3' + PHP_MHASH: '0' + ':ubuntu-12.04': + OS_FAMILY: 'ubuntu' + OS_VERSION: '12.04' + PHP_REDIS: '0' + PHP_APCU: '0' + ':ubuntu-14.04': + OS_FAMILY: 'ubuntu' + OS_VERSION: '14.04' + ':ubuntu-15.04': + OS_FAMILY: 'ubuntu' + OS_VERSION: '15.04' + ':ubuntu-15.10': + OS_FAMILY: 'ubuntu' + OS_VERSION: '15.10' + ':ubuntu-16.04': + OS_FAMILY: 'ubuntu' + OS_VERSION: '16.04' + ':debian-7': + OS_FAMILY: 'debian' + OS_VERSION: '7' + PHP_REDIS: '0' + PHP_APCU: '0' + ':debian-8': + OS_FAMILY: 'debian' + OS_VERSION: '8' + ':debian-8-php7': + OS_FAMILY: 'debian' + OS_VERSION: '8' + PHP_XDEBUG: '0' + ':debian-9': + OS_FAMILY: 'debian' + OS_VERSION: 'testing' + ':centos-7': + OS_FAMILY: 'redhat' + OS_VERSION: '7' + ':centos-7-php56': + OS_FAMILY: 'redhat' + OS_VERSION: '7' + PHP_APCU: '0' + PHP_REDIS: '0' + '/ssh': + OS_FAMILY: 'ubuntu' + OS_VERSION: '16.04' + '/vsftp': + OS_FAMILY: 'ubuntu' + OS_VERSION: '16.04' + '/postfix': + OS_FAMILY: 'ubuntu' + OS_VERSION: '16.04' + '/mail-sandbox': + OS_FAMILY: 'ubuntu' + OS_VERSION: '16.04' + '/liquibase': + OS_FAMILY: 'alpine' + OS_VERSION: '3' + '/certbot': + OS_FAMILY: 'alpine' + OS_VERSION: '3' + '/sphinx': + OS_FAMILY: 'alpine' + OS_VERSION: '3' + '/varnish': + OS_FAMILY: 'alpine' + OS_VERSION: '3' + '/samson-deployment': + OS_FAMILY: 'debian' + OS_VERSION: '8' diff --git a/bin/diagram.yml b/conf/diagram.yml similarity index 74% rename from bin/diagram.yml rename to conf/diagram.yml index a3c5ea875..138b0eeba 100644 --- a/bin/diagram.yml +++ b/conf/diagram.yml @@ -21,8 +21,41 @@ diagram: fontsize: "12" fontcolor: white groups: - common: + official: + name: Official images (Docker hub) + rank: 1 + styles: + graph: + style: dashed + fillcolor: gray + nodes: + fontname: Helvetica + shape: box + fontcolor: black + color: black + style: filled + fillcolor: "#e1e1e1" + docker: + - "^[^/]+$" + upstream: + name: Upstream images (Docker hub) + rank: 1 + styles: + graph: + style: dashed + fillcolor: gray + nodes: + fontname: Helvetica + shape: box + fontcolor: black + color: black + style: filled + fillcolor: "#e1e1e1" + docker: + - "^(?!webdevops)[^/]+/[^/]+" + base: name: Base images + rank: 2 styles: graph: style: filled @@ -36,12 +69,12 @@ diagram: fillcolor: "#e1e1e1" docker: - webdevops/bootstrap - - webdevops/base - - webdevops/base-app + - webdevops/base.* - webdevops/ansible - webdevops/storage php: name: PHP images + rank: 3 styles: graph: style: filled @@ -59,6 +92,7 @@ diagram: - webdevops/php-nginx php-dev: name: PHP development images + rank: 4 styles: graph: style: filled @@ -71,11 +105,10 @@ diagram: style: filled fillcolor: "#78c445" docker: - - webdevops/php-dev - - webdevops/php-apache-dev - - webdevops/php-nginx-dev + - webdevops/php.*-dev hhvm: name: HHVM images + rank: 3 styles: graph: style: filled @@ -88,11 +121,10 @@ diagram: style: filled fillcolor: "#7f84f1" docker: - - webdevops/hhvm - - webdevops/hhvm-apache - - webdevops/hhvm-nginx + - webdevops/hhvm.* application: name: Application images + rank: 3 styles: graph: style: filled @@ -109,6 +141,7 @@ diagram: - webdevops/piwik service: name: Service images + rank: 3 styles: graph: style: filled @@ -121,10 +154,8 @@ diagram: style: filled fillcolor: "#ffa35f" docker: - - webdevops/apache - - webdevops/apache-dev - - webdevops/nginx - - webdevops/nginx-dev + - webdevops/apache.* + - webdevops/nginx.* - webdevops/ssh - webdevops/vsftp - webdevops/postfix @@ -133,6 +164,7 @@ diagram: - webdevops/samson-deployment tools: name: Tools images + rank: 3 styles: graph: style: filled @@ -146,3 +178,5 @@ diagram: fillcolor: "#78c445" docker: - webdevops/sphinx + - webdevops/certbot + - webdevops/liquibase diff --git a/conf/provision.yml b/conf/provision.yml new file mode 100644 index 000000000..7dbfcff9a --- /dev/null +++ b/conf/provision.yml @@ -0,0 +1,108 @@ +--- +provision: + bootstrap: + baselayout: on + base: + configuration: + base/general: '*' + base/centos: centos-* + base/alpine: alpine-* + base-app: + configuration: + base-app/general: '*' + apache: + configuration: + apache/general : '*' + apache/centos : centos-* + apache/alpine : alpine-* + apache-dev: + configuration: + apache-dev/general: '*' + nginx: + configuration: + nginx/general : '*' + nginx/centos : centos-* + nginx/alpine : alpine-* + nginx-dev: + configuration: + nginx-dev/general : '*' + hhvm: + configuration: + hhvm/general : '*' + hhmv-apache: + configuration: + apache/general : '*' + hhvm-apache/general : '*' + hhvm-nginx: + configuration: + nginx/general : '*' + nginx/centos : centos-* + hhvm-nginx/general: '*' + php: + configuration: + php/general : '*' + php/ubuntu-12.04 : ubuntu-12.04 + php/alpine : alpine-* + # deploy php7 configuration to *-php7 containers + php/php7 : + - '*-php7' + - debian-9 + - ubuntu-16.04 + php-apache: + configuration: + apache/general : '*' + apache/centos : centos-* + apache/alpine : alpine-* + php-apache/general: '*' + php-nginx: + configuration: + nginx/general : '*' + nginx/centos : centos-* + nginx/alpine : alpine-* + php-nginx/general : '*' + php-dev: + configuration: + php-dev/general : '*' + php-apache-dev: + configuration: + apache/general : '*' + apache/centos : centos-* + apache/alpine : alpine-* + php-apache/general: '*' + php-dev/general : '*' + apache-dev/general: '*' + php-nginx-dev: + configuration: + nginx/general : '*' + nginx/centos : centos-* + nginx/alpine : alpine-* + php-nginx/general : '*' + php-dev/general : '*' + nginx-dev/general : '*' + postfix: + configuration: + postfix/general : '*' + mail-sandbox: + configuration: + mail-sandbox/general: '*' + vsftp: + configuration: + vsftp/general : '*' + typo3: + configuration: + typo3/general : '*' + piwik: + configuration: + piwik/general : '*' + varnish: + configuration: + varnish/general : '*' + liquibase: + configuration: + liquibase/general : '*' + samson-deployment: + baselayout: on + configuration: + base/general : latest + base-app/general : latest + samson-deployment/general: latest diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..b26fcf303 --- /dev/null +++ b/conftest.py @@ -0,0 +1,49 @@ +import os, time +import pytest, testinfra + +conftest_path = os.path.dirname(os.path.realpath(__file__)) +test_path = os.path.join(conftest_path, 'tests') +test_conf_path = os.path.join(test_path, 'conf') +test_conf_app_path = os.path.join(test_conf_path, 'app') + +# Use testinfra to get a handy function to run commands locally +check_output = testinfra.get_backend( + "local://" +).get_module("Command").check_output + + +@pytest.fixture +def TestinfraBackend(request): + # Override the TestinfraBackend fixture, + # all testinfra fixtures (i.e. modules) depend on it. + + docker_command = '' + docker_image = request.param + docker_sleeptime = 5 + + # Check for custom command in docker image name + if "#" in docker_image: + docker_image, docker_command = docker_image.split("#", 2) + + if docker_command == "loop": + docker_command = 'tail -f /dev/null' + docker_sleeptime = 0 + + docker_id = check_output( + "docker run -d -v \"%s:/app:ro\" %s " + docker_command, + test_conf_app_path, + docker_image + ) + + def teardown(): + check_output("docker rm -f %s", docker_id) + + # Destroy the container at the end of the fixture life + request.addfinalizer(teardown) + + # wait for getting the image up + if docker_sleeptime: + time.sleep(docker_sleeptime) + + # Return a dynamic created backend + return testinfra.get_backend("docker://" + docker_id) diff --git a/docker/base-app/alpine-3/Dockerfile b/docker/base-app/alpine-3/Dockerfile index 5a207ea87..5bde2b016 100644 --- a/docker/base-app/alpine-3/Dockerfile +++ b/docker/base-app/alpine-3/Dockerfile @@ -18,7 +18,8 @@ ENV APPLICATION_GID 1000 COPY conf/ /opt/docker/ -RUN /usr/local/bin/apk-install \ +RUN /usr/local/bin/apk-install-edge shadow \ + && /usr/local/bin/apk-install \ # Install services openssh \ mysql-client \ diff --git a/docker/base-app/alpine-3/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/alpine-3/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/alpine-3/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/alpine-3/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/centos-7/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/centos-7/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/centos-7/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/centos-7/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/debian-7/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/debian-7/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/debian-7/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/debian-7/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/debian-8/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/debian-8/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/debian-8/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/debian-8/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/debian-9/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/debian-9/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/debian-9/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/debian-9/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/ubuntu-12.04/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/ubuntu-12.04/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/ubuntu-12.04/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/ubuntu-12.04/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/ubuntu-14.04/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/ubuntu-14.04/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/ubuntu-14.04/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/ubuntu-14.04/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/ubuntu-15.04/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/ubuntu-15.04/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/ubuntu-15.04/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/ubuntu-15.04/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/ubuntu-15.10/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/ubuntu-15.10/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/ubuntu-15.10/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/ubuntu-15.10/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base-app/ubuntu-16.04/conf/bin/service.d/postfix.d/10-init.sh b/docker/base-app/ubuntu-16.04/conf/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/docker/base-app/ubuntu-16.04/conf/bin/service.d/postfix.d/10-init.sh +++ b/docker/base-app/ubuntu-16.04/conf/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/docker/base/alpine-3/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/alpine-3/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/alpine-3/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/alpine-3/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/alpine-3/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/alpine-3/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/alpine-3/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/centos-7/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/centos-7/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/centos-7/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/centos-7/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/centos-7/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/centos-7/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/centos-7/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/debian-7/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/debian-7/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/debian-7/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/debian-7/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/debian-7/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/debian-7/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/debian-7/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/debian-8/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/debian-8/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/debian-8/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/debian-8/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/debian-8/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/debian-8/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/debian-8/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/debian-9/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/debian-9/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/debian-9/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/debian-9/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/debian-9/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/debian-9/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/debian-9/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/ubuntu-12.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/ubuntu-12.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/ubuntu-12.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/ubuntu-12.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/ubuntu-12.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/ubuntu-12.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/ubuntu-12.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/ubuntu-14.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/ubuntu-14.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/ubuntu-14.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/ubuntu-14.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/ubuntu-14.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/ubuntu-14.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/ubuntu-14.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/ubuntu-15.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/ubuntu-15.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/ubuntu-15.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/ubuntu-15.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/ubuntu-15.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/ubuntu-15.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/ubuntu-15.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/ubuntu-15.10/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/ubuntu-15.10/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/ubuntu-15.10/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/ubuntu-15.10/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/ubuntu-15.10/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/ubuntu-15.10/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/ubuntu-15.10/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/base/ubuntu-16.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/base/ubuntu-16.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/base/ubuntu-16.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/base/ubuntu-16.04/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/base/ubuntu-16.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/base/ubuntu-16.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/base/ubuntu-16.04/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/docker/bootstrap/alpine-3/Dockerfile b/docker/bootstrap/alpine-3/Dockerfile index 3cbf2a4a6..a4fda92e7 100644 --- a/docker/bootstrap/alpine-3/Dockerfile +++ b/docker/bootstrap/alpine-3/Dockerfile @@ -21,7 +21,9 @@ RUN set -x \ # Fix root terminal && echo "export TERM=xterm" >> /root/.bashrc \ # Add testing - && echo http://nl.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories \ + && echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories \ + # Add community + && echo http://dl-cdn.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories \ # System update && /usr/local/bin/apk-upgrade \ # Install base stuff diff --git a/docker/bootstrap/alpine-3/baselayout.tar b/docker/bootstrap/alpine-3/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/alpine-3/baselayout.tar and b/docker/bootstrap/alpine-3/baselayout.tar differ diff --git a/docker/bootstrap/centos-7/baselayout.tar b/docker/bootstrap/centos-7/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/centos-7/baselayout.tar and b/docker/bootstrap/centos-7/baselayout.tar differ diff --git a/docker/bootstrap/debian-7/baselayout.tar b/docker/bootstrap/debian-7/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/debian-7/baselayout.tar and b/docker/bootstrap/debian-7/baselayout.tar differ diff --git a/docker/bootstrap/debian-8/baselayout.tar b/docker/bootstrap/debian-8/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/debian-8/baselayout.tar and b/docker/bootstrap/debian-8/baselayout.tar differ diff --git a/docker/bootstrap/debian-9/baselayout.tar b/docker/bootstrap/debian-9/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/debian-9/baselayout.tar and b/docker/bootstrap/debian-9/baselayout.tar differ diff --git a/docker/bootstrap/ubuntu-12.04/baselayout.tar b/docker/bootstrap/ubuntu-12.04/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/ubuntu-12.04/baselayout.tar and b/docker/bootstrap/ubuntu-12.04/baselayout.tar differ diff --git a/docker/bootstrap/ubuntu-14.04/baselayout.tar b/docker/bootstrap/ubuntu-14.04/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/ubuntu-14.04/baselayout.tar and b/docker/bootstrap/ubuntu-14.04/baselayout.tar differ diff --git a/docker/bootstrap/ubuntu-15.04/baselayout.tar b/docker/bootstrap/ubuntu-15.04/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/ubuntu-15.04/baselayout.tar and b/docker/bootstrap/ubuntu-15.04/baselayout.tar differ diff --git a/docker/bootstrap/ubuntu-15.10/baselayout.tar b/docker/bootstrap/ubuntu-15.10/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/ubuntu-15.10/baselayout.tar and b/docker/bootstrap/ubuntu-15.10/baselayout.tar differ diff --git a/docker/bootstrap/ubuntu-16.04/baselayout.tar b/docker/bootstrap/ubuntu-16.04/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/bootstrap/ubuntu-16.04/baselayout.tar and b/docker/bootstrap/ubuntu-16.04/baselayout.tar differ diff --git a/docker/certbot/latest/Dockerfile b/docker/certbot/latest/Dockerfile index c36f3e0ca..a6e802410 100644 --- a/docker/certbot/latest/Dockerfile +++ b/docker/certbot/latest/Dockerfile @@ -14,4 +14,5 @@ VOLUME /etc/letsencrypt VOLUME /var/www RUN /usr/local/bin/apk-install \ - certbot + certbot \ + && pip install pyRFC3339 configobj ConfigArgParse diff --git a/docker/hhvm/ubuntu-14.04/Dockerfile b/docker/hhvm/ubuntu-14.04/Dockerfile index 2c87c74b1..c5af47c55 100644 --- a/docker/hhvm/ubuntu-14.04/Dockerfile +++ b/docker/hhvm/ubuntu-14.04/Dockerfile @@ -24,6 +24,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && set -x \ hhvm \ imagemagick \ graphicsmagick \ + ghostscript \ && /usr/bin/update-alternatives --install /usr/bin/php php /usr/bin/hhvm 60 \ && curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin/ --filename=composer \ && /opt/docker/bin/provision run --tag bootstrap --role webdevops-hhvm \ diff --git a/docker/hhvm/ubuntu-16.04/Dockerfile b/docker/hhvm/ubuntu-16.04/Dockerfile index f8ed10daf..dda37d6a8 100644 --- a/docker/hhvm/ubuntu-16.04/Dockerfile +++ b/docker/hhvm/ubuntu-16.04/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ hhvm \ imagemagick \ graphicsmagick \ + ghostscript \ && /usr/bin/update-alternatives --install /usr/bin/php php /usr/bin/hhvm 60 \ && curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin/ --filename=composer \ && /opt/docker/bin/provision run --tag bootstrap --role webdevops-hhvm \ diff --git a/docker/liquibase/latest/Dockerfile b/docker/liquibase/latest/Dockerfile index daea3c279..7e4a0830e 100644 --- a/docker/liquibase/latest/Dockerfile +++ b/docker/liquibase/latest/Dockerfile @@ -10,7 +10,7 @@ LABEL vendor=WebDevOps.io LABEL io.webdevops.layout=8 LABEL io.webdevops.version=0.57.1 -ENV LIQUIBASE_VERSION "3.5.1" +ENV LIQUIBASE_VERSION "3.5.3" ENV LIQUIBASE_DRIVER "com.mysql.jdbc.Driver" ENV LIQUIBASE_CLASSPATH "/usr/share/java/mysql.jar" ENV LIQUIBASE_URL "" diff --git a/docker/liquibase/latest/Dockerfile.jinja2 b/docker/liquibase/latest/Dockerfile.jinja2 index e5738ce6e..ac4f7c1ef 100644 --- a/docker/liquibase/latest/Dockerfile.jinja2 +++ b/docker/liquibase/latest/Dockerfile.jinja2 @@ -2,7 +2,7 @@ {{ docker.version() }} -{{ environment.liquibase('3.5.1', 'com.mysql.jdbc.Driver', '/usr/share/java/mysql.jar') }} +{{ environment.liquibase('3.5.3', 'com.mysql.jdbc.Driver', '/usr/share/java/mysql.jar') }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/liquibase/mysql/Dockerfile b/docker/liquibase/mysql/Dockerfile index e2c3faaae..045451c4a 100644 --- a/docker/liquibase/mysql/Dockerfile +++ b/docker/liquibase/mysql/Dockerfile @@ -10,7 +10,7 @@ LABEL vendor=WebDevOps.io LABEL io.webdevops.layout=8 LABEL io.webdevops.version=0.57.1 -ENV LIQUIBASE_VERSION "3.5.1" +ENV LIQUIBASE_VERSION "3.5.3" ENV LIQUIBASE_DRIVER "com.mysql.jdbc.Driver" ENV LIQUIBASE_CLASSPATH "/usr/share/java/mysql.jar" ENV LIQUIBASE_URL "" diff --git a/docker/liquibase/mysql/Dockerfile.jinja2 b/docker/liquibase/mysql/Dockerfile.jinja2 index e5738ce6e..ac4f7c1ef 100644 --- a/docker/liquibase/mysql/Dockerfile.jinja2 +++ b/docker/liquibase/mysql/Dockerfile.jinja2 @@ -2,7 +2,7 @@ {{ docker.version() }} -{{ environment.liquibase('3.5.1', 'com.mysql.jdbc.Driver', '/usr/share/java/mysql.jar') }} +{{ environment.liquibase('3.5.3', 'com.mysql.jdbc.Driver', '/usr/share/java/mysql.jar') }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/liquibase/postgres/Dockerfile b/docker/liquibase/postgres/Dockerfile index da7a6db2a..5732cf9cc 100644 --- a/docker/liquibase/postgres/Dockerfile +++ b/docker/liquibase/postgres/Dockerfile @@ -10,7 +10,7 @@ LABEL vendor=WebDevOps.io LABEL io.webdevops.layout=8 LABEL io.webdevops.version=0.57.1 -ENV LIQUIBASE_VERSION "3.5.1" +ENV LIQUIBASE_VERSION "3.5.3" ENV LIQUIBASE_DRIVER "org.postgresql.Driver" ENV LIQUIBASE_CLASSPATH "/usr/share/java/postgresql.jar" ENV LIQUIBASE_URL "" diff --git a/docker/liquibase/postgres/Dockerfile.jinja2 b/docker/liquibase/postgres/Dockerfile.jinja2 index 7a401bef9..6bbd67313 100644 --- a/docker/liquibase/postgres/Dockerfile.jinja2 +++ b/docker/liquibase/postgres/Dockerfile.jinja2 @@ -2,7 +2,7 @@ {{ docker.version() }} -{{ environment.liquibase('3.5.1', 'org.postgresql.Driver', '/usr/share/java/postgresql.jar') }} +{{ environment.liquibase('3.5.3', 'org.postgresql.Driver', '/usr/share/java/postgresql.jar') }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/mail-sandbox/README.md b/docker/mail-sandbox/README.md index 74dc2f9ab..f7fe08606 100644 --- a/docker/mail-sandbox/README.md +++ b/docker/mail-sandbox/README.md @@ -2,14 +2,14 @@ Automated build and tested by [WebDevOps Build Server](https://build.webdevops.io/) -Mail sandbox which catches all mails and delivers them to a local user and is accessable via IMAP. +Mail sandbox which catches all mails and delivers them to a local user and is accessable via IMAP and via Web (roundcube) ## Environment variables Variable | Description -------------------- | ------------------------------------------------------------------------------ -`MAILBOX_USERNAME` | Username for mailbox (Default `sandbox`) -`MAILBOX_PASSWORD` | Password for mailbox (Default `mail`) +`MAILBOX_USERNAME` | Username for mailbox (Default `dev`) +`MAILBOX_PASSWORD` | Password for mailbox (Default `dev`) ## Container info diff --git a/docker/mail-sandbox/latest/Dockerfile b/docker/mail-sandbox/latest/Dockerfile index 34b12a409..4056f3994 100644 --- a/docker/mail-sandbox/latest/Dockerfile +++ b/docker/mail-sandbox/latest/Dockerfile @@ -3,15 +3,15 @@ # -- automatically generated -- #+++++++++++++++++++++++++++++++++++++++ -FROM webdevops/postfix:latest +FROM webdevops/php-nginx:latest MAINTAINER info@webdevops.io LABEL vendor=WebDevOps.io LABEL io.webdevops.layout=8 LABEL io.webdevops.version=0.57.1 -ENV MAILBOX_USERNAME "sandbox" -ENV MAILBOX_PASSWORD "mail" +ENV MAILBOX_USERNAME "dev" +ENV MAILBOX_PASSWORD "dev" COPY conf/ /opt/docker/ @@ -21,8 +21,20 @@ RUN /usr/local/bin/apt-install \ dovecot-imapd \ && /opt/docker/bin/control.sh service.enable postfix \ && /opt/docker/bin/control.sh service.enable dovecot \ + && /opt/docker/bin/provision add --tag entrypoint webdevops-mail-sandbox \ && /opt/docker/bin/provision run --tag bootstrap --role webdevops-mail-sandbox \ && /opt/docker/bin/bootstrap.sh +# Install Roundcube + plugins +RUN cd /app \ + && rm -rf * \ + && wget https://github.com/roundcube/roundcubemail/releases/download/1.2.2/roundcubemail-1.2.2-complete.tar.gz \ + && tar xf roundcubemail-1.2.2-complete.tar.gz --strip-components 1 \ + && rm -f roundcubemail-1.2.2-complete.tar.gz \ + && ls -l \ + && rm -rf .git installer \ + && ln -s /opt/docker/etc/roundcube/plugins/webdevops_autologin/ plugins/webdevops_autologin \ + && ln -s /opt/docker/etc/roundcube/config.php config/config.inc.php + EXPOSE 25 465 587 143 993 diff --git a/docker/mail-sandbox/latest/Dockerfile.jinja2 b/docker/mail-sandbox/latest/Dockerfile.jinja2 index ae5219d3e..16809eb41 100644 --- a/docker/mail-sandbox/latest/Dockerfile.jinja2 +++ b/docker/mail-sandbox/latest/Dockerfile.jinja2 @@ -1,4 +1,4 @@ -{{ docker.from("postfix") }} +{{ docker.from("php-nginx") }} {{ docker.version() }} @@ -8,4 +8,6 @@ {{ mailsandbox.ubuntu() }} +{{ mailsandbox.roundcube() }} + {{ docker.expose('25 465 587 143 993') }} diff --git a/docker/mail-sandbox/latest/conf/etc/roundcube/config.php b/docker/mail-sandbox/latest/conf/etc/roundcube/config.php new file mode 100644 index 000000000..2699a3ef6 --- /dev/null +++ b/docker/mail-sandbox/latest/conf/etc/roundcube/config.php @@ -0,0 +1,18 @@ + [ + 'verify_peer' => false, + 'verify_peer_name' => false, + 'allow_self_signed' => true, + ], +]; +$config['default_port'] = 143; +$config['smtp_port'] = 25; +$config['smtp_user'] = '%u'; +$config['smtp_pass'] = '%p'; + +$config['plugins'][] = 'webdevops_autologin'; diff --git a/docker/mail-sandbox/latest/conf/etc/roundcube/plugins/webdevops_autologin/webdevops_autologin.php b/docker/mail-sandbox/latest/conf/etc/roundcube/plugins/webdevops_autologin/webdevops_autologin.php new file mode 100644 index 000000000..97fbd5859 --- /dev/null +++ b/docker/mail-sandbox/latest/conf/etc/roundcube/plugins/webdevops_autologin/webdevops_autologin.php @@ -0,0 +1,32 @@ +add_hook('startup', array($this, 'startup')); + $this->add_hook('authenticate', array($this, 'authenticate')); + } + + function startup($args) + { + // change action to login + if (empty($_SESSION['user_id'])) { + $args['action'] = 'login'; + } + + return $args; + } + + function authenticate($args) { + $args['user'] = getenv('MAILBOX_USERNAME'); + $args['pass'] = getenv('MAILBOX_PASSWORD'); + $args['host'] = 'localhost'; + $args['cookiecheck'] = false; + $args['valid'] = true; + + return $args; + } +} diff --git a/docker/mail-sandbox/latest/conf/provision/roles/webdevops-mail-sandbox/tasks/bootstrap/postfix.yml b/docker/mail-sandbox/latest/conf/provision/roles/webdevops-mail-sandbox/tasks/bootstrap/postfix.yml index 52e0f82a6..044d0f372 100644 --- a/docker/mail-sandbox/latest/conf/provision/roles/webdevops-mail-sandbox/tasks/bootstrap/postfix.yml +++ b/docker/mail-sandbox/latest/conf/provision/roles/webdevops-mail-sandbox/tasks/bootstrap/postfix.yml @@ -14,6 +14,11 @@ - { variable: 'mydestination', value: 'localhost' } - { variable: 'message_size_limit', value: '102400000' } +- name: Postfix configuration (listen on port 1025) + lineinfile: + dest: /etc/postfix/master.cf + line: '1025 inet n - y - - smtpd' + - name: Create /etc/mailname raw: 'hostname > /etc/mailname' args: diff --git a/docker/php/alpine-3-php7/Dockerfile b/docker/php/alpine-3-php7/Dockerfile index 745838c96..41b4401d4 100644 --- a/docker/php/alpine-3-php7/Dockerfile +++ b/docker/php/alpine-3-php7/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apk-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7-fpm \ php7-json \ diff --git a/docker/php/alpine-3/Dockerfile b/docker/php/alpine-3/Dockerfile index 68d419d47..ae82ee413 100644 --- a/docker/php/alpine-3/Dockerfile +++ b/docker/php/alpine-3/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apk-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ diff --git a/docker/php/centos-7-php56/Dockerfile b/docker/php/centos-7-php56/Dockerfile index a75837d72..74be0aa16 100644 --- a/docker/php/centos-7-php56/Dockerfile +++ b/docker/php/centos-7-php56/Dockerfile @@ -21,6 +21,7 @@ RUN rpm -Uvh https://mirror.webtatic.com/yum/el7/webtatic-release.rpm \ # Install tools ImageMagick \ GraphicsMagick \ + ghostscript \ # Install php (cli/fpm) php56w-cli \ php56w-fpm \ diff --git a/docker/php/centos-7/Dockerfile b/docker/php/centos-7/Dockerfile index 622c5dfd0..6278a5776 100644 --- a/docker/php/centos-7/Dockerfile +++ b/docker/php/centos-7/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/yum-install \ # Install tools ImageMagick \ GraphicsMagick \ + ghostscript \ # Install php (cli/fpm) php-cli \ php-fpm \ diff --git a/docker/php/debian-7-php7/Dockerfile.disabled b/docker/php/debian-7-php7/Dockerfile.disabled index b73dd087e..5e830f3e6 100644 --- a/docker/php/debian-7-php7/Dockerfile.disabled +++ b/docker/php/debian-7-php7/Dockerfile.disabled @@ -15,7 +15,8 @@ RUN echo "deb http://packages.dotdeb.org wheezy all" >> /etc/apt/sources.list \ # Install tools RUN /usr/local/bin/apt-install \ imagemagick \ - graphicsmagick + graphicsmagick \ + ghostscript # Install php RUN /usr/local/bin/apt-install \ diff --git a/docker/php/debian-7/Dockerfile b/docker/php/debian-7/Dockerfile index 76e78c9f5..89ddce235 100644 --- a/docker/php/debian-7/Dockerfile +++ b/docker/php/debian-7/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ diff --git a/docker/php/debian-8-php7/Dockerfile b/docker/php/debian-8-php7/Dockerfile index b737d6fcf..2b047c61b 100644 --- a/docker/php/debian-8-php7/Dockerfile +++ b/docker/php/debian-8-php7/Dockerfile @@ -33,6 +33,7 @@ RUN /usr/local/bin/apt-install apt-transport-https lsb-release \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7.0-cli \ php7.0-fpm \ diff --git a/docker/php/debian-8/Dockerfile b/docker/php/debian-8/Dockerfile index 0b8bcbcb2..e1a6bbfa8 100644 --- a/docker/php/debian-8/Dockerfile +++ b/docker/php/debian-8/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ diff --git a/docker/php/debian-9/Dockerfile b/docker/php/debian-9/Dockerfile index 00c448f1a..736c55c51 100644 --- a/docker/php/debian-9/Dockerfile +++ b/docker/php/debian-9/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7.0-cli \ php7.0-fpm \ diff --git a/docker/php/ubuntu-12.04/Dockerfile b/docker/php/ubuntu-12.04/Dockerfile index 4a8f63ed9..ee65f1938 100644 --- a/docker/php/ubuntu-12.04/Dockerfile +++ b/docker/php/ubuntu-12.04/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ diff --git a/docker/php/ubuntu-14.04/Dockerfile b/docker/php/ubuntu-14.04/Dockerfile index 1ecd5a405..8133685b2 100644 --- a/docker/php/ubuntu-14.04/Dockerfile +++ b/docker/php/ubuntu-14.04/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ diff --git a/docker/php/ubuntu-15.04/Dockerfile b/docker/php/ubuntu-15.04/Dockerfile index 62de6ca1b..35e8230be 100644 --- a/docker/php/ubuntu-15.04/Dockerfile +++ b/docker/php/ubuntu-15.04/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ diff --git a/docker/php/ubuntu-15.10/Dockerfile b/docker/php/ubuntu-15.10/Dockerfile index 2934f635e..8774ffdf2 100644 --- a/docker/php/ubuntu-15.10/Dockerfile +++ b/docker/php/ubuntu-15.10/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ diff --git a/docker/php/ubuntu-16.04/Dockerfile b/docker/php/ubuntu-16.04/Dockerfile index 78c1e467d..b4c53e1cd 100644 --- a/docker/php/ubuntu-16.04/Dockerfile +++ b/docker/php/ubuntu-16.04/Dockerfile @@ -21,6 +21,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7.0-cli \ php7.0-fpm \ diff --git a/docker/samson-deployment/latest/baselayout.tar b/docker/samson-deployment/latest/baselayout.tar index cd19b173e..59e4b1d50 100644 Binary files a/docker/samson-deployment/latest/baselayout.tar and b/docker/samson-deployment/latest/baselayout.tar differ diff --git a/docker/samson-deployment/latest/conf/provision/roles/webdevops-base/tasks/bootstrap.yml b/docker/samson-deployment/latest/conf/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/docker/samson-deployment/latest/conf/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/docker/samson-deployment/latest/conf/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/docker/samson-deployment/latest/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/docker/samson-deployment/latest/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/docker/samson-deployment/latest/conf/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/documentation/docs/content/BuildingProcess/building.rst b/documentation/docs/content/BuildingProcess/building.rst deleted file mode 100644 index 545724d5b..000000000 --- a/documentation/docs/content/BuildingProcess/building.rst +++ /dev/null @@ -1,16 +0,0 @@ -======== -Building -======== - -Requirements ------------- - -* python -* coreutils and gnu-sed on OSX: ``brew install coreutils gnu-sed`` - - -Building the Dockerfiles ------------------------- - -For building the Docker images locally you can use ``make all`` to start the build process. -There are multiple targets inside the Makefile for building only specific images eg. ``make webdevops/apache`` diff --git a/documentation/docs/content/BuildingProcess/customizing.rst b/documentation/docs/content/BuildingProcess/customizing.rst deleted file mode 100644 index e104c3b76..000000000 --- a/documentation/docs/content/BuildingProcess/customizing.rst +++ /dev/null @@ -1,33 +0,0 @@ -=========== -Customizing -=========== - -Baselayout ----------- - -The ``baselayout`` directory contains a bunch of smaller scripts which will uploaded as tar file into -``webdevops/bootstrap``. - -``make baselayout`` will build these tar files and deploy them to the Dockerfile directories. - -Provision ---------- - -The ``provision/`` directory containers files, scripts and provision roles which are copied to the specific Dockerfile -directories. - -The rules which directory is processed in which order is specified in ``bin/provision.sh``. - -``make provision`` will build these files and deploy them to the Dockerfile directories. - - -Dockerfile.jinja2 and templates -------------------------------- - -All ``Dockerfiles`` are generated from their appropriate ``Dockerfile.jinja2`` files. Inside the jinja2 Dockerfile -templates there are macros which are defined inside ``template/`` directory. - -The script ``bin/buildDockerfile.py`` will search for ``Dockerfile.jinja2`` files, processes them and stores them as -``Dockerfile``. - -``make provision`` will build these files and deploy them to the Dockerfile directories. diff --git a/documentation/docs/content/BuildingProcess/index.rst b/documentation/docs/content/BuildingProcess/index.rst deleted file mode 100644 index 810c56292..000000000 --- a/documentation/docs/content/BuildingProcess/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -============================ -Dockerfiles Building Process -============================ - -This section is not done yet! - - -.. toctree:: - :maxdepth: 1 - - building - customizing - testing diff --git a/documentation/docs/content/BuildingProcess/testing.rst b/documentation/docs/content/BuildingProcess/testing.rst deleted file mode 100644 index ddc3bdd4f..000000000 --- a/documentation/docs/content/BuildingProcess/testing.rst +++ /dev/null @@ -1,18 +0,0 @@ -======= -Testing -======= - -Testing is done using serverspec_ and rspec_. - -The whole test runner can be executed by running ``make test`` from the root directory. Inside the ``test/`` directory -there is an alternative Makefile for running specific Docker image tests. - -The Testrunner ``test/run.sh`` defines which tests are executed with which options or environment. - -The tests for Docker images are specified in ``test/spec/docker`` directory. - -All collection tests are defined inside ``test/spec/collection`` which are using the smaller tests -from ``test/spec/shared``. - -.. _rspec: http://rspec.info/ -.. _serverspec: http://serverspec.org/ diff --git a/documentation/docs/content/Commands/index.rst b/documentation/docs/content/Commands/index.rst new file mode 100644 index 000000000..510363aa2 --- /dev/null +++ b/documentation/docs/content/Commands/index.rst @@ -0,0 +1,129 @@ +====================== +Commands (bin/console) +====================== + +Requirements +------------ + +* **python** and **PIP** +* **ruby** for serverspec tests + +Install dependencies +-------------------- + +The building process, we need some python packages as well as ruby rspec and serverspec packages: + +.. code-block:: bash + + make requirements + +Configuration +------------- + +All commands are using configuration options from ``conf/console.yml``. + +bin/console tasks +----------------- + +bin/console docker:build +~~~~~~~~~~~~~~~~~~~~~~~~ + +Build all Dockerfiles found in ``docker/`` directory. The directory structure defines the naming of the built images +(using convention over configuration). + +===================================== ================================================================================= =============================================== +Option Description Values +===================================== ================================================================================= =============================================== +-v Verbose output *option only* +--threads=n Run in parallized mode (currently multi-process instead of real threads) *numeric values*, auto, auto/2, auto*2, auto-2, auto+2 +--dry-run Don't really execute build process *option only* +--no-cache Don't use Docker caching *option only* +--retry=n Retry process multiple times (eg. for networking issues) *numeric values* +--whitelist=term Only build Docker images with *term* in name *string value* +--blacklist=term Don't build Docker images with *term* in name *string value* +===================================== ================================================================================= =============================================== + +bin/console docker:push +~~~~~~~~~~~~~~~~~~~~~~~ + +Push (upload) all built Docker images to registry (using convention over configuration). + +===================================== ================================================================================= =============================================== +Option Description Values +===================================== ================================================================================= =============================================== +-v Verbose output *option only* +--threads=n Run in parallized mode (currently multi-process instead of real threads) *numeric values*, auto, auto/2, auto*2, auto-2, auto+2 +--dry-run Don't really execute build process *option only* +--retry=n Retry process multiple times (eg. for networking issues) *numeric values* +--whitelist=term Only build Docker images with *term* in name *string value* +--blacklist=term Don't build Docker images with *term* in name *string value* +===================================== ================================================================================= =============================================== + +bin/console docker:pull +~~~~~~~~~~~~~~~~~~~~~~~ + +Pull (download) all built Docker images to registry (using convention over configuration). + +===================================== ================================================================================= =============================================== +Option Description Values +===================================== ================================================================================= =============================================== +-v Verbose output *option only* +--threads=n Run in parallized mode (currently multi-process instead of real threads) *numeric values*, auto, auto/2, auto*2, auto-2, auto+2 +--dry-run Don't really execute build process *option only* +--retry=n Retry process multiple times (eg. for networking issues) *numeric values* +--whitelist=term Only build Docker images with *term* in name *string value* +--blacklist=term Don't build Docker images with *term* in name *string value* +===================================== ================================================================================= =============================================== + +bin/console test:testinfra +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Test built images with testinfra_ (python module), spec files are inside ``tests/testinfra`` + +===================================== ================================================================================= =============================================== +Option Description Values +===================================== ================================================================================= =============================================== +-v Verbose output *option only* +--threads=n Run in parallized mode (currently multi-process instead of real threads) *numeric values*, auto, auto/2, auto*2, auto-2, auto+2 +--dry-run Don't really execute build process *option only* +--whitelist=term Only build Docker images with *term* in name *string value* +--blacklist=term Don't build Docker images with *term* in name *string value* +===================================== ================================================================================= =============================================== + +bin/console test:serverspec +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Test built images with serverspec_ (python module), spec files are inside ``tests/serverspec`` + +===================================== ================================================================================= =============================================== +Option Description Values +===================================== ================================================================================= =============================================== +--threads=n Run in parallized mode (currently multi-process instead of real threads) *numeric values*, auto, auto/2, auto*2, auto-2, auto+2 +--dry-run Don't really execute build process *option only* +--retry=n Retry process multiple times (eg. for networking issues) *numeric values* +--whitelist=term Only build Docker images with *term* in name *string value* +--blacklist=term Don't build Docker images with *term* in name *string value* +===================================== ================================================================================= =============================================== + +bin/console generate:graph +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generates Docker images dependency graph using graphviz. + +bin/console generate:dockerfile +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate Dockerfiles from ``Dockerfile.jinja2`` templates. + +Configuration is stored inside ``conf/diagram.yml``. + +bin/console generate:provision +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generate provision (common configuration files) and deploy them to the specified Dockerfile directories. + +Configuration is stored inside ``conf/provision.yml``. + + +.. _testinfra: https://github.com/philpep/testinfra +.. _serverspec: http://serverspec.org/ diff --git a/documentation/docs/content/DockerImages/dockerfiles/mail-sandbox.rst b/documentation/docs/content/DockerImages/dockerfiles/mail-sandbox.rst index f8f525776..ca7218842 100644 --- a/documentation/docs/content/DockerImages/dockerfiles/mail-sandbox.rst +++ b/documentation/docs/content/DockerImages/dockerfiles/mail-sandbox.rst @@ -4,7 +4,7 @@ webdevops/mail-catcher These image extends ``webdevops/base`` with a postfix daemon which is running on port 25 and dovecot on IMAP. -This images catches all emails sent to it and stores them locally. These mails are available via IMAP. +This images catches all emails sent to it and stores them locally. These mails are available via IMAP and web (roundcube) Docker image tags ----------------- @@ -22,5 +22,15 @@ Environment variables .. include:: include/environment-base-app.rst +Base environment variables +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +====================== ============================= ============= +Environment variable Description Default +====================== ============================= ============= +``MAILBOX_USERNAME`` IMAP user dev +``MAILBOX_PASSWORD`` IMAP user password dev +====================== ============================= ============= + Docker image layout ------------------- diff --git a/documentation/docs/index.rst b/documentation/docs/index.rst index f406685f4..1546de032 100644 --- a/documentation/docs/index.rst +++ b/documentation/docs/index.rst @@ -26,7 +26,7 @@ Overview :maxdepth: 2 content/DockerImages/index - content/Customization/index + content/Commands/index content/BuildingProcess/index content/KnownIssues/index content/FAQ/index diff --git a/documentation/docs/resources/images/docker-image-layout.gv b/documentation/docs/resources/images/docker-image-layout.gv index 369639658..7e31d3d4a 100644 --- a/documentation/docs/resources/images/docker-image-layout.gv +++ b/documentation/docs/resources/images/docker-image-layout.gv @@ -2,30 +2,45 @@ digraph webdevops { graph [bgcolor=white fontcolor=black fontsize=16 rankdir=TP] node [color=white fillcolor="#E1E1E1" fontcolor=black fontname=Helvetica shape=box style=filled] edge [arrowhead=open color=black fontcolor=white fontname=Courier fontsize=12 style=dashed] - label = "\n\nWebdevops Images\n at :30.06.2016" + label = "\n\nWebdevops Images\n2016-11-28" + newrank=true; subgraph cluster_php { graph [fillcolor="#c0e5a8" style=filled] node [color=black fillcolor="#78c445" fontcolor=black fontname=Helvetica shape=box style=filled] label = "PHP images" + "webdevops/php-dev" -> "webdevops/php-nginx-dev" "webdevops/php" -> "webdevops/php-nginx" + "webdevops/php-dev" -> "webdevops/php-apache-dev" "webdevops/php" "webdevops/php" -> "webdevops/php-apache" + "webdevops/php" -> "webdevops/php-dev" } subgraph cluster_service { graph [fillcolor="#fbd3b5" style=filled] node [color=black fillcolor="#ffa35f" fontcolor=black fontname=Helvetica shape=box style=filled] label = "Service images" "webdevops/varnish" - "webdevops/nginx" "webdevops/nginx" -> "webdevops/nginx-dev" "webdevops/ssh" - "webdevops/postfix" -> "webdevops/mail-sandbox" - "webdevops/apache" + "webdevops/mail-sandbox" "webdevops/vsftp" + "webdevops/nginx" + "webdevops/apache" "webdevops/samson-deployment" "webdevops/apache" -> "webdevops/apache-dev" "webdevops/postfix" } + subgraph cluster_official { + graph [fillcolor=gray style=dashed] + node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] + label = "Official images (Docker hub)" + busybox + centos + java + ubuntu + debian + alpine + } subgraph cluster_application { graph [fillcolor="#f2e3b5" style=filled] node [color=black fillcolor="#e5b931" fontcolor=black fontname=Helvetica shape=box style=filled] @@ -33,16 +48,22 @@ digraph webdevops { "webdevops/typo3" "webdevops/piwik" } - subgraph cluster_common { + subgraph cluster_base { graph [fillcolor=gray style=filled] node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] label = "Base images" "webdevops/bootstrap" -> "webdevops/ansible" - "webdevops/storage" "webdevops/base" -> "webdevops/base-app" + "webdevops/storage" "webdevops/bootstrap" "webdevops/bootstrap" -> "webdevops/base" } + subgraph cluster_upstream { + graph [fillcolor=gray style=dashed] + node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] + label = "Upstream images (Docker hub)" + "zendesk/samson" + } subgraph cluster_hhvm { graph [fillcolor="#c1c3f2" style=filled] node [color=black fillcolor="#7f84f1" fontcolor=black fontname=Helvetica shape=box style=filled] @@ -56,25 +77,184 @@ digraph webdevops { node [color=black fillcolor="#78c445" fontcolor=black fontname=Helvetica shape=box style=filled] label = "Tools images" "webdevops/sphinx" + "webdevops/liquibase" + "webdevops/certbot" } subgraph "cluster_php-dev" { graph [fillcolor="#c0e5a8" style=filled] node [color=black fillcolor="#78c445" fontcolor=black fontname=Helvetica shape=box style=filled] label = "PHP development images" - "webdevops/php-dev" -> "webdevops/php-nginx-dev" - "webdevops/php-dev" -> "webdevops/php-apache-dev" - "webdevops/php-dev" } "webdevops/base-app" -> "webdevops/php" "webdevops/php-apache" -> "webdevops/typo3" "webdevops/base-app" -> "webdevops/ssh" + "webdevops/php-nginx" -> "webdevops/mail-sandbox" "webdevops/base-app" -> "webdevops/postfix" - "webdevops/base" -> "webdevops/vsftp" - "webdevops/base" -> "webdevops/nginx" "webdevops/base" -> "webdevops/apache" - "webdevops/base-app" -> "webdevops/hhvm" + busybox -> "webdevops/storage" + "webdevops/base" -> "webdevops/nginx" + "webdevops/base" -> "webdevops/vsftp" + java -> "webdevops/liquibase" + ubuntu -> "webdevops/bootstrap" "webdevops/bootstrap" -> "webdevops/sphinx" "webdevops/php-nginx" -> "webdevops/piwik" "webdevops/base" -> "webdevops/varnish" - "webdevops/php" -> "webdevops/php-dev" + "zendesk/samson" -> "webdevops/samson-deployment" + "webdevops/bootstrap" -> "webdevops/certbot" + "webdevops/base-app" -> "webdevops/hhvm" + { "busybox" -> "webdevops/ansible" [style=invis] } + { "busybox" -> "webdevops/base-app" [style=invis] } + { "busybox" -> "webdevops/storage" [style=invis] } + { "busybox" -> "webdevops/bootstrap" [style=invis] } + { "busybox" -> "webdevops/base" [style=invis] } + { "centos" -> "webdevops/ansible" [style=invis] } + { "centos" -> "webdevops/base-app" [style=invis] } + { "centos" -> "webdevops/storage" [style=invis] } + { "centos" -> "webdevops/bootstrap" [style=invis] } + { "centos" -> "webdevops/base" [style=invis] } + { "zendesk/samson" -> "webdevops/ansible" [style=invis] } + { "zendesk/samson" -> "webdevops/base-app" [style=invis] } + { "zendesk/samson" -> "webdevops/storage" [style=invis] } + { "zendesk/samson" -> "webdevops/bootstrap" [style=invis] } + { "zendesk/samson" -> "webdevops/base" [style=invis] } + { "java" -> "webdevops/ansible" [style=invis] } + { "java" -> "webdevops/base-app" [style=invis] } + { "java" -> "webdevops/storage" [style=invis] } + { "java" -> "webdevops/bootstrap" [style=invis] } + { "java" -> "webdevops/base" [style=invis] } + { "ubuntu" -> "webdevops/ansible" [style=invis] } + { "ubuntu" -> "webdevops/base-app" [style=invis] } + { "ubuntu" -> "webdevops/storage" [style=invis] } + { "ubuntu" -> "webdevops/bootstrap" [style=invis] } + { "ubuntu" -> "webdevops/base" [style=invis] } + { "debian" -> "webdevops/ansible" [style=invis] } + { "debian" -> "webdevops/base-app" [style=invis] } + { "debian" -> "webdevops/storage" [style=invis] } + { "debian" -> "webdevops/bootstrap" [style=invis] } + { "debian" -> "webdevops/base" [style=invis] } + { "alpine" -> "webdevops/ansible" [style=invis] } + { "alpine" -> "webdevops/base-app" [style=invis] } + { "alpine" -> "webdevops/storage" [style=invis] } + { "alpine" -> "webdevops/bootstrap" [style=invis] } + { "alpine" -> "webdevops/base" [style=invis] } + { "webdevops/ansible" -> "webdevops/typo3" [style=invis] } + { "webdevops/ansible" -> "webdevops/php-nginx-dev" [style=invis] } + { "webdevops/ansible" -> "webdevops/piwik" [style=invis] } + { "webdevops/ansible" -> "webdevops/php-nginx" [style=invis] } + { "webdevops/ansible" -> "webdevops/sphinx" [style=invis] } + { "webdevops/ansible" -> "webdevops/varnish" [style=invis] } + { "webdevops/ansible" -> "webdevops/nginx-dev" [style=invis] } + { "webdevops/ansible" -> "webdevops/ssh" [style=invis] } + { "webdevops/ansible" -> "webdevops/mail-sandbox" [style=invis] } + { "webdevops/ansible" -> "webdevops/vsftp" [style=invis] } + { "webdevops/ansible" -> "webdevops/nginx" [style=invis] } + { "webdevops/ansible" -> "webdevops/apache" [style=invis] } + { "webdevops/ansible" -> "webdevops/hhvm-nginx" [style=invis] } + { "webdevops/ansible" -> "webdevops/php-apache-dev" [style=invis] } + { "webdevops/ansible" -> "webdevops/php" [style=invis] } + { "webdevops/ansible" -> "webdevops/php-apache" [style=invis] } + { "webdevops/ansible" -> "webdevops/php-dev" [style=invis] } + { "webdevops/ansible" -> "webdevops/samson-deployment" [style=invis] } + { "webdevops/ansible" -> "webdevops/apache-dev" [style=invis] } + { "webdevops/ansible" -> "webdevops/hhvm-apache" [style=invis] } + { "webdevops/ansible" -> "webdevops/postfix" [style=invis] } + { "webdevops/ansible" -> "webdevops/liquibase" [style=invis] } + { "webdevops/ansible" -> "webdevops/hhvm" [style=invis] } + { "webdevops/ansible" -> "webdevops/certbot" [style=invis] } + { "webdevops/base-app" -> "webdevops/typo3" [style=invis] } + { "webdevops/base-app" -> "webdevops/php-nginx-dev" [style=invis] } + { "webdevops/base-app" -> "webdevops/piwik" [style=invis] } + { "webdevops/base-app" -> "webdevops/php-nginx" [style=invis] } + { "webdevops/base-app" -> "webdevops/sphinx" [style=invis] } + { "webdevops/base-app" -> "webdevops/varnish" [style=invis] } + { "webdevops/base-app" -> "webdevops/nginx-dev" [style=invis] } + { "webdevops/base-app" -> "webdevops/ssh" [style=invis] } + { "webdevops/base-app" -> "webdevops/mail-sandbox" [style=invis] } + { "webdevops/base-app" -> "webdevops/vsftp" [style=invis] } + { "webdevops/base-app" -> "webdevops/nginx" [style=invis] } + { "webdevops/base-app" -> "webdevops/apache" [style=invis] } + { "webdevops/base-app" -> "webdevops/hhvm-nginx" [style=invis] } + { "webdevops/base-app" -> "webdevops/php-apache-dev" [style=invis] } + { "webdevops/base-app" -> "webdevops/php" [style=invis] } + { "webdevops/base-app" -> "webdevops/php-apache" [style=invis] } + { "webdevops/base-app" -> "webdevops/php-dev" [style=invis] } + { "webdevops/base-app" -> "webdevops/samson-deployment" [style=invis] } + { "webdevops/base-app" -> "webdevops/apache-dev" [style=invis] } + { "webdevops/base-app" -> "webdevops/hhvm-apache" [style=invis] } + { "webdevops/base-app" -> "webdevops/postfix" [style=invis] } + { "webdevops/base-app" -> "webdevops/liquibase" [style=invis] } + { "webdevops/base-app" -> "webdevops/hhvm" [style=invis] } + { "webdevops/base-app" -> "webdevops/certbot" [style=invis] } + { "webdevops/storage" -> "webdevops/typo3" [style=invis] } + { "webdevops/storage" -> "webdevops/php-nginx-dev" [style=invis] } + { "webdevops/storage" -> "webdevops/piwik" [style=invis] } + { "webdevops/storage" -> "webdevops/php-nginx" [style=invis] } + { "webdevops/storage" -> "webdevops/sphinx" [style=invis] } + { "webdevops/storage" -> "webdevops/varnish" [style=invis] } + { "webdevops/storage" -> "webdevops/nginx-dev" [style=invis] } + { "webdevops/storage" -> "webdevops/ssh" [style=invis] } + { "webdevops/storage" -> "webdevops/mail-sandbox" [style=invis] } + { "webdevops/storage" -> "webdevops/vsftp" [style=invis] } + { "webdevops/storage" -> "webdevops/nginx" [style=invis] } + { "webdevops/storage" -> "webdevops/apache" [style=invis] } + { "webdevops/storage" -> "webdevops/hhvm-nginx" [style=invis] } + { "webdevops/storage" -> "webdevops/php-apache-dev" [style=invis] } + { "webdevops/storage" -> "webdevops/php" [style=invis] } + { "webdevops/storage" -> "webdevops/php-apache" [style=invis] } + { "webdevops/storage" -> "webdevops/php-dev" [style=invis] } + { "webdevops/storage" -> "webdevops/samson-deployment" [style=invis] } + { "webdevops/storage" -> "webdevops/apache-dev" [style=invis] } + { "webdevops/storage" -> "webdevops/hhvm-apache" [style=invis] } + { "webdevops/storage" -> "webdevops/postfix" [style=invis] } + { "webdevops/storage" -> "webdevops/liquibase" [style=invis] } + { "webdevops/storage" -> "webdevops/hhvm" [style=invis] } + { "webdevops/storage" -> "webdevops/certbot" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/typo3" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/php-nginx-dev" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/piwik" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/php-nginx" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/sphinx" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/varnish" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/nginx-dev" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/ssh" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/mail-sandbox" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/vsftp" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/nginx" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/apache" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/hhvm-nginx" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/php-apache-dev" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/php" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/php-apache" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/php-dev" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/samson-deployment" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/apache-dev" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/hhvm-apache" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/postfix" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/liquibase" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/hhvm" [style=invis] } + { "webdevops/bootstrap" -> "webdevops/certbot" [style=invis] } + { "webdevops/base" -> "webdevops/typo3" [style=invis] } + { "webdevops/base" -> "webdevops/php-nginx-dev" [style=invis] } + { "webdevops/base" -> "webdevops/piwik" [style=invis] } + { "webdevops/base" -> "webdevops/php-nginx" [style=invis] } + { "webdevops/base" -> "webdevops/sphinx" [style=invis] } + { "webdevops/base" -> "webdevops/varnish" [style=invis] } + { "webdevops/base" -> "webdevops/nginx-dev" [style=invis] } + { "webdevops/base" -> "webdevops/ssh" [style=invis] } + { "webdevops/base" -> "webdevops/mail-sandbox" [style=invis] } + { "webdevops/base" -> "webdevops/vsftp" [style=invis] } + { "webdevops/base" -> "webdevops/nginx" [style=invis] } + { "webdevops/base" -> "webdevops/apache" [style=invis] } + { "webdevops/base" -> "webdevops/hhvm-nginx" [style=invis] } + { "webdevops/base" -> "webdevops/php-apache-dev" [style=invis] } + { "webdevops/base" -> "webdevops/php" [style=invis] } + { "webdevops/base" -> "webdevops/php-apache" [style=invis] } + { "webdevops/base" -> "webdevops/php-dev" [style=invis] } + { "webdevops/base" -> "webdevops/samson-deployment" [style=invis] } + { "webdevops/base" -> "webdevops/apache-dev" [style=invis] } + { "webdevops/base" -> "webdevops/hhvm-apache" [style=invis] } + { "webdevops/base" -> "webdevops/postfix" [style=invis] } + { "webdevops/base" -> "webdevops/liquibase" [style=invis] } + { "webdevops/base" -> "webdevops/hhvm" [style=invis] } + { "webdevops/base" -> "webdevops/certbot" [style=invis] } } \ No newline at end of file diff --git a/documentation/docs/resources/images/docker-image-layout.gv.png b/documentation/docs/resources/images/docker-image-layout.gv.png index 579f78a10..1b5d91359 100644 Binary files a/documentation/docs/resources/images/docker-image-layout.gv.png and b/documentation/docs/resources/images/docker-image-layout.gv.png differ diff --git a/provisioning/base-app/general/bin/service.d/postfix.d/10-init.sh b/provisioning/base-app/general/bin/service.d/postfix.d/10-init.sh index 923002f39..9dc7c8af2 100644 --- a/provisioning/base-app/general/bin/service.d/postfix.d/10-init.sh +++ b/provisioning/base-app/general/bin/service.d/postfix.d/10-init.sh @@ -4,10 +4,14 @@ cp -f /etc/hosts /var/spool/postfix/etc/hosts cp -f /etc/resolv.conf /var/spool/postfix/etc/resolv.conf cp -f /etc/services /var/spool/postfix/etc/services -if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; else +if [[ -n "${POSTFIX_RELAYHOST+x}" ]]; then + # replace line + sed -i '/relayhost[ ]* =/c\' main.cf echo "relayhost = $POSTFIX_RELAYHOST" >> /etc/postfix/main.cf fi -if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; else +if [[ -n "${POSTFIX_MYNETWORKS+x}" ]]; then + # replace line + sed -i '/mynetworks[ ]* =/c\' main.cf echo "mynetworks = $POSTFIX_MYNETWORKS" >> /etc/postfix/main.cf fi diff --git a/provisioning/base/general/provision/roles/webdevops-base/tasks/bootstrap.yml b/provisioning/base/general/provision/roles/webdevops-base/tasks/bootstrap.yml index 297ec7128..626ead4af 100644 --- a/provisioning/base/general/provision/roles/webdevops-base/tasks/bootstrap.yml +++ b/provisioning/base/general/provision/roles/webdevops-base/tasks/bootstrap.yml @@ -1,6 +1,7 @@ --- - include: bootstrap/misc.yml +- include: bootstrap/root.yml - include: bootstrap/supervisor.yml - include: bootstrap/syslog-ng.yml - include: bootstrap/logrotate.yml diff --git a/provisioning/base/general/provision/roles/webdevops-base/tasks/bootstrap/root.yml b/provisioning/base/general/provision/roles/webdevops-base/tasks/bootstrap/root.yml new file mode 100644 index 000000000..29e955fa4 --- /dev/null +++ b/provisioning/base/general/provision/roles/webdevops-base/tasks/bootstrap/root.yml @@ -0,0 +1,5 @@ +--- + +- file: + path: "/root/.profile" + state: absent diff --git a/provisioning/mail-sandbox/general/etc/roundcube/config.php b/provisioning/mail-sandbox/general/etc/roundcube/config.php new file mode 100644 index 000000000..2699a3ef6 --- /dev/null +++ b/provisioning/mail-sandbox/general/etc/roundcube/config.php @@ -0,0 +1,18 @@ + [ + 'verify_peer' => false, + 'verify_peer_name' => false, + 'allow_self_signed' => true, + ], +]; +$config['default_port'] = 143; +$config['smtp_port'] = 25; +$config['smtp_user'] = '%u'; +$config['smtp_pass'] = '%p'; + +$config['plugins'][] = 'webdevops_autologin'; diff --git a/provisioning/mail-sandbox/general/etc/roundcube/plugins/webdevops_autologin/webdevops_autologin.php b/provisioning/mail-sandbox/general/etc/roundcube/plugins/webdevops_autologin/webdevops_autologin.php new file mode 100644 index 000000000..97fbd5859 --- /dev/null +++ b/provisioning/mail-sandbox/general/etc/roundcube/plugins/webdevops_autologin/webdevops_autologin.php @@ -0,0 +1,32 @@ +add_hook('startup', array($this, 'startup')); + $this->add_hook('authenticate', array($this, 'authenticate')); + } + + function startup($args) + { + // change action to login + if (empty($_SESSION['user_id'])) { + $args['action'] = 'login'; + } + + return $args; + } + + function authenticate($args) { + $args['user'] = getenv('MAILBOX_USERNAME'); + $args['pass'] = getenv('MAILBOX_PASSWORD'); + $args['host'] = 'localhost'; + $args['cookiecheck'] = false; + $args['valid'] = true; + + return $args; + } +} diff --git a/provisioning/mail-sandbox/general/provision/roles/webdevops-mail-sandbox/tasks/bootstrap/postfix.yml b/provisioning/mail-sandbox/general/provision/roles/webdevops-mail-sandbox/tasks/bootstrap/postfix.yml index 52e0f82a6..044d0f372 100644 --- a/provisioning/mail-sandbox/general/provision/roles/webdevops-mail-sandbox/tasks/bootstrap/postfix.yml +++ b/provisioning/mail-sandbox/general/provision/roles/webdevops-mail-sandbox/tasks/bootstrap/postfix.yml @@ -14,6 +14,11 @@ - { variable: 'mydestination', value: 'localhost' } - { variable: 'message_size_limit', value: '102400000' } +- name: Postfix configuration (listen on port 1025) + lineinfile: + dest: /etc/postfix/master.cf + line: '1025 inet n - y - - smtpd' + - name: Create /etc/mailname raw: 'hostname > /etc/mailname' args: diff --git a/requirements.txt b/requirements.txt index cd9ffb84b..8d2f3adab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,9 @@ -Jinja2>=2.8 -graphviz>=0.4.10 \ No newline at end of file +Jinja2>=2.8 +graphviz>=0.4.10 +cleo>=0.5.0 +yamlordereddictloader>=0.1.0 +testinfra>=1.4.2 +doit>=0.29.0 +termcolor>=1.1.0 +pytest-timeout>=1.0.0 +pytest-rerunfailures>=1.0.0 diff --git a/template/Dockerfile/environment.jinja2 b/template/Dockerfile/environment.jinja2 index 375567f67..a9e3529a8 100644 --- a/template/Dockerfile/environment.jinja2 +++ b/template/Dockerfile/environment.jinja2 @@ -9,8 +9,8 @@ ENV DOCKER_CONF_HOME /opt/docker/ {%- endmacro %} {% macro mailbox() -%} -ENV MAILBOX_USERNAME "sandbox" -ENV MAILBOX_PASSWORD "mail" +ENV MAILBOX_USERNAME "dev" +ENV MAILBOX_PASSWORD "dev" {%- endmacro %} {% macro web() -%} diff --git a/template/Dockerfile/images/baseapp.jinja2 b/template/Dockerfile/images/baseapp.jinja2 index dec987a1c..8b5c91483 100644 --- a/template/Dockerfile/images/baseapp.jinja2 +++ b/template/Dockerfile/images/baseapp.jinja2 @@ -1,7 +1,8 @@ {% import 'Dockerfile/provision.jinja2' as provision %} {% macro alpine(role='') -%} -RUN /usr/local/bin/apk-install \ +RUN /usr/local/bin/apk-install-edge shadow \ + && /usr/local/bin/apk-install \ # Install services openssh \ mysql-client \ diff --git a/template/Dockerfile/images/bootstrap.jinja2 b/template/Dockerfile/images/bootstrap.jinja2 index 092a12264..d9bb61330 100644 --- a/template/Dockerfile/images/bootstrap.jinja2 +++ b/template/Dockerfile/images/bootstrap.jinja2 @@ -4,7 +4,9 @@ RUN set -x \ # Fix root terminal && echo "export TERM=xterm" >> /root/.bashrc \ # Add testing - && echo http://nl.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories \ + && echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories \ + # Add community + && echo http://dl-cdn.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories \ # System update && /usr/local/bin/apk-upgrade \ # Install base stuff diff --git a/template/Dockerfile/images/certbot.jinja2 b/template/Dockerfile/images/certbot.jinja2 index d2bf81b73..350155d15 100644 --- a/template/Dockerfile/images/certbot.jinja2 +++ b/template/Dockerfile/images/certbot.jinja2 @@ -7,5 +7,6 @@ ENV CERTBOT_DOMAIN "" {% macro alpine() -%} RUN /usr/local/bin/apk-install \ - certbot -{%- endmacro %} \ No newline at end of file + certbot \ + && pip install pyRFC3339 configobj ConfigArgParse +{%- endmacro %} diff --git a/template/Dockerfile/images/hhvm.jinja2 b/template/Dockerfile/images/hhvm.jinja2 index 9d2a2ad92..6fd9bb196 100644 --- a/template/Dockerfile/images/hhvm.jinja2 +++ b/template/Dockerfile/images/hhvm.jinja2 @@ -9,6 +9,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && set -x \ hhvm \ imagemagick \ graphicsmagick \ + ghostscript \ && /usr/bin/update-alternatives --install /usr/bin/php php /usr/bin/hhvm 60 \ && curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin/ --filename=composer \ {{ provision.runRoleInline('hhvm', role) }} @@ -21,6 +22,7 @@ RUN /usr/local/bin/apt-install \ hhvm \ imagemagick \ graphicsmagick \ + ghostscript \ && /usr/bin/update-alternatives --install /usr/bin/php php /usr/bin/hhvm 60 \ && curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin/ --filename=composer \ {{ provision.runRoleInline('hhvm', role) }} diff --git a/template/Dockerfile/images/mail-sandbox.jinja2 b/template/Dockerfile/images/mail-sandbox.jinja2 index 4fc0f1d8e..62536f146 100644 --- a/template/Dockerfile/images/mail-sandbox.jinja2 +++ b/template/Dockerfile/images/mail-sandbox.jinja2 @@ -7,5 +7,19 @@ RUN /usr/local/bin/apt-install \ dovecot-imapd \ && /opt/docker/bin/control.sh service.enable postfix \ && /opt/docker/bin/control.sh service.enable dovecot \ + && /opt/docker/bin/provision add --tag entrypoint webdevops-mail-sandbox \ {{ provision.runRoleInline('mail-sandbox', role) }} {%- endmacro %} + +{% macro roundcube(role='') -%} +# Install Roundcube + plugins +RUN cd /app \ + && rm -rf * \ + && wget https://github.com/roundcube/roundcubemail/releases/download/1.2.2/roundcubemail-1.2.2-complete.tar.gz \ + && tar xf roundcubemail-1.2.2-complete.tar.gz --strip-components 1 \ + && rm -f roundcubemail-1.2.2-complete.tar.gz \ + && ls -l \ + && rm -rf .git installer \ + && ln -s /opt/docker/etc/roundcube/plugins/webdevops_autologin/ plugins/webdevops_autologin \ + && ln -s /opt/docker/etc/roundcube/config.php config/config.inc.php +{%- endmacro %} diff --git a/template/Dockerfile/images/php5.jinja2 b/template/Dockerfile/images/php5.jinja2 index c945fd2a1..efc57491b 100644 --- a/template/Dockerfile/images/php5.jinja2 +++ b/template/Dockerfile/images/php5.jinja2 @@ -6,6 +6,7 @@ RUN /usr/local/bin/apk-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ @@ -67,6 +68,7 @@ RUN /usr/local/bin/yum-install \ # Install tools ImageMagick \ GraphicsMagick \ + ghostscript \ # Install php (cli/fpm) php-cli \ php-fpm \ @@ -109,6 +111,7 @@ RUN rpm -Uvh https://mirror.webtatic.com/yum/el7/webtatic-release.rpm \ # Install tools ImageMagick \ GraphicsMagick \ + ghostscript \ # Install php (cli/fpm) php56w-cli \ php56w-fpm \ @@ -148,6 +151,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ @@ -182,6 +186,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ @@ -218,6 +223,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ @@ -252,6 +258,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ @@ -287,6 +294,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ @@ -322,6 +330,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php5-cli \ php5-fpm \ diff --git a/template/Dockerfile/images/php7.jinja2 b/template/Dockerfile/images/php7.jinja2 index 35dece2cc..f04bfa206 100644 --- a/template/Dockerfile/images/php7.jinja2 +++ b/template/Dockerfile/images/php7.jinja2 @@ -6,6 +6,7 @@ RUN /usr/local/bin/apk-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7-fpm \ php7-json \ @@ -74,6 +75,7 @@ RUN echo "deb http://packages.dotdeb.org {{ distribution }} all" >> /etc/apt/sou # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7.0-cli \ php7.0-fpm \ @@ -114,6 +116,7 @@ RUN /usr/local/bin/apt-install apt-transport-https lsb-release \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7.0-cli \ php7.0-fpm \ @@ -146,6 +149,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7.0-cli \ php7.0-fpm \ @@ -179,6 +183,7 @@ RUN /usr/local/bin/apt-install \ # Install tools imagemagick \ graphicsmagick \ + ghostscript \ # Install php (cli/fpm) php7.0-cli \ php7.0-fpm \ diff --git a/test/Makefile b/test/Makefile deleted file mode 100644 index 6aeca0fd9..000000000 --- a/test/Makefile +++ /dev/null @@ -1,82 +0,0 @@ -ARGS = $(filter-out $@,$(MAKECMDGOALS)) -MAKEFLAGS += --silent -DOCKER_REPOSITORY=`cat DOCKER_REPOSITORY` -DOCKER_TAG_LATEST=`cat DOCKER_TAG_LATEST` - -list: - sh -c "echo; $(MAKE) -p no_targets__ | awk -F':' '/^[a-zA-Z0-9][^\$$#\\t=]*:([^=]|$$)/ {split(\$$1,A,/ /);for(i in A)print A[i]}' | grep -v '__\$$' | grep -v 'Makefile'| sort" - -all: - bash ./run.sh - -bootstrap: - bash ./run.sh bootstrap - -ansible: - bash ./run.sh ansible - -base: - bash ./run.sh base - -base-app: - bash ./run.sh base-app - -php: - bash ./run.sh php - -php-dev: - bash ./run.sh php-dev - -apache: - bash ./run.sh apache - -nginx: - bash ./run.sh nginx - -php-apache: - bash ./run.sh php-apache - -php-apache-dev: - bash ./run.sh php-apache-dev - -php-nginx: - bash ./run.sh php-nginx - -php-nginx-dev: - bash ./run.sh php-nginx-dev - -hhvm: - bash ./run.sh hhvm - -hhvm-apache: - bash ./run.sh hhvm-apache - -hhvm-nginx: - bash ./run.sh hhvm-nginx - -postfix: - bash ./run.sh postfix - -vsftp: - bash ./run.sh vsftp - -mail-sandbox: - bash ./run.sh mail-sandbox - -samson-deployment: - bash ./run.sh samson-deployment - -ssh: - bash ./run.sh ssh - -varnish: - bash ./run.sh varnish - -sphinx: - bash ./run.sh sphinx - -certbot: - bash ./run.sh certbot - -liquibase: - bash ./run.sh liquibase diff --git a/test/run.sh b/test/run.sh deleted file mode 100755 index f543d2dad..000000000 --- a/test/run.sh +++ /dev/null @@ -1,999 +0,0 @@ -#!/usr/bin/env bash - -if [ -n "$1" ]; then - TEST_TARGET="$1" -else - TEST_TARGET="all" -fi - -if [ -z "$DOCKER_PULL" ]; then - DOCKER_PULL=0 -fi - -if [ -z "$FAST" ]; then - FAST=1 -fi - -if [ -z "$DEBUG" ]; then - DEBUG=0 -fi - -set -o pipefail # trace ERR through pipes -set -o errtrace # trace ERR through 'time command' and other functions -set -o nounset ## set -u : exit the script if you try to use an uninitialised variable -set -o errexit ## set -e : exit the script if any statement returns a non-true return value - -READLINK='readlink' - -[[ `uname` == 'Darwin' ]] && { - which greadlink > /dev/null && { - READLINK='greadlink' - } || { - echo 'ERROR: GNU utils required for Mac. You may use homebrew to install them: brew install coreutils gnu-sed' - exit 1 - } -} - -SLEEP_TIME=1 -SCRIPT_DIR="$(dirname $($READLINK -f "$0"))" -BASE_DIR="$(dirname "$SCRIPT_DIR")" -COLUMNS="$(tput cols)" - -DOCKERFILE_EXTRA="" - -source "${BASE_DIR}/bin/functions.sh" - -cd "$SCRIPT_DIR" - -OS_VERSION="" - -DOCKER_REPOSITORY="$(cat "$BASE_DIR/DOCKER_REPOSITORY")" -DOCKER_TAG_LATEST="$(cat "$BASE_DIR/DOCKER_TAG_LATEST")" - -### - # Relative dir - # - # $1 -> build target (eg. "bootstrap", "base", "php" ...) - # stdout -> "1" if target is matched - # - ## -function checkTestTarget() { - if [ "$TEST_TARGET" == "all" -o "$TEST_TARGET" == "$1" ]; then - echo 1 - fi -} - - -### - # Init Environment - # - ## -function initEnvironment() { - bundle install --path=vendor -} - -### - # Run test for docker image tag - # - # $1 -> Docker tag - # $2 -> Spec file path - # - ## -function runTestForTag() { - DOCKER_TAG="$1" - DOCKER_IMAGE_WITH_TAG="${DOCKER_IMAGE}:${DOCKER_TAG}" - - if [[ -z "$(checkBlacklist "${DOCKER_IMAGE_WITH_TAG}")" ]]; then - return - fi - - if [ "$DOCKER_PULL" -eq 1 ]; then - echo " * Pulling $DOCKER_IMAGE_WITH_TAG from Docker hub ..." - docker pull "$DOCKER_IMAGE_WITH_TAG" - fi - - setupDockerTagEnvironment "$DOCKER_TAG" - - DOCKERFILE="Dockerfile.${DOCKER_IMAGE//\//-}-${DOCKER_TAG}.test" - - ## Build Dockerfile - echo "FROM $DOCKER_IMAGE_WITH_TAG -$DOCKERFILE_EXTRA -COPY conf/ /" > $DOCKERFILE - - - if [ "${DEBUG}" -eq 1 ]; then - echo "DOCKERFILE:" - echo "-----------" - echo ">>>" - cat "$DOCKERFILE" | sed 's/^/ /' - echo ">>>" - echo "" - fi - - if [ "${FAST}" -eq 1 ]; then - LOGFILE="$(mktemp /tmp/docker.test.XXXXXXXXXX)" - - echo ">> Starting test of ${DOCKER_IMAGE_WITH_TAG}" - - # Run testsuite for docker image - DOCKERFILE="$DOCKERFILE" OS_FAMILY="$OS_FAMILY" OS_VERSION="$OS_VERSION" DOCKER_IMAGE="$DOCKER_IMAGE_WITH_TAG" bundle exec rspec --pattern " spec/image.rb,$SPEC_PATH" &> $LOGFILE & - - addBackgroundPidToList "Test '$DOCKER_TAG' with spec '$(basename "$SPEC_PATH" _spec.rb)' [family: $OS_FAMILY, version: $OS_VERSION]" "$LOGFILE" - sleep "$SLEEP_TIME" - else - echo ">> Testing '$DOCKER_TAG' with spec '$(basename "$SPEC_PATH" _spec.rb)' [family: $OS_FAMILY, version: $OS_VERSION]" - - # Run testsuite for docker image - DOCKERFILE="$DOCKERFILE" OS_FAMILY="$OS_FAMILY" OS_VERSION="$OS_VERSION" DOCKER_IMAGE="$DOCKER_IMAGE_WITH_TAG" bundle exec rspec --pattern " spec/image.rb,$SPEC_PATH" - fi -} - -function waitForTestRun() { - if [ "${FAST}" -eq 1 ]; then - echo " -> waiting for background testing process..." - ALWAYS_SHOW_LOGS=1 waitForBackgroundProcesses - fi - - rm -f Dockerfile.*.test -} - -### - # Set environment OS_FAMILY - # - # $1 -> Distribution name for testing - # - ## -function setEnvironmentOsFamily() { - export OS_FAMILY="$1" -} - -### - # Set test spec test file - # - # $1 -> Target filename for spec - # - ## -function setSpecTest() { - ## Set test spec path - SPEC_PATH="spec/docker/${1}_spec.rb" -} - -### - # Setup test environment - # - # Sets DOCKER_IMAGE, SPEC_PATH and OS_FAMILY - # - # $1 -> Docker image name (without namespace) - # - ## -function setupTestEnvironment() { - echo "" - printRepeatedChar "=" - echo "=== Testing docker image $DOCKER_REPOSITORY/$1" - printRepeatedChar "=" - echo "" - - ## Set docker image - DOCKER_IMAGE="$DOCKER_REPOSITORY/$1" - - ## Set test spec path - setSpecTest "$1" - - ## Set default environment - setEnvironmentOsFamily "ubuntu" - - ## Reset custom docker environment settings - DOCKERFILE_CONF="" - DOCKERFILE_EXTRA="" -} - -function printRepeatedChar() { - printf "${1}%.0s" $(seq 1 50) - echo -} - -### - # Switch environment variables for test - # - # $1 -> Docker tag - # - ## -function setupDockerTagEnvironment() { - unset PHP_REDIS - unset PHP_APCU - unset PHP_XDEBUG - unset PHP_MHASH - - case "$1" in - ubuntu-12.04) - export PHP_REDIS=0 - export PHP_APCU=0 - ;; - - debian-7) - export PHP_REDIS=0 - export PHP_APCU=0 - ;; - - debian-8-php7) - export PHP_XDEBUG=0 - ;; - - alpine-3) - export PHP_MHASH=0 - ;; - - centos-7-php56) - export PHP_APCU=0 - export PHP_REDIS=0 - ;; - esac -} - -initEnvironment - -####################################### -# webdevops/bootstrap -####################################### - -[[ $(checkTestTarget bootstrap) ]] && { - setupTestEnvironment "bootstrap" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun -} - -####################################### -# webdevops/ansible -####################################### - -[[ $(checkTestTarget ansible) ]] && { - setupTestEnvironment "ansible" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun -} - -####################################### -# webdevops/base -####################################### - -[[ $(checkTestTarget base) ]] && { - setupTestEnvironment "base" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun -} - - -####################################### -# webdevops/base -####################################### - -[[ $(checkTestTarget base-app) ]] && { - setupTestEnvironment "base-app" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun -} - -####################################### -# webdevops/php -####################################### - -[[ $(checkTestTarget php) ]] && { - setupTestEnvironment "php" - - ########## - # PHP 5 - ########## - - setSpecTest "php5" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - OS_VERSION="7" runTestForTag "centos-7-php56" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun - - ########## - # PHP 7 - ########## - - setSpecTest "php7" - - setEnvironmentOsFamily "ubuntu" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "debian" - OS_VERSION="8" runTestForTag "debian-8-php7" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3-php7" - - waitForTestRun -} - -####################################### -# webdevops/php-dev -####################################### - -[[ $(checkTestTarget php-dev) ]] && { - setupTestEnvironment "php-dev" - - ############################# - # XDEBUG - ############################# - - ########## - # PHP 5 - ########## - - setSpecTest "php5-dev" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - OS_VERSION="7" runTestForTag "centos-7-php56" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun - - ########## - # PHP 7 - ########## - - setSpecTest "php7-dev" - - setEnvironmentOsFamily "ubuntu" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "debian" - OS_VERSION="8" runTestForTag "debian-8-php7" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3-php7" - - waitForTestRun -} - -[[ $(checkTestTarget php-dev-blackfire) ]] && { - setupTestEnvironment "php-dev" - - ############################# - # BLACKFIRE - ############################# - - export PHP_BLACKFIRE=1 - - ########## - # PHP 5 - ########## - - setSpecTest "php5-dev" - - DOCKERFILE_EXTRA=" -ENV PHP_DEBUGGER \"blackfire\" -" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - OS_VERSION="7" runTestForTag "centos-7-php56" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - - # blackfire not supported on alpine - #setEnvironmentOsFamily "alpine" - #OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun - - ########## - # PHP 7 - ########## - - setSpecTest "php7-dev" - - DOCKERFILE_EXTRA=" -ENV PHP_DEBUGGER \"blackfire\" -" - - setEnvironmentOsFamily "ubuntu" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "debian" - OS_VERSION="8" runTestForTag "debian-8-php7" - OS_VERSION="testing" runTestForTag "debian-9" - - # blackfire not supported on alpine - #setEnvironmentOsFamily "alpine" - #OS_VERSION="3" runTestForTag "alpine-3-php7" - - waitForTestRun - - export PHP_BLACKFIRE=0 -} - - -####################################### -# webdevops/apache -####################################### - -[[ $(checkTestTarget apache) ]] && { - setupTestEnvironment "apache" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun -} - -####################################### -# webdevops/apache-dev -####################################### - -[[ $(checkTestTarget apache-dev) ]] && { - setupTestEnvironment "apache-dev" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun -} - -####################################### -# webdevops/nginx -####################################### - -[[ $(checkTestTarget nginx) ]] && { - setupTestEnvironment "nginx" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun -} - -####################################### -# webdevops/nginx-dev -####################################### - -[[ $(checkTestTarget nginx-dev) ]] && { - setupTestEnvironment "nginx-dev" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun -} - - -####################################### -# webdevops/php-apache -####################################### - -[[ $(checkTestTarget php-apache) ]] && { - setupTestEnvironment "php-apache" - - ########## - # PHP 5 - ########## - - setSpecTest "php5-apache" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - OS_VERSION="7" runTestForTag "centos-7-php56" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun - - ########## - # PHP 7 - ########## - - setEnvironmentOsFamily "ubuntu" - setSpecTest "php7-apache" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "debian" - OS_VERSION="8" runTestForTag "debian-8-php7" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3-php7" - - waitForTestRun -} - - -####################################### -# webdevops/php-apache-dev -####################################### - -[[ $(checkTestTarget php-apache-dev) ]] && { - setupTestEnvironment "php-apache-dev" - - ########## - # PHP 5 - ########## - - setSpecTest "php5-apache-dev" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - OS_VERSION="7" runTestForTag "centos-7-php56" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun - - ########## - # PHP 7 - ########## - setSpecTest "php7-apache-dev" - - setEnvironmentOsFamily "ubuntu" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "debian" - OS_VERSION="8" runTestForTag "debian-8-php7" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3-php7" - - waitForTestRun -} - -####################################### -# webdevops/php-nginx -####################################### - -[[ $(checkTestTarget php-nginx) ]] && { - setupTestEnvironment "php-nginx" - - ########## - # PHP 5 - ########## - - setSpecTest "php5-nginx" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - OS_VERSION="7" runTestForTag "centos-7-php56" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun - - ########## - # PHP 7 - ########## - - setSpecTest "php7-nginx" - - setEnvironmentOsFamily "ubuntu" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "debian" - OS_VERSION="8" runTestForTag "debian-8-php7" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3-php7" - - waitForTestRun -} - - -####################################### -# webdevops/php-nginx-dev -####################################### - -[[ $(checkTestTarget php-nginx-dev) ]] && { - setupTestEnvironment "php-nginx-dev" - - ########## - # PHP 5 - ########## - - setSpecTest "php5-nginx-dev" - - OS_VERSION="12.04" runTestForTag "ubuntu-12.04" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="15.04" runTestForTag "ubuntu-15.04" - OS_VERSION="15.10" runTestForTag "ubuntu-15.10" - - setEnvironmentOsFamily "redhat" - OS_VERSION="7" runTestForTag "centos-7" - OS_VERSION="7" runTestForTag "centos-7-php56" - - setEnvironmentOsFamily "debian" - OS_VERSION="7" runTestForTag "debian-7" - OS_VERSION="8" runTestForTag "debian-8" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3" - - waitForTestRun - - ########## - # PHP 7 - ########## - setSpecTest "php7-nginx-dev" - - setEnvironmentOsFamily "ubuntu" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - setEnvironmentOsFamily "debian" - OS_VERSION="8" runTestForTag "debian-8-php7" - OS_VERSION="testing" runTestForTag "debian-9" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "alpine-3-php7" - - waitForTestRun -} - -####################################### -# webdevops/hhvm -####################################### - -[[ $(checkTestTarget hhvm) ]] && { - setupTestEnvironment "hhvm" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - - waitForTestRun -} - -####################################### -# webdevops/hhvm-apache -####################################### - -[[ $(checkTestTarget hhvm-apache) ]] && { - setupTestEnvironment "hhvm-apache" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - waitForTestRun -} - - -####################################### -# webdevops/hhvm-nginx -####################################### - -[[ $(checkTestTarget hhvm-nginx) ]] && { - setupTestEnvironment "hhvm-nginx" - OS_VERSION="14.04" runTestForTag "ubuntu-14.04" - OS_VERSION="16.04" runTestForTag "ubuntu-16.04" - - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/postfix -####################################### - -[[ $(checkTestTarget postfix) ]] && { - setupTestEnvironment "postfix" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/vsftp -####################################### - -[[ $(checkTestTarget vsftp) ]] && { - setupTestEnvironment "vsftp" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/mail-sandbox -####################################### - -[[ $(checkTestTarget mail-sandbox) ]] && { - setupTestEnvironment "mail-sandbox" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/ssh -####################################### - -[[ $(checkTestTarget ssh) ]] && { - setupTestEnvironment "ssh" - OS_VERSION="$DOCKER_TAG_LATEST" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/varnish -####################################### - -[[ $(checkTestTarget varnish) ]] && { - setupTestEnvironment "varnish" - - DOCKERFILE_EXTRA=" -ENV VARNISH_BACKEND_HOST \"google.com\" -" - - setEnvironmentOsFamily "alpine" - OS_VERSION="3" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/sphinx -####################################### - -[[ $(checkTestTarget sphinx) ]] && { - setupTestEnvironment "sphinx" - setEnvironmentOsFamily "alpine" - - OS_VERSION="3" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/samson-deployment -####################################### - -[[ $(checkTestTarget samson-deployment) ]] && { - setupTestEnvironment "samson-deployment" - setEnvironmentOsFamily "debian" - - OS_VERSION="8" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/certbot -####################################### - -[[ $(checkTestTarget certbot) ]] && { - setupTestEnvironment "certbot" - - # setSpecTest "base" - setEnvironmentOsFamily "alpine" - - OS_VERSION="3" runTestForTag "latest" - - waitForTestRun -} - -####################################### -# webdevops/liquibase -####################################### - -#[[ $(checkTestTarget liquibase) ]] && { -# setupTestEnvironment "liquibase" -# -# # setSpecTest "base" -# setEnvironmentOsFamily "debian" -# -# OS_VERSION="3" runTestForTag "latest" -# -# waitForTestRun -#} - -echo "" -echo " >>> finished, all tests PASSED <<<" -echo "" diff --git a/test/spec/docker/php5-apache-dev_spec.rb b/test/spec/docker/php5-apache-dev_spec.rb deleted file mode 100644 index 5061cc7ae..000000000 --- a/test/spec/docker/php5-apache-dev_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php5::development' - include_examples 'collection::php-fpm5' - include_examples 'collection::php-fpm5::public' - include_examples 'collection::php-tools' - include_examples 'collection::apache' - include_examples 'collection::php-fpm5::webserver-test::development' - -end diff --git a/test/spec/docker/php5-apache_spec.rb b/test/spec/docker/php5-apache_spec.rb deleted file mode 100644 index ed83bd830..000000000 --- a/test/spec/docker/php5-apache_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php5::production' - include_examples 'collection::php-fpm5' - include_examples 'collection::php-fpm5::local-only' - include_examples 'collection::apache' - include_examples 'collection::php-fpm5::webserver-test::production' - -end diff --git a/test/spec/docker/php5-dev_spec.rb b/test/spec/docker/php5-dev_spec.rb deleted file mode 100644 index 9f8024d57..000000000 --- a/test/spec/docker/php5-dev_spec.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php5::development' - include_examples 'collection::php-fpm5' - include_examples 'collection::php-fpm5::public' - include_examples 'collection::php-tools' - -end diff --git a/test/spec/docker/php5-nginx-dev_spec.rb b/test/spec/docker/php5-nginx-dev_spec.rb deleted file mode 100644 index a68e34bd1..000000000 --- a/test/spec/docker/php5-nginx-dev_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php5::development' - include_examples 'collection::php-fpm5' - include_examples 'collection::php-fpm5::public' - include_examples 'collection::php-tools' - include_examples 'collection::nginx' - include_examples 'collection::php-fpm5::webserver-test::development' - -end diff --git a/test/spec/docker/php5-nginx_spec.rb b/test/spec/docker/php5-nginx_spec.rb deleted file mode 100644 index e21cae1da..000000000 --- a/test/spec/docker/php5-nginx_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php5::production' - include_examples 'collection::php-fpm5' - include_examples 'collection::php-fpm5::local-only' - include_examples 'collection::nginx' - include_examples 'collection::php-fpm5::webserver-test::production' - -end diff --git a/test/spec/docker/php7-apache_spec.rb b/test/spec/docker/php7-apache_spec.rb deleted file mode 100644 index 80b030e0d..000000000 --- a/test/spec/docker/php7-apache_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php7::production' - include_examples 'collection::php-fpm7' - include_examples 'collection::php-fpm7::local-only' - include_examples 'collection::apache' - include_examples 'collection::php-fpm7::webserver-test::production' - -end diff --git a/test/spec/docker/php7-dev_spec.rb b/test/spec/docker/php7-dev_spec.rb deleted file mode 100644 index ad04b477a..000000000 --- a/test/spec/docker/php7-dev_spec.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php7::development' - include_examples 'collection::php-fpm7' - include_examples 'collection::php-fpm7::public' - include_examples 'collection::php-tools' - -end diff --git a/test/spec/docker/php7-nginx-dev_spec.rb b/test/spec/docker/php7-nginx-dev_spec.rb deleted file mode 100644 index 9838c8b12..000000000 --- a/test/spec/docker/php7-nginx-dev_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php7::development' - include_examples 'collection::php-fpm7' - include_examples 'collection::php-fpm7::public' - include_examples 'collection::php-tools' - include_examples 'collection::nginx' - include_examples 'collection::php-fpm7::webserver-test::development' - -end diff --git a/test/spec/docker/php7-nginx_spec.rb b/test/spec/docker/php7-nginx_spec.rb deleted file mode 100644 index 96c3a3818..000000000 --- a/test/spec/docker/php7-nginx_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'serverspec' -require 'docker' -require 'spec_helper' - -describe "Dockerfile" do - before(:all) do - @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) - set :docker_image, @image.id - end - - include_examples 'collection::bootstrap' - include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php7::production' - include_examples 'collection::php-fpm7' - include_examples 'collection::php-fpm7::local-only' - include_examples 'collection::nginx' - include_examples 'collection::php-fpm7::webserver-test::production' - -end diff --git a/test/spec/shared/misc/tools.rb b/test/spec/shared/misc/tools.rb deleted file mode 100644 index 831b85707..000000000 --- a/test/spec/shared/misc/tools.rb +++ /dev/null @@ -1,26 +0,0 @@ -shared_examples 'misc::graphicsmagick' do - it "should include graphicsmagick" do - expect(file("/usr/bin/gm")).to be_executable - end -end - -shared_examples 'misc::imagemagick' do - it "should include imagemagick" do - expect(file("/usr/bin/convert")).to be_executable - end -end - -shared_examples 'misc::graphviz' do - it "should include graphviz" do - expect(file("/usr/bin/dot")).to be_executable - end -end - -shared_examples 'misc::letsencrypt' do - it "should include letsencrypt" do - expect(file("/usr/bin/letsencrypt")).to be_executable - end - it "should include certbot" do - expect(file("/usr/bin/certbot")).to be_executable - end -end \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/serverspec/.bundle/config b/tests/serverspec/.bundle/config new file mode 100644 index 000000000..b81abe028 --- /dev/null +++ b/tests/serverspec/.bundle/config @@ -0,0 +1,3 @@ +--- +BUNDLE_PATH: vendor +BUNDLE_DISABLE_SHARED_GEMS: '1' diff --git a/test/.dockerignore b/tests/serverspec/.dockerignore similarity index 100% rename from test/.dockerignore rename to tests/serverspec/.dockerignore diff --git a/test/Gemfile b/tests/serverspec/Gemfile similarity index 100% rename from test/Gemfile rename to tests/serverspec/Gemfile diff --git a/test/Gemfile.lock b/tests/serverspec/Gemfile.lock similarity index 54% rename from test/Gemfile.lock rename to tests/serverspec/Gemfile.lock index afa792f28..73a9e107a 100644 --- a/test/Gemfile.lock +++ b/tests/serverspec/Gemfile.lock @@ -2,40 +2,40 @@ GEM remote: https://rubygems.org/ specs: diff-lcs (1.2.5) - docker-api (1.28.0) + docker-api (1.32.1) excon (>= 0.38.0) json - excon (0.49.0) - json (1.8.3) - multi_json (1.11.2) + excon (0.54.0) + json (2.0.2) + multi_json (1.12.1) net-scp (1.2.1) net-ssh (>= 2.6.5) - net-ssh (3.1.1) + net-ssh (3.2.0) net-telnet (0.1.1) - rake (11.1.2) - rspec (3.4.0) - rspec-core (~> 3.4.0) - rspec-expectations (~> 3.4.0) - rspec-mocks (~> 3.4.0) - rspec-core (3.4.4) - rspec-support (~> 3.4.0) - rspec-expectations (3.4.0) + rake (11.3.0) + rspec (3.5.0) + rspec-core (~> 3.5.0) + rspec-expectations (~> 3.5.0) + rspec-mocks (~> 3.5.0) + rspec-core (3.5.4) + rspec-support (~> 3.5.0) + rspec-expectations (3.5.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.4.0) + rspec-support (~> 3.5.0) rspec-its (1.2.0) rspec-core (>= 3.0.0) rspec-expectations (>= 3.0.0) - rspec-mocks (3.4.1) + rspec-mocks (3.5.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.4.0) - rspec-support (3.4.1) - serverspec (2.31.1) + rspec-support (~> 3.5.0) + rspec-support (3.5.0) + serverspec (2.37.2) multi_json rspec (~> 3.0) rspec-its specinfra (~> 2.53) - sfl (2.2) - specinfra (2.56.1) + sfl (2.3) + specinfra (2.66.0) net-scp net-ssh (>= 2.7, < 4.0) net-telnet diff --git a/test/Rakefile b/tests/serverspec/Rakefile similarity index 100% rename from test/Rakefile rename to tests/serverspec/Rakefile diff --git a/tests/serverspec/__init__.py b/tests/serverspec/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/conf/app/php-test.php b/tests/serverspec/conf/app/php-test.php similarity index 100% rename from test/conf/app/php-test.php rename to tests/serverspec/conf/app/php-test.php diff --git a/test/conf/app/pi-number.html b/tests/serverspec/conf/app/pi-number.html similarity index 100% rename from test/conf/app/pi-number.html rename to tests/serverspec/conf/app/pi-number.html diff --git a/tests/serverspec/conf/loop-entrypoint.sh b/tests/serverspec/conf/loop-entrypoint.sh new file mode 100644 index 000000000..6c64f290d --- /dev/null +++ b/tests/serverspec/conf/loop-entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/sh +tail -f /dev/null diff --git a/test/spec/collection/apache.rb b/tests/serverspec/spec/collection/apache.rb similarity index 100% rename from test/spec/collection/apache.rb rename to tests/serverspec/spec/collection/apache.rb diff --git a/test/spec/collection/base-app.rb b/tests/serverspec/spec/collection/base-app.rb similarity index 100% rename from test/spec/collection/base-app.rb rename to tests/serverspec/spec/collection/base-app.rb diff --git a/test/spec/collection/base.rb b/tests/serverspec/spec/collection/base.rb similarity index 100% rename from test/spec/collection/base.rb rename to tests/serverspec/spec/collection/base.rb diff --git a/test/spec/collection/bootstrap.rb b/tests/serverspec/spec/collection/bootstrap.rb similarity index 100% rename from test/spec/collection/bootstrap.rb rename to tests/serverspec/spec/collection/bootstrap.rb diff --git a/test/spec/collection/certbot.rb b/tests/serverspec/spec/collection/certbot.rb similarity index 100% rename from test/spec/collection/certbot.rb rename to tests/serverspec/spec/collection/certbot.rb diff --git a/test/spec/collection/dovecot.rb b/tests/serverspec/spec/collection/dovecot.rb similarity index 100% rename from test/spec/collection/dovecot.rb rename to tests/serverspec/spec/collection/dovecot.rb diff --git a/test/spec/collection/hhvm.rb b/tests/serverspec/spec/collection/hhvm.rb similarity index 100% rename from test/spec/collection/hhvm.rb rename to tests/serverspec/spec/collection/hhvm.rb diff --git a/test/spec/collection/liquibase.rb b/tests/serverspec/spec/collection/liquibase.rb similarity index 100% rename from test/spec/collection/liquibase.rb rename to tests/serverspec/spec/collection/liquibase.rb diff --git a/test/spec/collection/nginx.rb b/tests/serverspec/spec/collection/nginx.rb similarity index 100% rename from test/spec/collection/nginx.rb rename to tests/serverspec/spec/collection/nginx.rb diff --git a/test/spec/collection/php-fpm5.rb b/tests/serverspec/spec/collection/php-fpm5.rb similarity index 100% rename from test/spec/collection/php-fpm5.rb rename to tests/serverspec/spec/collection/php-fpm5.rb diff --git a/test/spec/collection/php-fpm7.rb b/tests/serverspec/spec/collection/php-fpm7.rb similarity index 100% rename from test/spec/collection/php-fpm7.rb rename to tests/serverspec/spec/collection/php-fpm7.rb diff --git a/test/spec/collection/php-tools.rb b/tests/serverspec/spec/collection/php-tools.rb similarity index 100% rename from test/spec/collection/php-tools.rb rename to tests/serverspec/spec/collection/php-tools.rb diff --git a/test/spec/collection/php5.rb b/tests/serverspec/spec/collection/php5.rb similarity index 96% rename from test/spec/collection/php5.rb rename to tests/serverspec/spec/collection/php5.rb index a265636a0..691499d9d 100644 --- a/test/spec/collection/php5.rb +++ b/tests/serverspec/spec/collection/php5.rb @@ -13,6 +13,7 @@ include_examples 'misc::graphicsmagick' include_examples 'misc::imagemagick' + include_examples 'misc::ghostscript' end shared_examples 'collection::php5::production' do diff --git a/test/spec/collection/php7.rb b/tests/serverspec/spec/collection/php7.rb similarity index 95% rename from test/spec/collection/php7.rb rename to tests/serverspec/spec/collection/php7.rb index 129771a16..614d4481f 100644 --- a/test/spec/collection/php7.rb +++ b/tests/serverspec/spec/collection/php7.rb @@ -12,6 +12,7 @@ include_examples 'misc::graphicsmagick' include_examples 'misc::imagemagick' + include_examples 'misc::ghostscript' end shared_examples 'collection::php7::production' do diff --git a/test/spec/collection/postfix.rb b/tests/serverspec/spec/collection/postfix.rb similarity index 100% rename from test/spec/collection/postfix.rb rename to tests/serverspec/spec/collection/postfix.rb diff --git a/test/spec/collection/samson-deployment.rb b/tests/serverspec/spec/collection/samson-deployment.rb similarity index 100% rename from test/spec/collection/samson-deployment.rb rename to tests/serverspec/spec/collection/samson-deployment.rb diff --git a/test/spec/collection/sphinx.rb b/tests/serverspec/spec/collection/sphinx.rb similarity index 100% rename from test/spec/collection/sphinx.rb rename to tests/serverspec/spec/collection/sphinx.rb diff --git a/test/spec/collection/ssh.rb b/tests/serverspec/spec/collection/ssh.rb similarity index 100% rename from test/spec/collection/ssh.rb rename to tests/serverspec/spec/collection/ssh.rb diff --git a/test/spec/collection/varnish.rb b/tests/serverspec/spec/collection/varnish.rb similarity index 100% rename from test/spec/collection/varnish.rb rename to tests/serverspec/spec/collection/varnish.rb diff --git a/test/spec/collection/vsftp.rb b/tests/serverspec/spec/collection/vsftp.rb similarity index 100% rename from test/spec/collection/vsftp.rb rename to tests/serverspec/spec/collection/vsftp.rb diff --git a/test/spec/docker/ansible_spec.rb b/tests/serverspec/spec/docker/ansible_spec.rb similarity index 100% rename from test/spec/docker/ansible_spec.rb rename to tests/serverspec/spec/docker/ansible_spec.rb diff --git a/test/spec/docker/apache_spec.rb b/tests/serverspec/spec/docker/apache-dev_spec.rb similarity index 100% rename from test/spec/docker/apache_spec.rb rename to tests/serverspec/spec/docker/apache-dev_spec.rb diff --git a/test/spec/docker/php7_spec.rb b/tests/serverspec/spec/docker/apache_spec.rb similarity index 63% rename from test/spec/docker/php7_spec.rb rename to tests/serverspec/spec/docker/apache_spec.rb index f0e0a2926..704ba62c4 100644 --- a/test/spec/docker/php7_spec.rb +++ b/tests/serverspec/spec/docker/apache_spec.rb @@ -10,9 +10,6 @@ include_examples 'collection::bootstrap' include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php7::production' - include_examples 'collection::php-fpm7' - include_examples 'collection::php-fpm7::public' + include_examples 'collection::apache' end diff --git a/test/spec/docker/base-app_spec.rb b/tests/serverspec/spec/docker/base-app_spec.rb similarity index 100% rename from test/spec/docker/base-app_spec.rb rename to tests/serverspec/spec/docker/base-app_spec.rb diff --git a/test/spec/docker/base_spec.rb b/tests/serverspec/spec/docker/base_spec.rb similarity index 100% rename from test/spec/docker/base_spec.rb rename to tests/serverspec/spec/docker/base_spec.rb diff --git a/test/spec/docker/bootstrap_spec.rb b/tests/serverspec/spec/docker/bootstrap_spec.rb similarity index 100% rename from test/spec/docker/bootstrap_spec.rb rename to tests/serverspec/spec/docker/bootstrap_spec.rb diff --git a/test/spec/docker/certbot_spec.rb b/tests/serverspec/spec/docker/certbot_spec.rb similarity index 100% rename from test/spec/docker/certbot_spec.rb rename to tests/serverspec/spec/docker/certbot_spec.rb diff --git a/test/spec/docker/hhvm-apache_spec.rb b/tests/serverspec/spec/docker/hhvm-apache_spec.rb similarity index 100% rename from test/spec/docker/hhvm-apache_spec.rb rename to tests/serverspec/spec/docker/hhvm-apache_spec.rb diff --git a/test/spec/docker/hhvm-nginx_spec.rb b/tests/serverspec/spec/docker/hhvm-nginx_spec.rb similarity index 100% rename from test/spec/docker/hhvm-nginx_spec.rb rename to tests/serverspec/spec/docker/hhvm-nginx_spec.rb diff --git a/test/spec/docker/hhvm_spec.rb b/tests/serverspec/spec/docker/hhvm_spec.rb similarity index 100% rename from test/spec/docker/hhvm_spec.rb rename to tests/serverspec/spec/docker/hhvm_spec.rb diff --git a/test/spec/docker/liquibase_spec.rb b/tests/serverspec/spec/docker/liquibase_spec.rb similarity index 100% rename from test/spec/docker/liquibase_spec.rb rename to tests/serverspec/spec/docker/liquibase_spec.rb diff --git a/test/spec/docker/mail-sandbox_spec.rb b/tests/serverspec/spec/docker/mail-sandbox_spec.rb similarity index 100% rename from test/spec/docker/mail-sandbox_spec.rb rename to tests/serverspec/spec/docker/mail-sandbox_spec.rb diff --git a/test/spec/docker/nginx_spec.rb b/tests/serverspec/spec/docker/nginx-dev_spec.rb similarity index 100% rename from test/spec/docker/nginx_spec.rb rename to tests/serverspec/spec/docker/nginx-dev_spec.rb diff --git a/test/spec/docker/php5_spec.rb b/tests/serverspec/spec/docker/nginx_spec.rb similarity index 63% rename from test/spec/docker/php5_spec.rb rename to tests/serverspec/spec/docker/nginx_spec.rb index 512512b4b..59a795608 100644 --- a/test/spec/docker/php5_spec.rb +++ b/tests/serverspec/spec/docker/nginx_spec.rb @@ -10,9 +10,7 @@ include_examples 'collection::bootstrap' include_examples 'collection::base' - include_examples 'collection::base-app' - include_examples 'collection::php5::production' - include_examples 'collection::php-fpm5' - include_examples 'collection::php-fpm5::public' + include_examples 'collection::nginx' + end diff --git a/tests/serverspec/spec/docker/php-apache-dev_spec.rb b/tests/serverspec/spec/docker/php-apache-dev_spec.rb new file mode 100644 index 000000000..79116c148 --- /dev/null +++ b/tests/serverspec/spec/docker/php-apache-dev_spec.rb @@ -0,0 +1,34 @@ +require 'serverspec' +require 'docker' +require 'spec_helper' + +describe "Dockerfile" do + before(:all) do + @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) + set :docker_image, @image.id + end + + include_examples 'collection::bootstrap' + include_examples 'collection::base' + include_examples 'collection::base-app' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php5::development' + include_examples 'collection::php-fpm5' + include_examples 'collection::php-fpm5::public' + else + include_examples 'collection::php7::development' + include_examples 'collection::php-fpm7' + include_examples 'collection::php-fpm7::public' + end + + include_examples 'collection::php-tools' + include_examples 'collection::apache' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php-fpm5::webserver-test::development' + else + include_examples 'collection::php-fpm7::webserver-test::development' + end + +end diff --git a/tests/serverspec/spec/docker/php-apache_spec.rb b/tests/serverspec/spec/docker/php-apache_spec.rb new file mode 100644 index 000000000..fc27e7fff --- /dev/null +++ b/tests/serverspec/spec/docker/php-apache_spec.rb @@ -0,0 +1,32 @@ +require 'serverspec' +require 'docker' +require 'spec_helper' + +describe "Dockerfile" do + before(:all) do + @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) + set :docker_image, @image.id + end + + include_examples 'collection::bootstrap' + include_examples 'collection::base' + include_examples 'collection::base-app' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php5::production' + include_examples 'collection::php-fpm5' + include_examples 'collection::php-fpm5::local-only' + else + include_examples 'collection::php7::production' + include_examples 'collection::php-fpm7' + include_examples 'collection::php-fpm7::local-only' + end + + include_examples 'collection::apache' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php-fpm5::webserver-test::production' + else + include_examples 'collection::php-fpm7::webserver-test::production' + end +end diff --git a/test/spec/docker/php7-apache-dev_spec.rb b/tests/serverspec/spec/docker/php-dev_spec.rb similarity index 52% rename from test/spec/docker/php7-apache-dev_spec.rb rename to tests/serverspec/spec/docker/php-dev_spec.rb index 499dcc6e6..43360b3a2 100644 --- a/test/spec/docker/php7-apache-dev_spec.rb +++ b/tests/serverspec/spec/docker/php-dev_spec.rb @@ -11,11 +11,17 @@ include_examples 'collection::bootstrap' include_examples 'collection::base' include_examples 'collection::base-app' - include_examples 'collection::php7::development' - include_examples 'collection::php-fpm7' - include_examples 'collection::php-fpm7::public' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php5::development' + include_examples 'collection::php-fpm5' + include_examples 'collection::php-fpm5::public' + else + include_examples 'collection::php7::development' + include_examples 'collection::php-fpm7' + include_examples 'collection::php-fpm7::public' + end + include_examples 'collection::php-tools' - include_examples 'collection::apache' - include_examples 'collection::php-fpm7::webserver-test::development' end diff --git a/tests/serverspec/spec/docker/php-nginx-dev_spec.rb b/tests/serverspec/spec/docker/php-nginx-dev_spec.rb new file mode 100644 index 000000000..dfb49c29d --- /dev/null +++ b/tests/serverspec/spec/docker/php-nginx-dev_spec.rb @@ -0,0 +1,34 @@ +require 'serverspec' +require 'docker' +require 'spec_helper' + +describe "Dockerfile" do + before(:all) do + @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) + set :docker_image, @image.id + end + + include_examples 'collection::bootstrap' + include_examples 'collection::base' + include_examples 'collection::base-app' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php5::development' + include_examples 'collection::php-fpm5' + include_examples 'collection::php-fpm5::public' + else + include_examples 'collection::php7::development' + include_examples 'collection::php-fpm7' + include_examples 'collection::php-fpm7::public' + end + + include_examples 'collection::php-tools' + include_examples 'collection::nginx' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php-fpm5::webserver-test::development' + else + include_examples 'collection::php-fpm7::webserver-test::development' + end + +end diff --git a/tests/serverspec/spec/docker/php-nginx_spec.rb b/tests/serverspec/spec/docker/php-nginx_spec.rb new file mode 100644 index 000000000..294446e39 --- /dev/null +++ b/tests/serverspec/spec/docker/php-nginx_spec.rb @@ -0,0 +1,33 @@ +require 'serverspec' +require 'docker' +require 'spec_helper' + +describe "Dockerfile" do + before(:all) do + @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) + set :docker_image, @image.id + end + + include_examples 'collection::bootstrap' + include_examples 'collection::base' + include_examples 'collection::base-app' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php5::production' + include_examples 'collection::php-fpm5' + include_examples 'collection::php-fpm5::local-only' + else + include_examples 'collection::php7::production' + include_examples 'collection::php-fpm7' + include_examples 'collection::php-fpm7::local-only' + end + + include_examples 'collection::nginx' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php-fpm5::webserver-test::production' + else + include_examples 'collection::php-fpm7::webserver-test::production' + end + +end diff --git a/tests/serverspec/spec/docker/php_spec.rb b/tests/serverspec/spec/docker/php_spec.rb new file mode 100644 index 000000000..098c60867 --- /dev/null +++ b/tests/serverspec/spec/docker/php_spec.rb @@ -0,0 +1,25 @@ +require 'serverspec' +require 'docker' +require 'spec_helper' + +describe "Dockerfile" do + before(:all) do + @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] }) + set :docker_image, @image.id + end + + include_examples 'collection::bootstrap' + include_examples 'collection::base' + include_examples 'collection::base-app' + + if ($testConfiguration[:php] == 5) + include_examples 'collection::php5::production' + include_examples 'collection::php-fpm5' + include_examples 'collection::php-fpm5::public' + else + include_examples 'collection::php7::production' + include_examples 'collection::php-fpm7' + include_examples 'collection::php-fpm7::public' + end + +end diff --git a/test/spec/docker/postfix_spec.rb b/tests/serverspec/spec/docker/postfix_spec.rb similarity index 100% rename from test/spec/docker/postfix_spec.rb rename to tests/serverspec/spec/docker/postfix_spec.rb diff --git a/test/spec/docker/samson-deployment_spec.rb b/tests/serverspec/spec/docker/samson-deployment_spec.rb similarity index 100% rename from test/spec/docker/samson-deployment_spec.rb rename to tests/serverspec/spec/docker/samson-deployment_spec.rb diff --git a/test/spec/docker/sphinx_spec.rb b/tests/serverspec/spec/docker/sphinx_spec.rb similarity index 100% rename from test/spec/docker/sphinx_spec.rb rename to tests/serverspec/spec/docker/sphinx_spec.rb diff --git a/test/spec/docker/ssh_spec.rb b/tests/serverspec/spec/docker/ssh_spec.rb similarity index 100% rename from test/spec/docker/ssh_spec.rb rename to tests/serverspec/spec/docker/ssh_spec.rb diff --git a/test/spec/docker/varnish_spec.rb b/tests/serverspec/spec/docker/varnish_spec.rb similarity index 100% rename from test/spec/docker/varnish_spec.rb rename to tests/serverspec/spec/docker/varnish_spec.rb diff --git a/test/spec/docker/vsftp_spec.rb b/tests/serverspec/spec/docker/vsftp_spec.rb similarity index 100% rename from test/spec/docker/vsftp_spec.rb rename to tests/serverspec/spec/docker/vsftp_spec.rb diff --git a/test/spec/image.rb b/tests/serverspec/spec/image.rb similarity index 100% rename from test/spec/image.rb rename to tests/serverspec/spec/image.rb diff --git a/test/spec/shared/apache/layout.rb b/tests/serverspec/spec/shared/apache/layout.rb similarity index 100% rename from test/spec/shared/apache/layout.rb rename to tests/serverspec/spec/shared/apache/layout.rb diff --git a/test/spec/shared/apache/listening.rb b/tests/serverspec/spec/shared/apache/listening.rb similarity index 100% rename from test/spec/shared/apache/listening.rb rename to tests/serverspec/spec/shared/apache/listening.rb diff --git a/test/spec/shared/apache/modules.rb b/tests/serverspec/spec/shared/apache/modules.rb similarity index 100% rename from test/spec/shared/apache/modules.rb rename to tests/serverspec/spec/shared/apache/modules.rb diff --git a/test/spec/shared/apache/service.rb b/tests/serverspec/spec/shared/apache/service.rb similarity index 100% rename from test/spec/shared/apache/service.rb rename to tests/serverspec/spec/shared/apache/service.rb diff --git a/test/spec/shared/base-app/application-user.rb b/tests/serverspec/spec/shared/base-app/application-user.rb similarity index 100% rename from test/spec/shared/base-app/application-user.rb rename to tests/serverspec/spec/shared/base-app/application-user.rb diff --git a/test/spec/shared/base-app/layout.rb b/tests/serverspec/spec/shared/base-app/layout.rb similarity index 100% rename from test/spec/shared/base-app/layout.rb rename to tests/serverspec/spec/shared/base-app/layout.rb diff --git a/test/spec/shared/base-app/locale.rb b/tests/serverspec/spec/shared/base-app/locale.rb similarity index 100% rename from test/spec/shared/base-app/locale.rb rename to tests/serverspec/spec/shared/base-app/locale.rb diff --git a/test/spec/shared/base-app/packages.rb b/tests/serverspec/spec/shared/base-app/packages.rb similarity index 100% rename from test/spec/shared/base-app/packages.rb rename to tests/serverspec/spec/shared/base-app/packages.rb diff --git a/test/spec/shared/base/layout.rb b/tests/serverspec/spec/shared/base/layout.rb similarity index 100% rename from test/spec/shared/base/layout.rb rename to tests/serverspec/spec/shared/base/layout.rb diff --git a/test/spec/shared/base/packages.rb b/tests/serverspec/spec/shared/base/packages.rb similarity index 100% rename from test/spec/shared/base/packages.rb rename to tests/serverspec/spec/shared/base/packages.rb diff --git a/test/spec/shared/base/supervisor.rb b/tests/serverspec/spec/shared/base/supervisor.rb similarity index 100% rename from test/spec/shared/base/supervisor.rb rename to tests/serverspec/spec/shared/base/supervisor.rb diff --git a/test/spec/shared/base/syslog-ng.rb b/tests/serverspec/spec/shared/base/syslog-ng.rb similarity index 100% rename from test/spec/shared/base/syslog-ng.rb rename to tests/serverspec/spec/shared/base/syslog-ng.rb diff --git a/test/spec/shared/bootstrap/ansible.rb b/tests/serverspec/spec/shared/bootstrap/ansible.rb similarity index 100% rename from test/spec/shared/bootstrap/ansible.rb rename to tests/serverspec/spec/shared/bootstrap/ansible.rb diff --git a/test/spec/shared/bootstrap/distribution.rb b/tests/serverspec/spec/shared/bootstrap/distribution.rb similarity index 100% rename from test/spec/shared/bootstrap/distribution.rb rename to tests/serverspec/spec/shared/bootstrap/distribution.rb diff --git a/test/spec/shared/bootstrap/layout.rb b/tests/serverspec/spec/shared/bootstrap/layout.rb similarity index 100% rename from test/spec/shared/bootstrap/layout.rb rename to tests/serverspec/spec/shared/bootstrap/layout.rb diff --git a/test/spec/shared/bootstrap/toolchain.rb b/tests/serverspec/spec/shared/bootstrap/toolchain.rb similarity index 100% rename from test/spec/shared/bootstrap/toolchain.rb rename to tests/serverspec/spec/shared/bootstrap/toolchain.rb diff --git a/test/spec/shared/certbot/layout.rb b/tests/serverspec/spec/shared/certbot/layout.rb similarity index 100% rename from test/spec/shared/certbot/layout.rb rename to tests/serverspec/spec/shared/certbot/layout.rb diff --git a/test/spec/shared/dovecot/layout.rb b/tests/serverspec/spec/shared/dovecot/layout.rb similarity index 100% rename from test/spec/shared/dovecot/layout.rb rename to tests/serverspec/spec/shared/dovecot/layout.rb diff --git a/test/spec/shared/dovecot/listening.rb b/tests/serverspec/spec/shared/dovecot/listening.rb similarity index 100% rename from test/spec/shared/dovecot/listening.rb rename to tests/serverspec/spec/shared/dovecot/listening.rb diff --git a/test/spec/shared/dovecot/service.rb b/tests/serverspec/spec/shared/dovecot/service.rb similarity index 100% rename from test/spec/shared/dovecot/service.rb rename to tests/serverspec/spec/shared/dovecot/service.rb diff --git a/test/spec/shared/hhvm/layout.rb b/tests/serverspec/spec/shared/hhvm/layout.rb similarity index 100% rename from test/spec/shared/hhvm/layout.rb rename to tests/serverspec/spec/shared/hhvm/layout.rb diff --git a/test/spec/shared/hhvm/listening.rb b/tests/serverspec/spec/shared/hhvm/listening.rb similarity index 100% rename from test/spec/shared/hhvm/listening.rb rename to tests/serverspec/spec/shared/hhvm/listening.rb diff --git a/test/spec/shared/hhvm/service.rb b/tests/serverspec/spec/shared/hhvm/service.rb similarity index 100% rename from test/spec/shared/hhvm/service.rb rename to tests/serverspec/spec/shared/hhvm/service.rb diff --git a/test/spec/shared/hhvm/version.rb b/tests/serverspec/spec/shared/hhvm/version.rb similarity index 100% rename from test/spec/shared/hhvm/version.rb rename to tests/serverspec/spec/shared/hhvm/version.rb diff --git a/test/spec/shared/liquibase/liquibase.rb b/tests/serverspec/spec/shared/liquibase/liquibase.rb similarity index 100% rename from test/spec/shared/liquibase/liquibase.rb rename to tests/serverspec/spec/shared/liquibase/liquibase.rb diff --git a/tests/serverspec/spec/shared/misc/tools.rb b/tests/serverspec/spec/shared/misc/tools.rb new file mode 100644 index 000000000..4d5f25fe7 --- /dev/null +++ b/tests/serverspec/spec/shared/misc/tools.rb @@ -0,0 +1,54 @@ +shared_examples 'misc::graphicsmagick' do + it "should include graphicsmagick" do + expect(file("/usr/bin/gm")).to be_executable + end + + describe command('/usr/bin/gm -version') do + its(:stdout) { should match %r!GraphicsMagick [0-9]+\.[0-9]+! } + its(:exit_status) { should eq 0 } + end +end + +shared_examples 'misc::imagemagick' do + it "should include imagemagick" do + expect(file("/usr/bin/convert")).to be_executable + end + + describe command('/usr/bin/convert --version') do + its(:stdout) { should match %r!Version: ImageMagick! } + its(:exit_status) { should eq 0 } + end +end + +shared_examples 'misc::ghostscript' do + it "should include ghostscript" do + expect(file("/usr/bin/gs")).to be_executable + end + + describe command('/usr/bin/gs --version') do + its(:stdout) { should match %r![0-9]+\.[0-9]+! } + its(:exit_status) { should eq 0 } + end +end + +shared_examples 'misc::graphviz' do + it "should include graphviz" do + expect(file("/usr/bin/dot")).to be_executable + end + + describe command('/usr/bin/dot -V') do + its(:stderr) { should match %r!graphviz version! } + its(:exit_status) { should eq 0 } + end +end + +shared_examples 'misc::letsencrypt' do + it "should include certbot" do + expect(file("/usr/bin/certbot")).to be_executable + end + + describe command('/usr/bin/certbot --version') do + its(:stderr) { should match %r!certbot! } + its(:exit_status) { should eq 0 } + end +end diff --git a/test/spec/shared/nginx/layout.rb b/tests/serverspec/spec/shared/nginx/layout.rb similarity index 100% rename from test/spec/shared/nginx/layout.rb rename to tests/serverspec/spec/shared/nginx/layout.rb diff --git a/test/spec/shared/nginx/listening.rb b/tests/serverspec/spec/shared/nginx/listening.rb similarity index 100% rename from test/spec/shared/nginx/listening.rb rename to tests/serverspec/spec/shared/nginx/listening.rb diff --git a/test/spec/shared/nginx/service.rb b/tests/serverspec/spec/shared/nginx/service.rb similarity index 100% rename from test/spec/shared/nginx/service.rb rename to tests/serverspec/spec/shared/nginx/service.rb diff --git a/test/spec/shared/php/cli.rb b/tests/serverspec/spec/shared/php/cli.rb similarity index 100% rename from test/spec/shared/php/cli.rb rename to tests/serverspec/spec/shared/php/cli.rb diff --git a/test/spec/shared/php/composer.rb b/tests/serverspec/spec/shared/php/composer.rb similarity index 100% rename from test/spec/shared/php/composer.rb rename to tests/serverspec/spec/shared/php/composer.rb diff --git a/test/spec/shared/php/configuration.rb b/tests/serverspec/spec/shared/php/configuration.rb similarity index 100% rename from test/spec/shared/php/configuration.rb rename to tests/serverspec/spec/shared/php/configuration.rb diff --git a/test/spec/shared/php/fpm.rb b/tests/serverspec/spec/shared/php/fpm.rb similarity index 100% rename from test/spec/shared/php/fpm.rb rename to tests/serverspec/spec/shared/php/fpm.rb diff --git a/test/spec/shared/php/layout.rb b/tests/serverspec/spec/shared/php/layout.rb similarity index 100% rename from test/spec/shared/php/layout.rb rename to tests/serverspec/spec/shared/php/layout.rb diff --git a/test/spec/shared/php/modules.rb b/tests/serverspec/spec/shared/php/modules.rb similarity index 100% rename from test/spec/shared/php/modules.rb rename to tests/serverspec/spec/shared/php/modules.rb diff --git a/test/spec/shared/php/pear.rb b/tests/serverspec/spec/shared/php/pear.rb similarity index 100% rename from test/spec/shared/php/pear.rb rename to tests/serverspec/spec/shared/php/pear.rb diff --git a/test/spec/shared/php/service.rb b/tests/serverspec/spec/shared/php/service.rb similarity index 100% rename from test/spec/shared/php/service.rb rename to tests/serverspec/spec/shared/php/service.rb diff --git a/test/spec/shared/php/test.rb b/tests/serverspec/spec/shared/php/test.rb similarity index 100% rename from test/spec/shared/php/test.rb rename to tests/serverspec/spec/shared/php/test.rb diff --git a/test/spec/shared/php/tools.rb b/tests/serverspec/spec/shared/php/tools.rb similarity index 100% rename from test/spec/shared/php/tools.rb rename to tests/serverspec/spec/shared/php/tools.rb diff --git a/test/spec/shared/php/version.rb b/tests/serverspec/spec/shared/php/version.rb similarity index 100% rename from test/spec/shared/php/version.rb rename to tests/serverspec/spec/shared/php/version.rb diff --git a/test/spec/shared/postfix/layout.rb b/tests/serverspec/spec/shared/postfix/layout.rb similarity index 100% rename from test/spec/shared/postfix/layout.rb rename to tests/serverspec/spec/shared/postfix/layout.rb diff --git a/test/spec/shared/postfix/listening.rb b/tests/serverspec/spec/shared/postfix/listening.rb similarity index 100% rename from test/spec/shared/postfix/listening.rb rename to tests/serverspec/spec/shared/postfix/listening.rb diff --git a/test/spec/shared/postfix/service.rb b/tests/serverspec/spec/shared/postfix/service.rb similarity index 100% rename from test/spec/shared/postfix/service.rb rename to tests/serverspec/spec/shared/postfix/service.rb diff --git a/test/spec/shared/samson-deployment/deployment.rb b/tests/serverspec/spec/shared/samson-deployment/deployment.rb similarity index 100% rename from test/spec/shared/samson-deployment/deployment.rb rename to tests/serverspec/spec/shared/samson-deployment/deployment.rb diff --git a/test/spec/shared/samson-deployment/layout.rb b/tests/serverspec/spec/shared/samson-deployment/layout.rb similarity index 100% rename from test/spec/shared/samson-deployment/layout.rb rename to tests/serverspec/spec/shared/samson-deployment/layout.rb diff --git a/test/spec/shared/samson-deployment/listening.rb b/tests/serverspec/spec/shared/samson-deployment/listening.rb similarity index 100% rename from test/spec/shared/samson-deployment/listening.rb rename to tests/serverspec/spec/shared/samson-deployment/listening.rb diff --git a/test/spec/shared/sphinx/sphinx.rb b/tests/serverspec/spec/shared/sphinx/sphinx.rb similarity index 100% rename from test/spec/shared/sphinx/sphinx.rb rename to tests/serverspec/spec/shared/sphinx/sphinx.rb diff --git a/test/spec/shared/ssh/layout.rb b/tests/serverspec/spec/shared/ssh/layout.rb similarity index 100% rename from test/spec/shared/ssh/layout.rb rename to tests/serverspec/spec/shared/ssh/layout.rb diff --git a/test/spec/shared/ssh/listening.rb b/tests/serverspec/spec/shared/ssh/listening.rb similarity index 100% rename from test/spec/shared/ssh/listening.rb rename to tests/serverspec/spec/shared/ssh/listening.rb diff --git a/test/spec/shared/ssh/service.rb b/tests/serverspec/spec/shared/ssh/service.rb similarity index 100% rename from test/spec/shared/ssh/service.rb rename to tests/serverspec/spec/shared/ssh/service.rb diff --git a/test/spec/shared/varnish/layout.rb b/tests/serverspec/spec/shared/varnish/layout.rb similarity index 100% rename from test/spec/shared/varnish/layout.rb rename to tests/serverspec/spec/shared/varnish/layout.rb diff --git a/test/spec/shared/varnish/listening.rb b/tests/serverspec/spec/shared/varnish/listening.rb similarity index 100% rename from test/spec/shared/varnish/listening.rb rename to tests/serverspec/spec/shared/varnish/listening.rb diff --git a/test/spec/shared/vendor/alpine.rb b/tests/serverspec/spec/shared/vendor/alpine.rb similarity index 100% rename from test/spec/shared/vendor/alpine.rb rename to tests/serverspec/spec/shared/vendor/alpine.rb diff --git a/test/spec/shared/vsftp/layout.rb b/tests/serverspec/spec/shared/vsftp/layout.rb similarity index 100% rename from test/spec/shared/vsftp/layout.rb rename to tests/serverspec/spec/shared/vsftp/layout.rb diff --git a/test/spec/shared/vsftp/listening.rb b/tests/serverspec/spec/shared/vsftp/listening.rb similarity index 100% rename from test/spec/shared/vsftp/listening.rb rename to tests/serverspec/spec/shared/vsftp/listening.rb diff --git a/test/spec/shared/vsftp/service.rb b/tests/serverspec/spec/shared/vsftp/service.rb similarity index 100% rename from test/spec/shared/vsftp/service.rb rename to tests/serverspec/spec/shared/vsftp/service.rb diff --git a/test/spec/shared/vsftp/user.rb b/tests/serverspec/spec/shared/vsftp/user.rb similarity index 100% rename from test/spec/shared/vsftp/user.rb rename to tests/serverspec/spec/shared/vsftp/user.rb diff --git a/test/spec/shared/web/test.rb b/tests/serverspec/spec/shared/web/test.rb similarity index 100% rename from test/spec/shared/web/test.rb rename to tests/serverspec/spec/shared/web/test.rb diff --git a/test/spec/spec_helper.rb b/tests/serverspec/spec/spec_helper.rb similarity index 69% rename from test/spec/spec_helper.rb rename to tests/serverspec/spec/spec_helper.rb index ed86f79da..6cb7f22fb 100644 --- a/test/spec/spec_helper.rb +++ b/tests/serverspec/spec/spec_helper.rb @@ -9,9 +9,15 @@ set :docker_container, ENV['DOCKER_IMAGE'] set :os, :family => ENV['OS_FAMILY'], :version => ENV['OS_VERSION'], :arch => 'x86_64' + Excon.defaults[:write_timeout] = 1000 Excon.defaults[:read_timeout] = 1000 +$dockerInfo = {} +$dockerInfo[:image] = ENV['DOCKER_IMAGE'] +$dockerInfo[:tag] = ENV['DOCKER_TAG'] +$dockerInfo[:dockerfile] = ENV['DOCKERFILE'] + $packageVersions = {} $packageVersions[:ansible] = %r!ansible 2.([0-9]\.?)+! $packageVersions[:ansiblePlaybook] = %r!ansible-playbook 2.([0-9]\.?)+! @@ -24,12 +30,30 @@ $testConfiguration[:ansiblePath] = "/usr/local/bin" end +$testConfiguration[:php] = 7 $testConfiguration[:phpXdebug] = true $testConfiguration[:phpApcu] = true $testConfiguration[:phpRedis] = true $testConfiguration[:phpMhash] = true $testConfiguration[:phpBlackfire] = false +if ((os[:family] == 'ubuntu' and os[:version] == '12.04') or + (os[:family] == 'ubuntu' and os[:version] == '14.04') or + (os[:family] == 'ubuntu' and os[:version] == '15.04') or + (os[:family] == 'ubuntu' and os[:version] == '15.10') or + (os[:family] == 'redhat' and os[:version] == '7') or + (os[:family] == 'debian' and os[:version] == '7') or + (os[:family] == 'debian' and os[:version] == '8') or + (os[:family] == 'alpine' and os[:version] == '8') or + ($dockerInfo[:tag].match('php5'))) + $testConfiguration[:php] = 5 +end + +if ($dockerInfo[:tag].match('php7')) + $testConfiguration[:php] = 7 +end + + if ENV['PHP_XDEBUG'] and ENV['PHP_XDEBUG'] == "0" $testConfiguration[:phpXdebug] = false end diff --git a/tests/testinfra/__init__.py b/tests/testinfra/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/testinfra/conf/app/php-test.php b/tests/testinfra/conf/app/php-test.php new file mode 100644 index 000000000..ed3eb6b80 --- /dev/null +++ b/tests/testinfra/conf/app/php-test.php @@ -0,0 +1,30 @@ +