diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 626411ab..36f03b10 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -21,7 +21,7 @@ jobs: msodbc_version: ["17", "18"] runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}:${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} + image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: - name: AZ CLI login run: az login --service-principal --username="${AZURE_CLIENT_ID:}" --password="${AZURE_CLIENT_SECRET:}" --tenant="${AZURE_TENANT_ID:}" @@ -41,7 +41,7 @@ jobs: DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} DBT_AZURESQL_UID: ${{ secrets.DBT_AZURESQL_UID }} DBT_AZURESQL_PWD: ${{ secrets.DBT_AZURESQL_PWD }} - run: python devops/wakeup_azure.py + run: python devops/scripts/wakeup_azure.py - name: Configure test users run: sqlcmd -b -I -i devops/create_sql_users.sql diff --git a/.github/workflows/integration-tests-sqlserver.yml b/.github/workflows/integration-tests-sqlserver.yml index a46aa8d3..37c7522e 100644 --- a/.github/workflows/integration-tests-sqlserver.yml +++ b/.github/workflows/integration-tests-sqlserver.yml @@ -22,29 +22,25 @@ jobs: sqlserver_version: ["2017", "2019", "2022"] runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}:${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} + image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} services: sqlserver: - image: mcr.microsoft.com/mssql/server:${{ matrix.sqlserver_version }}-latest + image: ghcr.io/${{ github.repository }}:server-${{ matrix.sqlserver_version }} env: ACCEPT_EULA: 'Y' SA_PASSWORD: 5atyaNadella - steps: - - uses: actions/checkout@v3 - - - name: Configure test users - run: sqlcmd -b -I -i devops/create_sql_users.sql - env: DBT_TEST_USER_1: DBT_TEST_USER_1 DBT_TEST_USER_2: DBT_TEST_USER_2 DBT_TEST_USER_3: DBT_TEST_USER_3 - SQLCMDUSER: SA - SQLCMDPASSWORD: 5atyaNadella - SQLCMDSERVER: sqlserver - SQLCMDDBNAME: msdb + steps: + - uses: actions/checkout@v3 - name: Install dependencies run: pip install -r dev_requirements.txt - name: Run functional tests run: pytest tests/functional --profile "${{ matrix.profile }}" + env: + DBT_TEST_USER_1: DBT_TEST_USER_1 + DBT_TEST_USER_2: DBT_TEST_USER_2 + DBT_TEST_USER_3: DBT_TEST_USER_3 diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 007deb68..49c5163c 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -3,13 +3,13 @@ name: Publish Docker images for CI/CD on: # yamllint disable-line rule:truthy push: paths: - - 'devops/Dockerfile' + - 'devops/**' - '.github/workflows/publish-docker.yml' branches: - 'master' jobs: - publish-docker: + publish-docker-client: strategy: matrix: python_version: ["3.7", "3.8", "3.9", "3.10"] @@ -29,13 +29,42 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push Docker images + - name: Build and push Docker image uses: docker/build-push-action@v3.1.1 with: context: . build-args: PYTHON_VERSION=${{ matrix.python_version }} - file: devops/Dockerfile + file: devops/CI.Dockerfile push: true platforms: linux/amd64 target: ${{ matrix.docker_target }} - tags: ghcr.io/${{ github.repository }}:${{ matrix.python_version }}-${{ matrix.docker_target }} + tags: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.docker_target }} + + publish-docker-server: + strategy: + matrix: + mssql_version: ["2017", "2019", "2022"] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v3.1.1 + with: + context: . + build-args: MSSQL_VERSION=${{ matrix.mssql_version }} + file: devops/server.Dockerfile + push: true + platforms: linux/amd64 + tags: ghcr.io/${{ github.repository }}:server-${{ matrix.mssql_version }} diff --git a/devops/Dockerfile b/devops/CI.Dockerfile similarity index 100% rename from devops/Dockerfile rename to devops/CI.Dockerfile diff --git a/devops/create_sql_users.sql b/devops/scripts/create_sql_users.sql similarity index 99% rename from devops/create_sql_users.sql rename to devops/scripts/create_sql_users.sql index a6bb8376..27c1a469 100644 --- a/devops/create_sql_users.sql +++ b/devops/scripts/create_sql_users.sql @@ -1,6 +1,8 @@ IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$(DBT_TEST_USER_1)') CREATE USER [$(DBT_TEST_USER_1)] WITHOUT LOGIN; + IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$(DBT_TEST_USER_2)') CREATE USER [$(DBT_TEST_USER_2)] WITHOUT LOGIN; + IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$(DBT_TEST_USER_3)') CREATE USER [$(DBT_TEST_USER_3)] WITHOUT LOGIN; diff --git a/devops/scripts/entrypoint.sh b/devops/scripts/entrypoint.sh new file mode 100755 index 00000000..d2f40463 --- /dev/null +++ b/devops/scripts/entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +/opt/init_scripts/init_db.sh & /opt/mssql/bin/sqlservr diff --git a/devops/scripts/init_db.sh b/devops/scripts/init_db.sh new file mode 100755 index 00000000..309c0816 --- /dev/null +++ b/devops/scripts/init_db.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +for i in {1..50}; +do + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${SA_PASSWORD}" -d msdb -I -i create_sql_users.sql + if [ $? -eq 0 ] + then + echo "create_sql_users.sql completed" + break + else + echo "not ready yet..." + sleep 1 + fi +done diff --git a/devops/wakeup_azure.py b/devops/scripts/wakeup_azure.py similarity index 100% rename from devops/wakeup_azure.py rename to devops/scripts/wakeup_azure.py diff --git a/devops/server.Dockerfile b/devops/server.Dockerfile new file mode 100644 index 00000000..2c58bac0 --- /dev/null +++ b/devops/server.Dockerfile @@ -0,0 +1,10 @@ +ARG MSSQL_VERSION="2022" +FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest + +USER root +RUN mkdir -p /opt/init_scripts +WORKDIR /opt/init_scripts +COPY scripts/* /opt/init_scripts/ + +USER mssql +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/devops/wait-for-it.sh b/devops/wait-for-it.sh deleted file mode 100755 index d990e0d3..00000000 --- a/devops/wait-for-it.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available - -WAITFORIT_cmdname=${0##*/} - -echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - else - echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" - fi - WAITFORIT_start_ts=$(date +%s) - while : - do - if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then - nc -z $WAITFORIT_HOST $WAITFORIT_PORT - WAITFORIT_result=$? - else - (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 - WAITFORIT_result=$? - fi - if [[ $WAITFORIT_result -eq 0 ]]; then - WAITFORIT_end_ts=$(date +%s) - echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" - break - fi - sleep 1 - done - return $WAITFORIT_result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $WAITFORIT_QUIET -eq 1 ]]; then - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - else - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - fi - WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT - wait $WAITFORIT_PID - WAITFORIT_RESULT=$? - if [[ $WAITFORIT_RESULT -ne 0 ]]; then - echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - fi - return $WAITFORIT_RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - WAITFORIT_hostport=(${1//:/ }) - WAITFORIT_HOST=${WAITFORIT_hostport[0]} - WAITFORIT_PORT=${WAITFORIT_hostport[1]} - shift 1 - ;; - --child) - WAITFORIT_CHILD=1 - shift 1 - ;; - -q | --quiet) - WAITFORIT_QUIET=1 - shift 1 - ;; - -s | --strict) - WAITFORIT_STRICT=1 - shift 1 - ;; - -h) - WAITFORIT_HOST="$2" - if [[ $WAITFORIT_HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - WAITFORIT_HOST="${1#*=}" - shift 1 - ;; - -p) - WAITFORIT_PORT="$2" - if [[ $WAITFORIT_PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - WAITFORIT_PORT="${1#*=}" - shift 1 - ;; - -t) - WAITFORIT_TIMEOUT="$2" - if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - WAITFORIT_TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - WAITFORIT_CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} -WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} -WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} -WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} - -# Check to see if timeout is from busybox? -WAITFORIT_TIMEOUT_PATH=$(type -p timeout) -WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) - -WAITFORIT_BUSYTIMEFLAG="" -if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then - WAITFORIT_ISBUSY=1 - # Check if busybox timeout uses -t flag - # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then - WAITFORIT_BUSYTIMEFLAG="-t" - fi -else - WAITFORIT_ISBUSY=0 -fi - -if [[ $WAITFORIT_CHILD -gt 0 ]]; then - wait_for - WAITFORIT_RESULT=$? - exit $WAITFORIT_RESULT -else - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - wait_for_wrapper - WAITFORIT_RESULT=$? - else - wait_for - WAITFORIT_RESULT=$? - fi -fi - -if [[ $WAITFORIT_CLI != "" ]]; then - if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then - echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" - exit $WAITFORIT_RESULT - fi - exec "${WAITFORIT_CLI[@]}" -else - exit $WAITFORIT_RESULT -fi diff --git a/docker-compose.yml b/docker-compose.yml index 0c2aea66..34a189ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,14 @@ services: sqlserver: - image: mcr.microsoft.com/mssql/server:2022-latest + build: + context: devops + dockerfile: server.Dockerfile + args: + MSSQL_VERSION: "2022" environment: SA_PASSWORD: "L0calTesting!" ACCEPT_EULA: "Y" - ports: - - "1433:1433" - - init-db: - image: mcr.microsoft.com/mssql/server:2022-latest - depends_on: - - sqlserver - restart: 'no' - volumes: - - ./devops:/mnt/scripts - environment: - SQLCMDUSER: SA - SQLCMDPASSWORD: "L0calTesting!" - SQLCMDSERVER: sqlserver - SQLCMDDBNAME: msdb env_file: - test.env - command: sh -c "/mnt/scripts/wait-for-it.sh sqlserver:1433 -t 60 -- sleep 5 && /opt/mssql-tools/bin/sqlcmd -b -I -i /mnt/scripts/create_sql_users.sql" + ports: + - "1433:1433"